diff --git a/docs/internal/agent-rules/toolchain.md b/docs/internal/agent-rules/toolchain.md index 153730ca1..36eae562a 100644 --- a/docs/internal/agent-rules/toolchain.md +++ b/docs/internal/agent-rules/toolchain.md @@ -162,7 +162,10 @@ screen, which means real devices and the slow lane. The lanes cover what this repo writes -- the stream module, the drivers' capture code against fakes, the envelope -- and the console lane runs a stand-in program that writes the same envelope from stills (`e2e/console/screen-capture.ts`), named by the fake -driver's `screenCaptureCommand`. A change to the Swift program is checked by +driver's `screenCaptureCommand`. On a Mac with Xcode, `swift test +--package-path vendor/screen-capture` runs the Swift tests (the envelope's +size changes, the log-line parser, the panel follower and its watchdog). A +change to the Swift program that reads a real screen is checked by hand on a machine with a simulator, where `node vendor/screen-capture/scripts/unframe.mjs out.264` turns the bytes it wrote into Annex-B and counts each tag, and `ffprobe out.264` says @@ -202,6 +205,7 @@ The machine can run a line only if: | Android | `command -v emulator adb` finds both | | iOS window | the iOS check; `mdfind "kMDItemCFBundleIdentifier == 'com.apple.dt.Devices'"` prints a path; `stat -f %Su /dev/console` is not `root` (someone is logged in); `pgrep -x DeviceHub` finds nothing (the test starts DeviceHub itself with `-shutdownStartedDevicesOnQuit NO` and quits it at the end). Run `scripts/slow-e2e.sh e2e/slow-ios-window.test.ts` | | Physical iOS | `uname -s` is `Darwin`; `SIMLOCK_PHYSICAL_IOS` is the UDID of an iPhone on USB (trusted, Developer Mode on, unlocked, Auto-Lock Never); `SIMLOCK_PHYSICAL_IOS_APP` is a `.app` signed for it. `/usr/libexec/PlistBuddy -c "Print :CFBundleShortVersionString" /Library/Developer/PrivateFrameworks/CoreDevice.framework/Resources/Info.plist` prints a version whose major is 642 or later (streaming a phone needs the CoreDevice of Xcode 27 or later; the framework is machine-wide and does not follow `xcode-select`). DeviceHub and Xcode are not running during the lane. The stream cases need the iPhone on iOS 27 or later; an older iPhone runs the refusal case instead. Run `scripts/slow-e2e.sh --physical e2e/slow-ios-physical.test.ts` | +| iOS foldable | the iOS check; `xcrun simctl list runtimes` lists iOS 27.1 or later and `xcrun simctl list devicetypes` lists `iPhone Duo`; `SIMLOCK_E2E_AGENT_DEVICE` names agent-device 0.21.35 or later; `command -v ffmpeg` finds it. Run `scripts/slow-e2e.sh e2e/slow-ios-foldable.test.ts` | | agent-device through simlock | `node -v` is 22.12.0 or later (agent-device's own minimum); the iOS check and `uname -m` is `arm64` (the fixture app is arm64 only), or the Android check; `SIMLOCK_E2E_AGENT_DEVICE` (in `.slow-lane.env`) is the absolute path of `bin/agent-device.mjs` in an install of the agent-device version `package.json` pins for `@agent-device/proxy` (`npm install -g agent-device@`, or a checkout at tag `v` built with `pnpm install && pnpm build && pnpm build:android`, the last building the Android snapshot helper); room for two devices: `node -p "os.totalmem() / 2**30"` prints at least 7 for iOS or 12 for Android (simlock's 4 GiB OS reserve plus two of its default 1.5 GiB iOS or 4 GiB Android per-device budgets). Run `scripts/slow-e2e.sh e2e/slow-agent-device.test.ts` or `scripts/slow-e2e.sh e2e/slow-agent-device-gateway.test.ts` | | Physical Android | `command -v adb` finds it; `adb devices -l` lists `$SIMLOCK_PHYSICAL_ANDROID` on a USB transport (a `usb:` field) in state `device`; `SIMLOCK_PHYSICAL_ANDROID_APK` names an APK. Run `scripts/slow-e2e.sh --physical e2e/slow-android-physical.test.ts` | | agent-device with a physical phone | the agent-device through simlock checks above (the Android or iOS one), and the Physical Android checks with `SIMLOCK_PHYSICAL_ANDROID`, or the Physical iOS checks with `SIMLOCK_PHYSICAL_IOS`; for iOS also `SIMLOCK_E2E_IOS_TEAM_ID` and `SIMLOCK_E2E_IOS_BUNDLE_ID` (in `.slow-lane.env`); set `SIMLOCK_E2E_IOS_PROVISIONING_PROFILE` and `SIMLOCK_E2E_IOS_SIGNING_IDENTITY` only for manual signing, which an Xcode-managed profile fails. Run `scripts/slow-e2e.sh --physical e2e/slow-agent-device-physical.test.ts` | diff --git a/e2e/slow-ios-foldable.test.ts b/e2e/slow-ios-foldable.test.ts new file mode 100644 index 000000000..8abdc672d --- /dev/null +++ b/e2e/slow-ios-foldable.test.ts @@ -0,0 +1,562 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { afterAll, describe, expect, it } from "vitest"; + +import { + DEVICE_READY_MS, + REPO_ROOT, + SCREEN_TAGS, + deviceStreamUrl, + freeLoopbackPort, + readScreenFrames, + waitFor, + withSharedDaemon, +} from "./helpers/index.js"; +import type { ScreenFrame } from "./helpers/screen-frames.js"; +import type { SharedTestEnv as TestEnv } from "./helpers/env.js"; + +const execFileAsync = promisify(execFile); + +const AGENT_DEVICE = process.env.SIMLOCK_E2E_AGENT_DEVICE; +const DUO_MODEL = "iPhone Duo"; +const MIN_RUNTIME = [27, 1] as const; +const UNFRAME = join(REPO_ROOT, "vendor", "screen-capture", "scripts", "unframe.mjs"); +const SWITCH_BOUND_MS = 5_000; +const STREAM_BOUND_MS = 240_000; +const FOLD_SESSION = "simlock-foldable-e2e"; +// why: a decoded picture counts as lit when its mean luminance is above 2% of white, the same line +// the capture's own panel read uses; a black panel reads 0. +const LIT_ABOVE = 0.02; +const COMPARE_WIDTH = 64; +const COMPARE_HEIGHT = 45; +const SAME_PICTURE_BELOW = 0.1; + +async function has(command: string, args: string[]): Promise { + return execFileAsync(command, args).then( + () => true, + () => false, + ); +} + +function atLeast(version: string): boolean { + const [major = 0, minor = 0] = version.split(".").map(Number); + return major > MIN_RUNTIME[0] || (major === MIN_RUNTIME[0] && minor >= MIN_RUNTIME[1]); +} + +// why: the slow lane has no fold of its own; without agent-device, an iPhone Duo device type, an +// iOS 27.1 runtime or ffmpeg the file cannot prove anything, and says which one is missing. +async function whyNotRunnable(): Promise { + if (process.platform !== "darwin") return "not macOS"; + if (AGENT_DEVICE === undefined) return "SIMLOCK_E2E_AGENT_DEVICE is not set"; + if (!(await has("ffmpeg", ["-version"]))) return "ffmpeg is not installed"; + const types = await execFileAsync("xcrun", ["simctl", "list", "devicetypes"]).catch(() => ({ + stdout: "", + })); + if (!types.stdout.includes(DUO_MODEL)) return "no iPhone Duo device type"; + const runtimes = await execFileAsync("xcrun", ["simctl", "list", "runtimes"]).catch(() => ({ + stdout: "", + })); + const found = [...runtimes.stdout.matchAll(/^iOS (\d+(?:\.\d+)?) /gm)].some((match) => + atLeast(match[1]!), + ); + return found ? undefined : "no iOS 27.1 or later runtime"; +} + +const notRunnable = await whyNotRunnable(); +if (notRunnable !== undefined) console.warn(`slow-ios-foldable skipped: ${notRunnable}`); + +async function within(work: Promise, bound: number, what: string): Promise { + let timer: NodeJS.Timeout | undefined; + const stalled = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`timed out after ${String(bound)} ms waiting for ${what}`)), + bound, + ); + }); + try { + return await Promise.race([work, stalled]); + } finally { + clearTimeout(timer); + } +} + +interface Meta { + readonly width: number; + readonly height: number; +} + +function metaOf(frame: ScreenFrame): Meta { + return JSON.parse(new TextDecoder().decode(frame.payload)) as Meta; +} + +// why: a stream with a size change is several decodable parts, one per meta frame; ffmpeg given the +// whole thing would scale every part to the first one's size and hide the very change under test. +function partsOf(frames: readonly ScreenFrame[]): ScreenFrame[][] { + const parts: ScreenFrame[][] = []; + for (const frame of frames) { + if (frame.tag === SCREEN_TAGS.meta) parts.push([]); + parts.at(-1)?.push(frame); + } + return parts; +} + +function envelope(frames: readonly ScreenFrame[]): Buffer { + return Buffer.concat( + frames.map((frame) => { + const head = Buffer.alloc(5); + head.writeUInt32BE(frame.payload.length + 1, 0); + head[4] = frame.tag; + return Buffer.concat([head, frame.payload]); + }), + ); +} + +async function decodeLast( + part: readonly ScreenFrame[], + dir: string, + name: string, +): Promise { + const raw = join(dir, `${name}.bin`); + const annexB = join(dir, `${name}.264`); + await writeFile(raw, envelope(part)); + await execFileAsync("node", [UNFRAME, raw, annexB]); + const decoded = await execFileAsync( + "ffmpeg", + [ + "-v", + "error", + "-i", + annexB, + "-vf", + `scale=${String(COMPARE_WIDTH)}:${String(COMPARE_HEIGHT)},format=gray`, + "-f", + "rawvideo", + "-", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + const size = COMPARE_WIDTH * COMPARE_HEIGHT; + expect(decoded.stdout.length, `${name} did not decode a picture`).toBeGreaterThanOrEqual(size); + return decoded.stdout.subarray(decoded.stdout.length - size); +} + +function meanOf(gray: Buffer): number { + return gray.reduce((sum, value) => sum + value, 0) / gray.length / 255; +} + +async function screenshotGray( + deviceSet: string, + udid: string, + display: number, + dir: string, +): Promise { + const png = join(dir, `screenshot-${String(display)}.png`); + await execFileAsync("xcrun", [ + "simctl", + "--set", + deviceSet, + "io", + udid, + "screenshot", + "--type=png", + `--display=${String(display)}`, + png, + ]); + const decoded = await execFileAsync( + "ffmpeg", + [ + "-v", + "error", + "-i", + png, + "-vf", + `scale=${String(COMPARE_WIDTH)}:${String(COMPARE_HEIGHT)},format=gray`, + "-f", + "rawvideo", + "-", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + return decoded.stdout; +} + +function meanAbsoluteDifference(left: Buffer, right: Buffer): number { + let total = 0; + for (let index = 0; index < left.length; index++) total += Math.abs(left[index]! - right[index]!); + return total / left.length / 255; +} + +async function pidsOf(args: string[]): Promise { + const found = await execFileAsync("pgrep", args).catch(() => ({ stdout: "" })); + return found.stdout + .split("\n") + .filter((line) => line.trim() !== "") + .map(Number); +} + +// why: `pgrep -x` matches the program's own name only; a pattern match would also find the shell +// running this check. +const capturePids = () => pidsOf(["-x", "simlock-screen-capture"]); + +// why: the follower's children are `simctl spawn log stream|show` under a watchdog; matching +// the udid keeps another simulator's log out of the count. +const logChildren = (udid: string) => pidsOf(["-f", `simctl .*spawn ${udid} log (stream|show)`]); + +// why: the simulator's runtimes ship no `ps`; its processes are host processes whose parent is the +// simulator's `launchd_sim`, whose arguments name the udid, so the in-simulator `log stream` is that +// launchd's child. +async function simulatorLogStreamPids(udid: string): Promise { + const launchds = await pidsOf(["-f", `^launchd_sim .*/${udid}/`]); + const streams = await Promise.all( + launchds.map((pid) => pidsOf(["-P", String(pid), "-f", "^log stream"])), + ); + return streams.flat(); +} + +async function simulatorLogStreams(udid: string): Promise { + return (await simulatorLogStreamPids(udid)).length; +} + +// why: a stream that outlives its capture is named by its command line, so a failure says whose it is. +async function survivors(udid: string): Promise { + const pids = await simulatorLogStreamPids(udid); + if (pids.length === 0) return "none"; + const listed = await execFileAsync("ps", [ + "-o", + "pid=,etime=,command=", + "-p", + pids.join(","), + ]).catch(() => ({ stdout: pids.join(",") })); + return listed.stdout.trim(); +} + +interface Lease { + readonly id: string; + readonly deviceId: string; + readonly udid: string; + readonly deviceSet: string; +} + +async function duoOs(env: TestEnv): Promise { + const catalog = await env.cli(["catalog", "--json", "--platform", "ios"]); + expect(catalog.code, `catalog failed: ${catalog.stderr}`).toBe(0); + const platforms = ( + catalog.json as { + platforms: { platform: string; modelRuntimes: Record }[]; + } + ).platforms; + const runtimes = platforms.find((p) => p.platform === "ios")?.modelRuntimes[DUO_MODEL] ?? []; + const os = runtimes + .filter(atLeast) + .sort((a, b) => b.localeCompare(a, undefined, { numeric: true }))[0]; + expect(os, "the catalog pairs no iPhone Duo with an iOS 27.1 or later runtime").toBeDefined(); + return os!; +} + +describe.skipIf(notRunnable !== undefined)( + "iOS foldable simulator stream (real simulator)", + { tags: ["slow", "ios"] }, + () => { + let dir: string | undefined; + let teardown: (() => Promise) | undefined; + let started: + | Promise<{ env: TestEnv; baseUrl: string; authorization: string; lease: Lease }> + | undefined; + + // why: one lease serves every case, because leasing and booting a Duo is most of a case's time. + function setup() { + started ??= (async () => { + dir = await mkdtemp(join(tmpdir(), "simlock-foldable-")); + const port = await freeLoopbackPort(); + // why: the shared daemon belongs to the file, not to the first test: `withDaemon`'s env is + // torn down after the test that made it, which would leave the second test a stopped + // daemon, and the file's own `nuke` would start a new one that nothing stops. + const env = await withSharedDaemon({ + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port }, + // why: the file leases one Duo; a second one booting in the warm pool beside it is load + // on agent-device's 3 s reads of the leased one, and proves nothing. + warmPool: { enabled: false }, + capabilities: { + stream: { bitrateKbps: 1500, enabled: true, fps: 30, maxDimension: 720 }, + }, + }, + driver: "real", + }); + teardown = async () => { + try { + await env + .cli(["nuke", "--delete-devices", "--yes"], { timeout: 120_000 }) + .catch(() => undefined); + } finally { + await env.teardown(); + } + }; + const granted = await env.cli( + [ + "lease", + "--platform", + "ios", + "--device", + DUO_MODEL, + "--os", + await duoOs(env), + "--agent-id", + "ios-foldable", + "--detach", + ], + { timeout: DEVICE_READY_MS }, + ); + expect(granted.code, `lease failed: ${granted.stdout} ${granted.stderr}`).toBe(0); + const grant = granted.json as { + device: { id: string; driverDeviceId: string }; + environment: Record; + lease: { id: string }; + }; + const lease: Lease = { + deviceId: grant.device.id, + deviceSet: grant.environment.SIMLOCK_IOS_DEVICE_SET as string, + id: grant.lease.id, + udid: grant.device.driverDeviceId, + }; + const created = await env.cli(["token", "create", "--role", "operator"]); + expect(created.code, created.stderr).toBe(0); + const authorization = `Bearer ${(created.json as { secret: string }).secret}`; + // why: agent-device finds the simulator in Simlock's own set through its scoped-set flag; + // a session on the lease's UDID is what `fold` needs. + await execFileAsync("node", [ + AGENT_DEVICE!, + "open", + "com.apple.Preferences", + "--platform", + "ios", + "--udid", + lease.udid, + "--ios-simulator-device-set", + lease.deviceSet, + "--session", + FOLD_SESSION, + ]); + return { authorization, baseUrl: `http://127.0.0.1:${String(port)}`, env, lease }; + })(); + return started; + } + + afterAll(async () => { + if (started !== undefined) { + await execFileAsync("node", [AGENT_DEVICE!, "close", "--session", FOLD_SESSION]).catch( + () => undefined, + ); + } + await teardown?.(); + if (dir !== undefined) await rm(dir, { force: true, recursive: true }); + }, 180_000); + + // why: agent-device reads the hinge and the panel table back to confirm a pose, each under a + // 3 s limit of its own; a simulator just booted on a loaded host misses it though the pose was + // sent, and sending it again is harmless. The failure is the test's when the retries end. + const UNCONFIRMED = /no panel table|no hinge-angle sample|returned no sample/; + const FOLD_ATTEMPTS = 3; + + // why: the 5 s bound counts from the start of the `agent-device fold` command, the first attempt's + // start when it is retried, so it holds the fold command and the stream's switch together. + async function fold(pose: "open" | "closed"): Promise { + const started = Date.now(); + for (let attempt = 1; ; attempt++) { + try { + await execFileAsync("node", [AGENT_DEVICE!, "fold", pose, "--session", FOLD_SESSION], { + timeout: 60_000, + }); + return started; + } catch (error: unknown) { + const failed = error as { stderr?: string; stdout?: string; message: string }; + const message = failed.stderr ?? failed.message; + if (attempt < FOLD_ATTEMPTS && UNCONFIRMED.test(message)) continue; + // why: its own failure is the test's failure, with its message (I10). + throw new Error(`agent-device fold ${pose} failed: ${message}`); + } + } + } + + async function open(signal: AbortSignal) { + const { authorization, baseUrl, env, lease } = await setup(); + const response = await within( + fetch(await deviceStreamUrl(baseUrl, env.home, lease.deviceId), { + headers: { authorization }, + signal, + }), + STREAM_BOUND_MS, + "the stream to open", + ); + const detail = response.status === 200 ? "" : await response.text().catch(() => ""); + expect(response.status, detail).toBe(200); + if (response.body === null) throw new Error("the stream answered without a body"); + return response.body.getReader(); + } + + it( + "a viewer of a leased Duo sees the cover folded and the inner panel unfolded, upright and lit, and the stream switches within 5 s of each fold", + { timeout: 900_000 }, + async () => { + const { lease } = await setup(); + const work = dir!; + await fold("closed"); + + const folded = new AbortController(); + const reader = await open(folded.signal); + const switches: { at: number; count: number }[] = []; + let settled = false; + const frames: ScreenFrame[] = []; + const reading = readScreenFrames(reader, (read) => { + frames.length = 0; + frames.push(...read); + const metas = read.filter((frame) => frame.tag === SCREEN_TAGS.meta).length; + if (metas > (switches.at(-1)?.count ?? 0)) + switches.push({ at: Date.now(), count: metas }); + const last = partsOf(read).at(-1) ?? []; + const pictures = last.filter( + (frame) => frame.tag === SCREEN_TAGS.keyframe || frame.tag === 0x03, + ); + return metas >= 3 && pictures.length >= 3 && settled; + }); + // why: a failing test ends the daemon, which cuts the stream; that rejection must not + // surface as an unhandled one. + reading.catch(() => undefined); + const metaCount = () => switches.at(-1)?.count ?? 0; + await waitFor(() => metaCount() >= 1 && frames.length >= 3, { + interval: 250, + label: "the folded stream's first pictures", + timeout: 120_000, + }); + const opening = partsOf(frames)[0]!; + expect(metaOf(opening[0]!)).toMatchObject({ height: 720, width: 494 }); + expect( + meanOf(await decodeLast(opening, work, "folded")), + "the folded cover is black", + ).toBeGreaterThan(LIT_ABOVE); + + for (const [pose, count] of [ + ["open", 2], + ["closed", 3], + ] as const) { + const foldedAt = await fold(pose); + await waitFor(() => metaCount() >= count, { + interval: 100, + label: `the meta frame after folding ${pose}`, + timeout: 30_000, + }); + const switched = switches.find((entry) => entry.count === count)!.at - foldedAt; + expect(switched, `the switch after folding ${pose} took too long`).toBeLessThan( + SWITCH_BOUND_MS, + ); + } + // why: the log names the panel before SpringBoard has painted it, so the first pictures + // after a switch can be black on a loaded host; the claim is that the panel is lit once it + // has been painted, which the last picture shows. + const litBy = Date.now() + 60_000; + for (;;) { + const last = partsOf(frames).at(-1) ?? []; + if (last.some((frame) => frame.tag === SCREEN_TAGS.keyframe || frame.tag === 0x03)) { + const gray = await decodeLast(last, work, "settle"); + if (meanOf(gray) > LIT_ABOVE) break; + } + if (Date.now() > litBy) break; + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + settled = true; + const all = await within(reading, 60_000, "the pictures after the last switch"); + folded.abort(); + const parts = partsOf(all); + expect(parts.map((part) => metaOf(part[0]!))).toMatchObject([ + { height: 720, width: 494 }, + { height: 506, width: 720 }, + { height: 720, width: 494 }, + ]); + for (const [index, part] of parts.entries()) { + expect( + meanOf(await decodeLast(part, work, `part-${String(index)}`)), + `part ${String(index)} is black`, + ).toBeGreaterThan(LIT_ABOVE); + } + + await fold("open"); + const unfolded = new AbortController(); + const inner = await open(unfolded.signal); + const pictures = await within( + readScreenFrames(inner, (read) => { + const last = partsOf(read).at(-1) ?? []; + return last.filter((frame) => frame.tag === 0x03).length >= 3; + }), + STREAM_BOUND_MS, + "the unfolded stream's pictures", + ); + unfolded.abort(); + const first = partsOf(pictures)[0]!; + expect(metaOf(first[0]!)).toMatchObject({ height: 506, width: 720 }); + const seen = await decodeLast(first, work, "unfolded"); + expect(meanOf(seen), "the unfolded inner panel is black").toBeGreaterThan(LIT_ABOVE); + const shot = await screenshotGray(lease.deviceSet, lease.udid, 3, work); + expect( + meanAbsoluteDifference(seen, shot), + "the stream's inner picture is not the screenshot of the inner panel", + ).toBeLessThan(SAME_PICTURE_BELOW); + }, + ); + + it( + "the capture's log children end within 5 s of its stdin closing, a SIGTERM and a SIGKILL", + { timeout: 600_000 }, + async () => { + const { lease } = await setup(); + for (const how of ["stdin closed", "SIGTERM", "SIGKILL"] as const) { + const viewer = new AbortController(); + const reader = await open(viewer.signal); + await within( + readScreenFrames(reader, (read) => + read.some((frame) => frame.tag === SCREEN_TAGS.keyframe), + ), + STREAM_BOUND_MS, + `${how}: the stream's key frame`, + ); + await waitFor(async () => (await logChildren(lease.udid)).length > 0, { + interval: 250, + label: `${how}: the capture started its log children`, + timeout: 30_000, + }); + await waitFor(async () => (await simulatorLogStreams(lease.udid)) > 0, { + interval: 250, + label: `${how}: the simulator runs the capture's log stream`, + timeout: 30_000, + }); + const [capture] = await capturePids(); + expect(capture, `${how}: no capture is running`).toBeDefined(); + if (how === "stdin closed") viewer.abort(); + else process.kill(capture!, how); + await waitFor(async () => (await logChildren(lease.udid)).length === 0, { + interval: 250, + label: `${how}: the log children of the ended capture`, + timeout: 5_000, + }); + await waitFor(async () => (await simulatorLogStreams(lease.udid)) === 0, { + interval: 250, + label: `${how}: the log stream inside the simulator of the ended capture`, + timeout: 5_000, + }).catch(async (error: unknown) => { + throw new Error( + `${(error as Error).message}; still running: ${await survivors(lease.udid)}`, + ); + }); + viewer.abort(); + await waitFor(async () => (await capturePids()).length === 0, { + interval: 250, + label: `${how}: the capture to be gone`, + timeout: 15_000, + }); + } + }, + ); + }, +); diff --git a/e2e/slow-ios-stream.test.ts b/e2e/slow-ios-stream.test.ts index 94892d849..43e0212f9 100644 --- a/e2e/slow-ios-stream.test.ts +++ b/e2e/slow-ios-stream.test.ts @@ -47,6 +47,15 @@ async function capturesRunning(): Promise { return found.stdout.split("\n").filter((line) => line.trim() !== "").length; } +// why: a one-panel device starts no panel follower, so nothing runs `log` inside its simulator; the +// pattern names the follower's children (`simctl spawn log stream|show`), not this shell. +async function panelLogChildren(): Promise { + const found = await execFileAsync("pgrep", ["-f", "simctl .*spawn .* log (stream|show)"]).catch( + () => ({ stdout: "" }), + ); + return found.stdout.split("\n").filter((line) => line.trim() !== "").length; +} + describe.skipIf(process.platform !== "darwin")( "iOS screen stream (real simulator)", { tags: ["slow", "ios"] }, @@ -126,7 +135,7 @@ describe.skipIf(process.platform !== "darwin")( }); const second = await open(new AbortController().signal); - await within( + const opening = await within( readFrames(second.reader, (read) => read.some((frame) => frame.tag === TAG.keyframe)), KEY_FRAME_BOUND, "the second stream's key frame", @@ -142,6 +151,11 @@ describe.skipIf(process.platform !== "darwin")( expect(held.length, "a still simulator's stream sent nothing for 35 s").toBeGreaterThan( 0, ); + expect( + [...opening, ...held].filter((frame) => frame.tag === TAG.meta).length, + "a one-panel device's stream sent more than one meta frame", + ).toBe(1); + expect(await panelLogChildren(), "a one-panel device started a panel follower").toBe(0); const after = await within( second.reader.read(), KEY_FRAME_BOUND, diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index eb6712d8a..d89cb8774 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -56,8 +56,9 @@ Tunnels (`device-tcp.ts`): - **A recovered slim device runs full but its record says `slim` (#174)** until its next `makeReady`. - **A cold slim lease takes a full cold boot.** MCP clients need `resetTimeoutOnProgress` or a longer timeout. - **Concurrent Simulator.app windows:** the second may not appear or warn. -- **A screen stream can freeze after a reboot under its lease.** The capture program checks the simulator is booted only when it starts, so after the health monitor reboots a crashed simulator the viewer sees the old picture until the Screen tab is reopened. -- **An iPhone rotated mid-stream keeps the first picture's scale**, as a simulator does: the scaler is sized once. An iPhone that stays connected but stops sending video shows the last picture until the lease ends. +- **A screen stream can freeze after a reboot under its lease.** The capture checks the simulator is booted only at start, so after a crash reboot the viewer sees the old picture until the Screen tab is reopened. +- **An iPhone's rotation has not run on hardware**; only a Duo's panel switch is proven. A silent iPhone shows its last picture until the lease ends. +- **A foldable's stream follows SpringBoard's `DisplayContentMode` log** (ADR 0075): another wording leaves it on the panel picked at open. Taps ignore the panel in use (#1535). - **A capture killed by SIGKILL cannot stop its CoreDevice stream**; CoreDevice is left to notice its client is gone. The same for a stop while `startMediaStream` is still running past its limit. - **A new Xcode major turns iPhone streams off** until `vendor/screen-capture/Stubs/` is checked against that release and the pin moves. -- **Untested on an iPhone:** DeviceHub or Xcode showing the same device during a stream; the holder's `devicectl` commands during one; the source ending after the first picture; `stop()` really stopping the stream. Nothing can observe CoreDevice's state, so the slow lane proves only that the process is gone and the device streams again. Two viewers of one lease ask for two streams (#602). +- **Untested on an iPhone:** DeviceHub or Xcode showing the same device during a stream; the holder's `devicectl` commands during one; the source ending after the first picture; `stop()` really stopping the stream. The slow lane proves only that the process is gone and the device streams again. Two viewers of one lease ask for two streams (#602). diff --git a/src/stream/definition.ts b/src/stream/definition.ts index 553ab9232..e5c5a3105 100644 --- a/src/stream/definition.ts +++ b/src/stream/definition.ts @@ -13,8 +13,10 @@ export interface StreamModuleConfig { const STREAM_HEADERS = { "simlock-stream-codec": "The codec of the body: h264 or mjpeg.", "simlock-stream-fps": "Frames per second.", - "simlock-stream-height": "Picture height in pixels.", - "simlock-stream-width": "Picture width in pixels.", + "simlock-stream-height": + "Picture height in pixels when the stream opens; a meta frame in the body gives each later size.", + "simlock-stream-width": + "Picture width in pixels when the stream opens; a meta frame in the body gives each later size.", }; export const streamModule: OptionalModuleDefinition = { diff --git a/vendor/screen-capture/NOTICE.md b/vendor/screen-capture/NOTICE.md index 8e3fded10..03f647109 100644 --- a/vendor/screen-capture/NOTICE.md +++ b/vendor/screen-capture/NOTICE.md @@ -45,25 +45,27 @@ otherwise all ours). Upstream keeps the same files in two targets, `Sources/SimNative/` and `Sources/SimNativeSupport/`, and their paths are given per row. -| File | Upstream path | Origin | -| ----------------------------------------------------------------------------------- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `simlock-screen-capture/SimFrameworks.swift` | `Sources/SimNative/` | theirs, unchanged | -| `simlock-screen-capture/FramebufferSurfaceSelector.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | -| `simlock-screen-capture/DeviceDisplaySizes.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | -| `simlock-screen-capture/FrameCapture.swift` | `Sources/SimNative/` | theirs, **one patch** (deviation 1) | -| `simlock-screen-capture/HIDInjector.swift` | `Sources/SimNative/` | theirs, **modified by us** — Apache-2.0, `MODIFIED FILE` header; the Duo, Device Hub, pose, Home, app switcher and Lock paths are deleted (deviation 3) | -| `SimlockCapture/H264Encoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | -| `SimlockCapture/VideoEncoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | -| `SimlockCapture/PixelBufferUtils.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | -| `SimlockCapture/Xcode.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | -| `simlock-screen-capture/InputMode.swift` | — | ours, entirely: the `--input` mode that drives `HIDInjector` from stdin | -| `SimlockCapture/Lifecycle.swift`, `SimlockCapture/Options.swift` | — | ours, entirely: the option parser, the envelope, the lifecycle, the scaler, the encode loop | -| `simlock-screen-capture/main.swift`, `simlock-screen-capture/SimulatorSource.swift` | — | ours, entirely: the simulator program and its framebuffer as a `ScreenSource` | -| `simlock-device-capture/*` (five files) | — | ours, entirely: the iPhone program — the version gate, the CoreDevice source, the RTP receiver, the decoder — rewritten from this repo's own PoC (ADR 0031) | -| `SimlockMediaBridge/*` | — | ours, entirely: the negotiator bridge; it carries the `idevice` HEVC feature string below | -| `Stubs/*` (three interfaces) | — | ours, entirely: hand-written declarations for the parts of CoreDevice we call | -| `scripts/unframe.mjs` | — | ours, entirely: a hand-run diagnostic | -| `Package.swift` | — | ours, entirely | +| File | Upstream path | Origin | +| ----------------------------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `simlock-screen-capture/SimFrameworks.swift` | `Sources/SimNative/` | theirs, unchanged | +| `simlock-screen-capture/FramebufferSurfaceSelector.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | +| `simlock-screen-capture/DeviceDisplaySizes.swift` | `Sources/SimNativeSupport/` | theirs, unchanged | +| `simlock-screen-capture/FrameCapture.swift` | `Sources/SimNative/` | theirs, **one patch** (deviation 1) | +| `simlock-screen-capture/HIDInjector.swift` | `Sources/SimNative/` | theirs, **modified by us** — Apache-2.0, `MODIFIED FILE` header; the Duo, Device Hub, pose, Home, app switcher and Lock paths are deleted (deviation 3) | +| `SimlockCapture/H264Encoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | +| `SimlockCapture/VideoEncoder.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | +| `SimlockCapture/PixelBufferUtils.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | +| `SimlockCapture/Xcode.swift` | `Sources/SimNative/` | theirs, **rewritten by us** — a modified file, Apache-2.0, `MODIFIED FILE` header | +| `simlock-screen-capture/InputMode.swift` | — | ours, entirely: the `--input` mode that drives `HIDInjector` from stdin | +| `SimlockCapture/Lifecycle.swift`, `SimlockCapture/Options.swift` | — | ours, entirely: the option parser, the envelope, the lifecycle, the scaler, the encode loop | +| `simlock-screen-capture/main.swift`, `simlock-screen-capture/SimulatorSource.swift` | — | ours, entirely: the simulator program and its framebuffer as a `ScreenSource` | +| `simlock-screen-capture/Panels.swift` | — | ours, entirely: which panel of a foldable simulator to stream, the follower of SpringBoard's panel moves, the watchdog for its `log` children, and the upright turn of the inner panel (ADR 0075) | +| `Tests/SimlockCaptureTests/*` | — | ours, entirely: the Swift tests (`swift test --package-path vendor/screen-capture`) | +| `simlock-device-capture/*` (five files) | — | ours, entirely: the iPhone program — the version gate, the CoreDevice source, the RTP receiver, the decoder — rewritten from this repo's own PoC (ADR 0031) | +| `SimlockMediaBridge/*` | — | ours, entirely: the negotiator bridge; it carries the `idevice` HEVC feature string below | +| `Stubs/*` (three interfaces) | — | ours, entirely: hand-written declarations for the parts of CoreDevice we call | +| `scripts/unframe.mjs` | — | ours, entirely: a hand-run diagnostic | +| `Package.swift` | — | ours, entirely | To diff any of the nine serve-sim files, use the command at the end with the upstream path from its row and the file's path under `Sources/`. diff --git a/vendor/screen-capture/Package.swift b/vendor/screen-capture/Package.swift index f841d4c10..01087b92b 100644 --- a/vendor/screen-capture/Package.swift +++ b/vendor/screen-capture/Package.swift @@ -45,6 +45,14 @@ let package = Package( path: "Sources/simlock-screen-capture", swiftSettings: [.swiftLanguageMode(.v5)] ), + // The Swift tests (`swift test --package-path vendor/screen-capture`): the lifecycle's wire + // output and the simulator program's panel follower. + .testTarget( + name: "SimlockCaptureTests", + dependencies: ["SimlockCapture", "simlock-screen-capture"], + path: "Tests/SimlockCaptureTests", + swiftSettings: [.swiftLanguageMode(.v5)] + ), // The one piece of Objective-C in the package, and the only reason it has two languages: // SwiftPM refuses a target that mixes Swift and Objective-C sources. .target( diff --git a/vendor/screen-capture/Sources/SimlockCapture/Lifecycle.swift b/vendor/screen-capture/Sources/SimlockCapture/Lifecycle.swift index 5db6bab6a..fac6ab41f 100644 --- a/vendor/screen-capture/Sources/SimlockCapture/Lifecycle.swift +++ b/vendor/screen-capture/Sources/SimlockCapture/Lifecycle.swift @@ -84,6 +84,9 @@ final class Session: @unchecked Sendable { private var writtenBytes = 0 private var _viewerGone = false var scaler: Scaler? + /// The picture size `scaler` was built for: a source whose picture changes size (a foldable + /// switching panels, an iPhone rotating) gets a scaler of its own for the new size. + var scalerInput = Dimensions(width: 0, height: 0) var scaledSize = Dimensions(width: 0, height: 0) /// The way out every end goes through: nil for a clean stop, else the one line to print last. let exitRequest = OneShot() @@ -182,10 +185,11 @@ func prepareForEncode( guard options.maxDimension > 0, max(width, height) > options.maxDimension else { return buffer } - // why: sized once, from the first picture — a rotation changes the long edge and the scale factor - // with it, and rebuilding the scaler mid-stream is out of scope for this task (see the module's - // known gaps). - if session.scaler == nil { + // why: rebuilt whenever the picture changes size, so a foldable that switches panels or an iPhone + // that rotates keeps its own proportions instead of being squeezed into the first picture's. + let input = Dimensions(width: width, height: height) + if session.scaler == nil || session.scalerInput != input { + session.scalerInput = input let factor = Double(options.maxDimension) / Double(max(width, height)) // Even dimensions: H.264 4:2:0 rejects an odd chroma plane. let targetWidth = max(2, Int(Double(width) * factor) & ~1) @@ -210,7 +214,7 @@ func encodeLoop(_ frames: AsyncStream, options: Options, session: Sessi let intervalNanos = UInt64(1_000_000_000 / Double(options.fps)) var lastSent: UInt64 = 0 var started = false - var metaEmitted = false + var metaSize: Dimensions? let encoder = H264Encoder(fps: options.fps, bitrate: options.bitrate) let jpegEncoder = VideoEncoder(quality: 0.7) for await frame in frames { @@ -223,9 +227,15 @@ func encodeLoop(_ frames: AsyncStream, options: Options, session: Sessi continue } // The meta frame rides the first picture because that is the first moment the encoded - // geometry (post-scale) is known. - if !metaEmitted { - metaEmitted = true + // geometry (post-scale) is known. A picture of another size gets a meta frame of its own, + // ahead of the fresh parameter sets and keyframe the encoder sends for it, so a viewer learns the + // new size before the first picture drawn at it. + let size = Dimensions(width: CVPixelBufferGetWidth(prepared), height: CVPixelBufferGetHeight(prepared)) + if metaSize != size { + if metaSize != nil { + log("picture size changed to \(size.width)x\(size.height)") + } + metaSize = size let meta: [String: Any] = [ "udid": options.udid, "codec": options.codec == "avcc" ? "h264" : "mjpeg", diff --git a/vendor/screen-capture/Sources/SimlockCapture/Options.swift b/vendor/screen-capture/Sources/SimlockCapture/Options.swift index 00c7c31d2..6da5810c1 100644 --- a/vendor/screen-capture/Sources/SimlockCapture/Options.swift +++ b/vendor/screen-capture/Sources/SimlockCapture/Options.swift @@ -96,6 +96,9 @@ public enum Tag: UInt8 { /// The video channel: a private duplicate of the stdout the program was started with. public enum Wire { + /// Where `emit` writes instead of the descriptor, set by a test. + nonisolated(unsafe) static var sink: ((Tag, Data) -> Void)? + /// Set by `claimStdout`, which every program's `main.swift` calls as its first statement. public private(set) static var fd: Int32 = -1 @@ -128,6 +131,10 @@ public enum Wire { out.append(tag.rawValue) out.append(data) session.recordWritten(out.count) + if let sink { + sink(tag, data) + return + } out.withUnsafeBytes { pointer in var written = 0 let base = pointer.baseAddress! diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/Panels.swift b/vendor/screen-capture/Sources/simlock-screen-capture/Panels.swift new file mode 100644 index 000000000..22ffb02c0 --- /dev/null +++ b/vendor/screen-capture/Sources/simlock-screen-capture/Panels.swift @@ -0,0 +1,520 @@ +// simlock's own, MIT-licensed (see NOTICE.md): which panel of a foldable simulator to stream (#1534). +import CoreVideo +import Foundation +import ImageIO +import SimlockCapture +import VideoToolbox + +enum PanelRole: String { + case cover, inner +} + +enum Panels { + /// The panel holding the picture, from one SpringBoard `DisplayContentMode` line, or nil when the + /// line names none. + /// + /// why: only a settled line counts. A line whose roles carry an arrow (`.inner:off→pri`) is a + /// transition that has just started and may still turn back; the swap line after it has none. + /// + /// why: the line must carry SpringBoard's own category tag. `log` opens its output with a header + /// that repeats the filter, `.cover:pri` and `.inner:pri` both, and that header is no move. + static func primary(inLogLine line: String) -> PanelRole? { + guard line.contains("[com.apple.SpringBoard:DisplayContentMode]") else { return nil } + let pattern = try! NSRegularExpression(pattern: #"\.(inner|cover):([a-z]+)(→)?"#) + let range = NSRange(line.startIndex..., in: line) + var primaries: [PanelRole] = [] + for match in pattern.matches(in: line, range: range) { + if match.range(at: 3).location != NSNotFound { return nil } + guard let role = Range(match.range(at: 1), in: line), let state = Range(match.range(at: 2), in: line) else { continue } + if line[state] == "pri", let named = PanelRole(rawValue: String(line[role])) { primaries.append(named) } + } + return primaries.count == 1 ? primaries[0] : nil + } +} + +/// One built-in touch panel of a simulator. +struct Panel: Equatable { + let name: String + /// The id `simctl io enumerate` lists the panel under; also its touch target. + let screenID: Int + let width: Int + let height: Int + /// Degrees the panel is mounted at relative to its framebuffer; the inner Duo panel's is 270, so + /// its framebuffer holds the picture on its side. + let nativeRotation: Int + /// Set only on a device with exactly two panels, where the smaller is the cover. + let role: PanelRole? +} + +/// Which of a device's panels are lit right now. +protocol PanelSampler: Sendable { + /// A panel that failed or was late to read is dark. + func lit(_ panels: [Panel]) async -> [Int: Bool] +} + +/// SpringBoard's panel moves, as log lines. +protocol PanelLog: AnyObject, Sendable { + /// This boot's moves, oldest first; empty when they could not be read within `timeout`. + func bootLines(timeout: TimeInterval) async -> [String] + /// Streams lines as they come; `onEnd` runs when the stream ends by itself. + func live(onLine: @escaping @Sendable (String) -> Void, onEnd: @escaping @Sendable () -> Void) + func stop() +} + +protocol PanelClock: Sendable { + func sleep(seconds: TimeInterval) async +} + +/// Picks the panel in use when it starts, then reports each move SpringBoard makes. +final class PanelFollower: @unchecked Sendable { + /// How long a start waits for this boot's log when the lit panel does not say which is in use. + static let bootReadSeconds: TimeInterval = 10 + /// How long without a line before the follower says it hears nothing. + static let silenceSeconds: TimeInterval = 30 + + private let panels: [Panel] + private let sampler: PanelSampler + private let source: PanelLog + private let clock: PanelClock + private let note: @Sendable (String) -> Void + private let lock = NSLock() + private var current: Panel? + private var onPanel: (@Sendable (Panel) -> Void)? + private var stopped = false + private var heardLine = false + private var heardMove = false + + static let toStderr: @Sendable (String) -> Void = { message in log(message) } + + init( + panels: [Panel], sampler: PanelSampler, log: PanelLog, clock: PanelClock, + note: @escaping @Sendable (String) -> Void = PanelFollower.toStderr + ) { + self.note = note + self.panels = panels + self.sampler = sampler + self.source = log + self.clock = clock + } + + /// Reports the first panel before it returns, then each change. + func start(onPanel: @escaping @Sendable (Panel) -> Void) async { + guard panels.count >= 2 else { return } + self.onPanel = onPanel + guard panels.count == 2, + let cover = panels.first(where: { $0.role == .cover }), + let inner = panels.first(where: { $0.role == .inner }) + else { + note("[panels] \(panels.count) panels; staying on the first") + report(panels[0]) + return + } + // why: the live log starts before the lit panel is read, so a fold during the read is not + // missed, and a move heard live is newer than anything read after it, so it wins. + startLive() + let lit = await sampler.lit(panels) + let coverLit = lit[cover.screenID] ?? false + let innerLit = lit[inner.screenID] ?? false + if !hasReported { report(innerLit && !coverLit ? inner : cover) } + Task { + await clock.sleep(seconds: Self.silenceSeconds) + if !isSilenceBroken { note("[panels] no panel move heard; staying on \(currentRole)") } + } + if coverLit == innerLit { + // why: a black app lights neither panel and a bright one lights both, so the lit panel + // does not say which one SpringBoard shows; this boot's last move does. + Task { + let lines = await source.bootLines(timeout: Self.bootReadSeconds) + if !heardMove, let role = lines.reversed().lazy.compactMap({ Panels.primary(inLogLine: $0) }).first, + let panel = panels.first(where: { $0.role == role }) { + report(panel) + } + } + } + } + + private var hasReported: Bool { lock.lock(); defer { lock.unlock() }; return current != nil } + private var isSilenceBroken: Bool { lock.lock(); defer { lock.unlock() }; return heardLine || stopped } + private var currentRole: String { lock.lock(); defer { lock.unlock() }; return current?.role?.rawValue ?? "?" } + + private func startLive() { + source.live( + onLine: { [weak self] line in self?.heard(line) }, + onEnd: { [weak self] in self?.liveEnded() } + ) + } + + private func heard(_ line: String) { + guard let role = Panels.primary(inLogLine: line) else { return } + lock.lock() + heardLine = true + heardMove = true + lock.unlock() + if let panel = panels.first(where: { $0.role == role }) { report(panel) } + } + + private func liveEnded() { + Task { + await clock.sleep(seconds: 1) + if !lock.withLock({ stopped }) { startLive() } + } + } + + func stop() { + lock.lock() + stopped = true + lock.unlock() + source.stop() + } + + /// Hands the panel on unless it is the one already reported. + private func report(_ panel: Panel) { + lock.lock() + let changed = current != panel && !stopped + if changed { current = panel } + lock.unlock() + if changed { onPanel?(panel) } + } +} + +/// Runs a command so that it ends when the process holding the other end of its stdin pipe ends, by +/// any way, SIGKILL included. +enum Watchdog { + // why: the command runs in the background with stdin off, and a second background job holds the + // pipe on fd 3; `cat` returns when every writer is gone, which is the moment the holder ended, + // and the command is killed then. `-9` because `simctl spawn … log` is only reliably ended by it. + // The reader is killed once the command ends first, so neither outlives the other. + // why: `then` runs from the main script once the command is gone, not from the reader that kills + // it, which the main script ends. Killing `simctl spawn` leaves the process it started inside the + // simulator to CoreSimulator, which on a loaded host takes longer than the caller can wait; + // `then` ends that process itself. + private static let script = """ + exec 3<&0 + "$@" 3<&- /dev/null; kill -9 "$child" 2>/dev/null; } & + reader=$! + wait "$child" + status=$? + eval "$WATCHDOG_THEN" + pkill -P "$reader" 2>/dev/null + kill "$reader" 2>/dev/null + exit "$status" + """ + + /// Not started; the caller keeps the `standardInput` pipe open for as long as the command should run. + /// `then` is a shell snippet that runs once the pipe has closed and the command has been killed. + static func process(running command: [String], then: String = "") -> Process { + let process = Process() + process.environment = ProcessInfo.processInfo.environment.merging(["WATCHDOG_THEN": then]) { _, new in new } + process.executableURL = URL(fileURLWithPath: "/bin/sh") + process.arguments = ["-c", script, "sh"] + command + process.standardInput = Pipe() + return process + } +} + +// MARK: - The device + +extension Panels { + /// SpringBoard's panel moves, narrowed to the lines that name the panel in use. + static let logPredicate = + #"process == "SpringBoard" AND category == "DisplayContentMode" AND (eventMessage CONTAINS ".cover:pri" OR eventMessage CONTAINS ".inner:pri")"# + + /// `xcrun simctl [--set ] `, the form every child of this file takes. + static func simctl(_ arguments: [String]) -> [String] { + ["/usr/bin/xcrun", "simctl"] + (FrameCapture.deviceSetPath.map { ["--set", $0] } ?? []) + arguments + } + + /// The device type's built-in touch panels, from its `capabilities.plist` and the ids `simctl io + /// enumerate` lists them under. Of two panels the smaller is the cover. + static func read(for device: NSObject, udid: String) -> [Panel] { + guard + let deviceType = device.perform(NSSelectorFromString("deviceType"))?.takeUnretainedValue() as? NSObject, + let bundle = deviceType.value(forKey: "bundlePath") as? String, + let data = try? Data(contentsOf: URL(fileURLWithPath: bundle).appendingPathComponent("Contents/Resources/capabilities.plist")), + let root = try? PropertyListSerialization.propertyList(from: data, format: nil) as? [String: Any] + else { return [] } + let capabilities = root["capabilities"] as? [String: Any] ?? root + let displays = capabilities["displays"] as? [[String: Any]] ?? [] + let builtIn = displays.filter { display in + display["displayType"] as? String == "integrated" + && (display["hasDigitizer"] as? NSNumber)?.boolValue == true + && !((display["chromeIdentifier"] as? String) ?? "").isEmpty + } + guard builtIn.count > 1 else { return [] } + let ids = screenIDs(inEnumerate: run(simctl(["io", udid, "enumerate"]), seconds: 3)) + var panels: [Panel] = builtIn.compactMap { display in + guard let name = display["deviceName"] as? String, let screenID = ids[name], + let width = (display["width"] as? NSNumber)?.intValue, + let height = (display["height"] as? NSNumber)?.intValue + else { return nil } + return Panel( + name: name, screenID: screenID, width: width, height: height, + nativeRotation: (display["nativeRotation"] as? NSNumber)?.intValue ?? 0, role: nil + ) + } + if panels.count == 2 { + panels.sort { $0.width * $0.height < $1.width * $1.height } + panels = [panels[0].with(role: .cover), panels[1].with(role: .inner)] + } + return panels + } + + /// Each screen's `Device Name` and `Screen ID` from `simctl io enumerate`. + static func screenIDs(inEnumerate text: String) -> [String: Int] { + var ids: [String: Int] = [:] + var screenID: Int? + for line in text.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + if trimmed.hasPrefix("Screen ID:") { + screenID = Int(trimmed.dropFirst("Screen ID:".count).trimmingCharacters(in: .whitespaces)) + } else if trimmed.hasPrefix("Device Name:"), let screenID { + ids[trimmed.dropFirst("Device Name:".count).trimmingCharacters(in: .whitespaces)] = screenID + } + } + return ids + } + + /// Runs a short command and returns what it printed, or nothing when it fails or outlives `seconds`. + static func run(_ command: [String], seconds: TimeInterval) -> String { + String(decoding: runData(command, seconds: seconds) ?? Data(), as: UTF8.self) + } + + static func runData(_ command: [String], seconds: TimeInterval) -> Data? { + let process = Process() + process.executableURL = URL(fileURLWithPath: command[0]) + process.arguments = Array(command.dropFirst()) + let output = Pipe() + process.standardOutput = output + process.standardError = FileHandle.nullDevice + process.standardInput = FileHandle.nullDevice + guard (try? process.run()) != nil else { return nil } + let cut = DispatchWorkItem { process.terminate() } + DispatchQueue.global().asyncAfter(deadline: .now() + seconds, execute: cut) + let data = output.fileHandleForReading.readDataToEndOfFile() + process.waitUntilExit() + cut.cancel() + return process.terminationStatus == 0 ? data : nil + } +} + +extension Panel { + func with(role: PanelRole) -> Panel { + Panel(name: name, screenID: screenID, width: width, height: height, nativeRotation: nativeRotation, role: role) + } + + /// Whether a framebuffer of this size is this panel's, whichever way it is turned. + func matches(width: Int, height: Int) -> Bool { + (self.width == width && self.height == height) || (self.width == height && self.height == width) + } +} + +/// Reads whether each panel shows a picture, from `simctl io screenshot` of it. +struct SimctlPanelSampler: PanelSampler { + let udid: String + /// Mean luminance above which a panel counts as lit. + static let litAbove = 0.02 + static let cutSeconds: TimeInterval = 3 + + func lit(_ panels: [Panel]) async -> [Int: Bool] { + await withTaskGroup(of: (Int, Bool).self) { group in + for panel in panels { + group.addTask { [udid] in + // why: a file, not `-`: simctl takes `-` as the name of a file in the working directory + // and writes nothing to stdout. + let file = NSTemporaryDirectory() + "simlock-panel-\(UUID().uuidString).png" + defer { try? FileManager.default.removeItem(atPath: file) } + let command = Panels.simctl(["io", udid, "screenshot", "--type=png", "--display=\(panel.screenID)", file]) + guard Panels.runData(command, seconds: Self.cutSeconds) != nil, + let png = FileManager.default.contents(atPath: file) + else { return (panel.screenID, false) } + return (panel.screenID, Self.meanLuminance(ofPNG: png) > Self.litAbove) + } + } + var answer: [Int: Bool] = [:] + for await (id, isLit) in group { answer[id] = isLit } + return answer + } + } + + /// 0 for black to 1 for white, over a 32x32 reduction of the picture. + static func meanLuminance(ofPNG data: Data) -> Double { + guard let source = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { return 0 } + let side = 32 + var pixels = [UInt8](repeating: 0, count: side * side) + let drew = pixels.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext( + data: buffer.baseAddress, width: side, height: side, bitsPerComponent: 8, bytesPerRow: side, + space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue + ) else { return false } + context.interpolationQuality = .medium + context.draw(image, in: CGRect(x: 0, y: 0, width: side, height: side)) + return true + } + guard drew else { return 0 } + return Double(pixels.reduce(0) { $0 + Int($1) }) / Double(pixels.count * 255) + } +} + +/// SpringBoard's log, read by `simctl spawn … log` children under a `Watchdog`. +final class SimctlPanelLog: PanelLog, @unchecked Sendable { + private let udid: String + private let lock = NSLock() + private var children: [Process] = [] + private var stopped = false + + init(udid: String) { self.udid = udid } + + /// The `log` command for one verb and the snippet that ends it inside the simulator, where it is a host + /// process whose parent is the simulator's `launchd_sim`. + func commands(_ verb: [String]) -> (log: [String], end: String) { + // why: a token of its own per command, so ending one (`show`) never ends another (`stream`); it is a + // clause that is always true and puts a name on the in-simulator command line. + let token = UUID().uuidString + let predicate = Panels.logPredicate + #" AND "\#(token)" != """# + return ( + Panels.simctl(["spawn", udid, "log"] + verb + ["--style", "compact", "--predicate", predicate]), + #"for l in $(pgrep -f "^launchd_sim .*/\#(udid)/"); do pkill -9 -P "$l" -f "\#(token)"; done"# + ) + } + + private func launch(_ verb: [String], stdout: Pipe) -> Process? { + let (log, end) = commands(verb) + let process = Watchdog.process(running: log, then: end) + process.standardOutput = stdout + process.standardError = FileHandle.nullDevice + lock.lock() + defer { lock.unlock() } + guard !stopped, (try? process.run()) != nil else { return nil } + children.append(process) + return process + } + + private func release(_ process: Process) { + // why: closing the pipe is what ends the child; the watchdog kills it when no writer is left. + try? (process.standardInput as? Pipe)?.fileHandleForWriting.close() + lock.lock() + children.removeAll { $0 === process } + lock.unlock() + } + + func bootLines(timeout: TimeInterval) async -> [String] { + let output = Pipe() + guard let process = launch(["show", "--last", "boot"], stdout: output) else { return [] } + let cut = DispatchWorkItem { [weak self] in self?.release(process) } + DispatchQueue.global().asyncAfter(deadline: .now() + timeout, execute: cut) + let data = output.fileHandleForReading.readDataToEndOfFile() + cut.cancel() + release(process) + return String(decoding: data, as: UTF8.self).split(separator: "\n").map(String.init) + } + + func live(onLine: @escaping @Sendable (String) -> Void, onEnd: @escaping @Sendable () -> Void) { + let output = Pipe() + guard let process = launch(["stream"], stdout: output) else { + onEnd() + return + } + let pending = LineBuffer(onLine) + output.fileHandleForReading.readabilityHandler = { handle in + let chunk = handle.availableData + if chunk.isEmpty { + handle.readabilityHandler = nil + return + } + pending.add(String(decoding: chunk, as: UTF8.self)) + } + process.terminationHandler = { [weak self] ended in + output.fileHandleForReading.readabilityHandler = nil + self?.release(ended) + onEnd() + } + } + + func stop() { + lock.lock() + stopped = true + let open = children + lock.unlock() + for process in open { release(process) } + } +} + +/// Cuts a stream of text into lines. +final class LineBuffer: @unchecked Sendable { + private let lock = NSLock() + private var text = "" + private let onLine: @Sendable (String) -> Void + + init(_ onLine: @escaping @Sendable (String) -> Void) { self.onLine = onLine } + + func add(_ chunk: String) { + lock.lock() + text += chunk + var lines = text.components(separatedBy: "\n") + text = String(lines.removeLast().suffix(4096)) + lock.unlock() + for line in lines { onLine(line) } + } +} + +struct SystemPanelClock: PanelClock { + func sleep(seconds: TimeInterval) async { + try? await Task.sleep(nanoseconds: UInt64(seconds * 1_000_000_000)) + } +} + +/// Turns a panel mounted on its side upright, on the GPU. +/// +/// Not thread-safe by itself: the capture calls it from its one callback queue. +final class PanelRotator { + private var session: VTPixelRotationSession? + private var pool: CVPixelBufferPool? + private var poolSize = (width: 0, height: 0) + private var rotation: CFString? + + /// The VideoToolbox rotation that undoes a panel's mounting, or nil for an upright panel. + static func rotation(undoing nativeRotation: Int) -> CFString? { + switch (nativeRotation % 360 + 360) % 360 { + case 90: return kVTRotation_CCW90 + case 180: return kVTRotation_180 + case 270: return kVTRotation_CW90 + default: return nil + } + } + + func rotate(_ source: CVPixelBuffer, by rotation: CFString) -> CVPixelBuffer? { + if session == nil { + var created: VTPixelRotationSession? + guard VTPixelRotationSessionCreate(kCFAllocatorDefault, &created) == noErr else { return nil } + session = created + } + guard let session else { return nil } + if self.rotation != rotation { + VTSessionSetProperty(session, key: kVTPixelRotationPropertyKey_Rotation, value: rotation) + self.rotation = rotation + } + let quarterTurn = rotation != kVTRotation_180 + let width = quarterTurn ? CVPixelBufferGetHeight(source) : CVPixelBufferGetWidth(source) + let height = quarterTurn ? CVPixelBufferGetWidth(source) : CVPixelBufferGetHeight(source) + if pool == nil || poolSize != (width, height) { + let attributes: [String: Any] = [ + kCVPixelBufferWidthKey as String: width, + kCVPixelBufferHeightKey as String: height, + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any](), + ] + CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, attributes as CFDictionary, &pool) + poolSize = (width, height) + } + guard let pool else { return nil } + var target: CVPixelBuffer? + guard CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &target) == noErr, let target, + VTPixelRotationSessionRotateImage(session, source, target) == noErr + else { return nil } + return target + } +} diff --git a/vendor/screen-capture/Sources/simlock-screen-capture/SimulatorSource.swift b/vendor/screen-capture/Sources/simlock-screen-capture/SimulatorSource.swift index 35f37238a..7808e800a 100644 --- a/vendor/screen-capture/Sources/simlock-screen-capture/SimulatorSource.swift +++ b/vendor/screen-capture/Sources/simlock-screen-capture/SimulatorSource.swift @@ -10,6 +10,8 @@ import SimlockCapture final class SimulatorSource: ScreenSource { private let capture = FrameCapture() + private let rotator = PanelRotator() + private var follower: PanelFollower? private var stopped = false func start( @@ -21,16 +23,65 @@ final class SimulatorSource: ScreenSource { // calling back, and FrameCapture reports nothing else. The daemon notices the dead simulator // through the device's own state and closes the capture, so `onEnd` stays unused here. _ = onEnd + SimFrameworks.load() + let panels = FrameCapture.findSimDevice(udid: deviceID).map { Panels.read(for: $0, udid: deviceID) } ?? [] + if panels.count > 1 { try await follow(panels, udid: deviceID) } + let rotator = self.rotator try await capture.start(deviceUDID: deviceID) { pixelBuffer, _ in + let width = CVPixelBufferGetWidth(pixelBuffer) + let height = CVPixelBufferGetHeight(pixelBuffer) + // why: by size, not by the panel in use, because a picture of the panel just left can still + // be on its way after a switch, and it must be turned the way its own panel is mounted. + if let panel = panels.first(where: { $0.matches(width: width, height: height) }), + let rotation = PanelRotator.rotation(undoing: panel.nativeRotation) { + if let upright = rotator.rotate(pixelBuffer, by: rotation) { onFrame(upright) } + return + } onFrame(pixelBuffer) } } + /// Streams the panel in use: the one picked now, then each one SpringBoard moves the picture to. + private func follow(_ panels: [Panel], udid: String) async throws { + let capture = self.capture + let follower = PanelFollower( + panels: panels, sampler: SimctlPanelSampler(udid: udid), log: SimctlPanelLog(udid: udid), + clock: SystemPanelClock() + ) + self.follower = follower + let first = Box(nil) + await follower.start { panel in + first.setIfEmpty(panel) + log("[panels] streaming the \(panel.name) panel (\(panel.width)x\(panel.height))") + Task { await capture.setPreferredScreenSize(width: panel.width, height: panel.height) } + } + // why: the first panel is set before the capture starts, not through the task above, so the + // capture opens on it and not on the smallest surface for a moment. + if let panel = first.value { await capture.setPreferredScreenSize(width: panel.width, height: panel.height) } + } + func stop() async { // why: idempotent — the lifecycle calls it on every way out, and a second `stop` would // unregister callbacks this source no longer owns. guard !stopped else { return } stopped = true + follower?.stop() await capture.stop() } } + +/// A value set from a callback and read after it. +final class Box: @unchecked Sendable { + private let lock = NSLock() + private var stored: Value + init(_ initial: Value) { stored = initial } + var value: Value { lock.lock(); defer { lock.unlock() }; return stored } +} + +extension Box { + func setIfEmpty(_ new: Wrapped) where Value == Wrapped? { + lock.lock() + if stored == nil { stored = new } + lock.unlock() + } +} diff --git a/vendor/screen-capture/Tests/SimlockCaptureTests/EncodeLoopTests.swift b/vendor/screen-capture/Tests/SimlockCaptureTests/EncodeLoopTests.swift new file mode 100644 index 000000000..2853eeda8 --- /dev/null +++ b/vendor/screen-capture/Tests/SimlockCaptureTests/EncodeLoopTests.swift @@ -0,0 +1,45 @@ +import CoreVideo +import Foundation +import XCTest + +@testable import SimlockCapture + +final class EncodeLoopTests: XCTestCase { + private func picture(_ width: Int, _ height: Int) -> CVPixelBuffer { + var buffer: CVPixelBuffer? + CVPixelBufferCreate( + kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, + [kCVPixelBufferIOSurfacePropertiesKey as String: [String: Any]()] as CFDictionary, &buffer + ) + return buffer! + } + + func testAPictureThatChangesSizeGetsANewMetaFrameBeforeItsFirstFrameAndAKeyframe() async throws { + let cover = (width: 1398, height: 2034) + let inner = (width: 2853, height: 2007) + let sizes = [cover, cover, inner, inner, cover] + let (frames, input) = AsyncStream.makeStream(of: Pending.self) + for (index, size) in sizes.enumerated() { + input.yield(Pending(buffer: picture(size.width, size.height), atNanos: UInt64(index) * 100_000_000)) + } + input.finish() + + nonisolated(unsafe) var written: [(tag: Tag, data: Data)] = [] + Wire.sink = { tag, data in written.append((tag, data)) } + defer { Wire.sink = nil } + await encodeLoop(frames, options: Options(), session: Session()) + + let metas = try written.filter { $0.tag == .meta }.map { entry -> String in + let json = try XCTUnwrap(JSONSerialization.jsonObject(with: entry.data) as? [String: Any]) + return "\(json["width"] as! Int)x\(json["height"] as! Int)" + } + XCTAssertEqual(metas, ["494x720", "720x506", "494x720"]) + let tags = written.map(\.tag) + for (index, tag) in tags.enumerated() where tag == .meta { + let first = tags[(index + 1)...].first { $0 == .keyframe || $0 == .delta } + XCTAssertEqual(first, .keyframe, "the first picture after the meta frame at \(index) is not a keyframe") + } + XCTAssertEqual(tags.filter { $0 == .keyframe }.count, 3) + XCTAssertEqual(tags.filter { $0 == .delta }.count, 2) + } +} diff --git a/vendor/screen-capture/Tests/SimlockCaptureTests/PanelFollowerTests.swift b/vendor/screen-capture/Tests/SimlockCaptureTests/PanelFollowerTests.swift new file mode 100644 index 000000000..3dd2c34c3 --- /dev/null +++ b/vendor/screen-capture/Tests/SimlockCaptureTests/PanelFollowerTests.swift @@ -0,0 +1,202 @@ +import XCTest + +@testable import simlock_screen_capture + +final class FakeSampler: PanelSampler { + let answer: [Int: Bool] + init(_ answer: [Int: Bool]) { self.answer = answer } + func lit(_ panels: [Panel]) async -> [Int: Bool] { answer } +} + +final class FakeLog: PanelLog, @unchecked Sendable { + private let lock = NSLock() + private let boot: [String] + private(set) var liveStarts = 0 + private(set) var bootTimeouts: [TimeInterval] = [] + private var onLine: (@Sendable (String) -> Void)? + private var onEnd: (@Sendable () -> Void)? + + private var bootGate: CheckedContinuation? + private let holdsBoot: Bool + + init(boot: [String] = [], holdsBoot: Bool = false) { + self.boot = boot + self.holdsBoot = holdsBoot + } + + func bootLines(timeout: TimeInterval) async -> [String] { + lock.withLock { bootTimeouts.append(timeout) } + if holdsBoot { + await withCheckedContinuation { gate in + lock.lock() + bootGate = gate + lock.unlock() + } + } + return boot + } + + func releaseBoot() { lock.lock(); let gate = bootGate; bootGate = nil; lock.unlock(); gate?.resume() } + func live(onLine: @escaping @Sendable (String) -> Void, onEnd: @escaping @Sendable () -> Void) { + lock.lock() + liveStarts += 1 + self.onLine = onLine + self.onEnd = onEnd + lock.unlock() + } + func stop() {} + func say(_ line: String) { lock.lock(); let handler = onLine; lock.unlock(); handler?(line) } + func end() { lock.lock(); let handler = onEnd; lock.unlock(); handler?() } +} + +/// A clock that moves only when a test says so. +final class FakeClock: PanelClock, @unchecked Sendable { + private let lock = NSLock() + private var now: TimeInterval = 0 + private var waiting: [(deadline: TimeInterval, wake: CheckedContinuation)] = [] + + func sleep(seconds: TimeInterval) async { + await withCheckedContinuation { wake in + lock.lock() + waiting.append((now + seconds, wake)) + lock.unlock() + } + } + + func advance(_ seconds: TimeInterval) { + lock.lock() + now += seconds + let due = waiting.filter { $0.deadline <= now } + waiting.removeAll { $0.deadline <= now } + lock.unlock() + for sleeper in due { sleeper.wake.resume() } + } +} + +final class Picks: @unchecked Sendable { + private let lock = NSLock() + private var names: [String] = [] + func add(_ panel: Panel) { lock.lock(); names.append(panel.name); lock.unlock() } + var all: [String] { lock.lock(); defer { lock.unlock() }; return names } +} + +final class Notes: @unchecked Sendable { + private let lock = NSLock() + private var lines: [String] = [] + func add(_ line: String) { lock.lock(); lines.append(line); lock.unlock() } + var all: [String] { lock.lock(); defer { lock.unlock() }; return lines } +} + +/// Lets the follower's background tasks run on the other threads of the pool; patience, never a bound +/// a test asserts. +func settle() async { for _ in 0..<50 { try? await Task.sleep(nanoseconds: 2_000_000) } } + +func move(_ role: PanelRole) -> String { + let other = role == .cover ? "inner" : "cover" + return "2026-11-11 10:00:00.1 Df SpringBoard[88:1f] [com.apple.SpringBoard:DisplayContentMode] swapped .\(role.rawValue):pri .\(other):aux" +} + +let cover = Panel(name: "cover", screenID: 1, width: 1398, height: 2034, nativeRotation: 0, role: .cover) +let inner = Panel(name: "inner", screenID: 3, width: 2007, height: 2853, nativeRotation: 270, role: .inner) + +final class PanelFollowerStartTests: XCTestCase { + func testTheFollowerPicksThePanelAtStart() async { + let third = Panel(name: "third", screenID: 5, width: 100, height: 100, nativeRotation: 0, role: nil) + let lone = Panel(name: "lone", screenID: 1, width: 100, height: 200, nativeRotation: 0, role: nil) + let cases: [(title: String, panels: [Panel], lit: [Int: Bool], boot: [String], picks: [String], liveStarts: Int)] = [ + ("one panel: not started", [lone], [1: true], [], [], 0), + ("cover lit", [cover, inner], [1: true, 3: false], [move(.inner)], ["cover"], 1), + ("inner lit", [cover, inner], [1: false, 3: true], [move(.cover)], ["inner"], 1), + ("both dark, boot says inner", [cover, inner], [1: false, 3: false], [move(.cover), move(.inner)], ["cover", "inner"], 1), + ("both lit, boot says inner", [cover, inner], [1: true, 3: true], [move(.inner)], ["cover", "inner"], 1), + ("both dark, boot says cover", [cover, inner], [1: false, 3: false], [move(.inner), move(.cover)], ["cover"], 1), + ("both dark, boot log unread", [cover, inner], [1: false, 3: false], [], ["cover"], 1), + ("more than two panels: the first", [inner, cover, third], [1: true, 3: true, 5: true], [], ["inner"], 0), + ] + for row in cases { + let log = FakeLog(boot: row.boot) + let notes = Notes() + let follower = PanelFollower( + panels: row.panels, sampler: FakeSampler(row.lit), log: log, clock: FakeClock(), note: { notes.add($0) } + ) + let picks = Picks() + await follower.start { picks.add($0) } + XCTAssertEqual(picks.all.count, row.picks.isEmpty ? 0 : 1, "\(row.title): the first panel is not reported before start returns") + await settle() + XCTAssertEqual(picks.all, row.picks, row.title) + XCTAssertEqual(log.liveStarts, row.liveStarts, row.title) + XCTAssertEqual(notes.all, row.panels.count > 2 ? ["[panels] 3 panels; staying on the first"] : [], row.title) + // the boot log is asked for within 10 s, and only when the lit panels do not say which is in use + let asked = row.liveStarts == 1 && row.lit[1] == row.lit[3] + XCTAssertEqual(log.bootTimeouts, asked ? [10] : [], row.title) + follower.stop() + } + } +} + +final class PanelFollowerLiveTests: XCTestCase { + private func started( + lit: [Int: Bool] = [1: true, 3: false], log: FakeLog = FakeLog() + ) async -> (follower: PanelFollower, log: FakeLog, clock: FakeClock, picks: Picks, notes: Notes) { + let clock = FakeClock() + let picks = Picks() + let notes = Notes() + let follower = PanelFollower( + panels: [cover, inner], sampler: FakeSampler(lit), log: log, clock: clock, note: { notes.add($0) } + ) + await follower.start { picks.add($0) } + return (follower, log, clock, picks, notes) + } + + func testTheFollowerFollowsLiveMoves() async { + // a live move is reported once, and the same panel again is not + let run = await started() + run.log.say(move(.inner)) + await settle() + run.log.say(move(.inner)) + await settle() + XCTAssertEqual(run.picks.all, ["cover", "inner"], "a live move, then the same panel again") + run.log.say(move(.cover)) + await settle() + XCTAssertEqual(run.picks.all, ["cover", "inner", "cover"], "a move back") + run.follower.stop() + + // a live move heard while the boot log is still read wins over what the boot log says + let slow = await started(lit: [1: false, 3: false], log: FakeLog(boot: [move(.cover)], holdsBoot: true)) + slow.log.say(move(.inner)) + await settle() + slow.log.releaseBoot() + await settle() + XCTAssertEqual(slow.picks.all, ["cover", "inner"], "the live move wins over the boot log") + slow.follower.stop() + + // a live log that ends while wanted is started again after 1 s, and not after a stop + let ending = await started() + ending.log.end() + await settle() + ending.clock.advance(0.9) + await settle() + XCTAssertEqual(ending.log.liveStarts, 1, "restarted before the second passed") + ending.clock.advance(0.1) + await settle() + XCTAssertEqual(ending.log.liveStarts, 2, "not restarted after 1 s") + ending.follower.stop() + ending.log.end() + await settle() + ending.clock.advance(1) + await settle() + XCTAssertEqual(ending.log.liveStarts, 2, "restarted after a stop") + + // no line ever: it stays on the panel it picked + let silent = await started(lit: [1: false, 3: true]) + await settle() + silent.clock.advance(29) + await settle() + XCTAssertEqual(silent.notes.all, [], "said it heard nothing before 30 s") + silent.clock.advance(1) + await settle() + XCTAssertEqual(silent.notes.all, ["[panels] no panel move heard; staying on inner"], "one log line after 30 s") + XCTAssertEqual(silent.picks.all, ["inner"], "moved without a line") + silent.follower.stop() + } +} diff --git a/vendor/screen-capture/Tests/SimlockCaptureTests/PanelsTests.swift b/vendor/screen-capture/Tests/SimlockCaptureTests/PanelsTests.swift new file mode 100644 index 000000000..d730362c4 --- /dev/null +++ b/vendor/screen-capture/Tests/SimlockCaptureTests/PanelsTests.swift @@ -0,0 +1,25 @@ +import XCTest + +@testable import simlock_screen_capture + +// why: the parser is pure logic with many cases, and SpringBoard's log text is the part a new iOS +// release changes; the follower's tests drive it through `PanelFollower`. +final class PanelLogLineTests: XCTestCase { + private let tag = "[com.apple.SpringBoard:DisplayContentMode]" + + func testALogLineNamesThePanelInUse() { + let cases: [(line: String, expected: PanelRole?)] = [ + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] \(tag) swapped .inner:pri .cover:aux", .inner), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] \(tag) swapped .cover:pri .inner:aux", .cover), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] \(tag) transitioning .inner:off→pri .cover:aux", nil), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] \(tag) transitioning .cover:pri→off .inner:off→pri", nil), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] swapped .inner:pri .cover:aux", nil), + ("Filtering the log data using \"... CONTAINS \".cover:pri\" OR eventMessage CONTAINS \".inner:pri\"\"", nil), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] \(tag) swapped .inner:pri .cover:pri", nil), + ("2026-11-11 10:00:00.1 Df SpringBoard[88:1f] [com.apple.SpringBoard:Other] hello", nil), + ] + for (line, expected) in cases { + XCTAssertEqual(Panels.primary(inLogLine: line), expected, line) + } + } +} diff --git a/vendor/screen-capture/Tests/SimlockCaptureTests/WatchdogTests.swift b/vendor/screen-capture/Tests/SimlockCaptureTests/WatchdogTests.swift new file mode 100644 index 000000000..f1dc93440 --- /dev/null +++ b/vendor/screen-capture/Tests/SimlockCaptureTests/WatchdogTests.swift @@ -0,0 +1,56 @@ +import Foundation +import XCTest + +@testable import simlock_screen_capture + +final class WatchdogTests: XCTestCase { + private func alive(_ pid: pid_t) -> Bool { kill(pid, 0) == 0 } + + /// Patience for a process to go; the assertion after it names what did not happen. + private func waitUntil(_ done: () -> Bool) async { + for _ in 0..<500 where !done() { try? await Task.sleep(nanoseconds: 10_000_000) } + } + + private func startCommand(_ body: String, then: (String) -> String = { _ in "" }) async throws -> (watchdog: Process, child: pid_t) { + let file = NSTemporaryDirectory() + "watchdog-\(UUID().uuidString).pid" + let watchdog = Watchdog.process(running: ["/bin/sh", "-c", "echo $$ > \(file); \(body)"], then: then(file)) + try watchdog.run() + await waitUntil { FileManager.default.fileExists(atPath: file) && ((try? String(contentsOfFile: file)) ?? "").hasSuffix("\n") } + let text = (try? String(contentsOfFile: file, encoding: .utf8)) ?? "" + let pid = try XCTUnwrap(pid_t(text.trimmingCharacters(in: .whitespacesAndNewlines)), "the command never started") + return (watchdog, pid) + } + + func testTheWatchdogEndsItsCommandWhenTheStdinPipeClosesAndEndsWhenTheCommandDoes() async throws { + let held = try await startCommand("exec sleep 60") + XCTAssertTrue(alive(held.child), "the command is not running while the pipe is open") + try (held.watchdog.standardInput as! Pipe).fileHandleForWriting.close() + await waitUntil { !alive(held.child) } + XCTAssertFalse(alive(held.child), "the command outlived the closed pipe") + + let short = try await startCommand("exec sleep 0.2") + await waitUntil { !short.watchdog.isRunning } + XCTAssertFalse(short.watchdog.isRunning, "the watchdog outlived its command") + try (short.watchdog.standardInput as! Pipe).fileHandleForWriting.close() + } + + func testTheWatchdogRunsItsFollowUpOnlyOnceTheCommandIsGone() async throws { + let marker = NSTemporaryDirectory() + "watchdog-then-\(UUID().uuidString)" + // The follow-up records the moment the command is already dead; run earlier, it records nothing. + let held = try await startCommand("exec sleep 60") { pidFile in + #"kill -0 "$(cat \#(pidFile))" 2>/dev/null || touch \#(marker)"# + } + try (held.watchdog.standardInput as! Pipe).fileHandleForWriting.close() + await waitUntil { FileManager.default.fileExists(atPath: marker) } + XCTAssertTrue(FileManager.default.fileExists(atPath: marker), "the follow-up ran before the command was gone, or never") + try? FileManager.default.removeItem(atPath: marker) + } + + func testEachLogCommandIsNamedAndEndedByATokenOfItsOwn() { + let log = SimctlPanelLog(udid: "UDID") + let show = log.commands(["show"]), stream = log.commands(["stream"]) + XCTAssertNotEqual(show.end, stream.end, "ending one log command would end the other") + XCTAssertFalse(stream.log.joined(separator: " ").contains(show.end.components(separatedBy: "\"")[5]), + "the stream carries the token that ends the show") + } +} diff --git a/website/src/content/docs/docs/using-devices/watching-the-screen.mdx b/website/src/content/docs/docs/using-devices/watching-the-screen.mdx index 976722280..2f01132ea 100644 --- a/website/src/content/docs/docs/using-devices/watching-the-screen.mdx +++ b/website/src/content/docs/docs/using-devices/watching-the-screen.mdx @@ -76,7 +76,9 @@ A device that isn't running has no picture, and the page says why: shut down, st - **A dropped stream reconnects.** If the picture started and the stream then ends without a reason, such as when your sign-in through a proxy expires, the page opens it again. - **A frozen device doesn't look live.** A stream that gets no new frames for 30 seconds ends with an error. - **Each viewer is a separate capture.** Two browser tabs on one device capture it twice. -- **Rotation.** A rotated iOS simulator or iPhone keeps the scale of the first picture. An Android device's picture follows the rotation. +- **Rotation.** An Android device's picture follows the rotation. So does an iOS simulator's or an iPhone's whenever its picture changes size: a new meta frame gives the new size, and nobody has run it on a physical iPhone yet. +- **Foldables.** An iPhone Duo simulator shows the panel in use, upright, and the open stream switches panels within about 5 seconds after the device folds or unfolds, with no reconnect. A stream opened while the screen is all black can show the cover for a few seconds before it switches. If SpringBoard's log never names the move, the stream stays on the panel it opened on. +- **The size can change.** The size headers give the picture's size when the stream opens; a meta frame in the body gives each later size. A viewer that reads the size once draws a stretched picture after a fold until it reconnects. ## When it doesn't work