feat: choose a path in an exact Open or Save panel

dist/build.js

@@ -53,6 +53,7 @@ const patches = [
     "peekaboo-focus-activation.patch",
     "peekaboo-preparation-message.patch",
     "peekaboo-sck-quarantine.patch",
+    "peekaboo-dialog-path.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.js

@@ -485,10 +485,16 @@ function validateAction(request) {
                 throw new Error("Use one exact dialog button title of at most 512 UTF-16 code units.");
             return;
         }
+        if ("path" in request && keys.length === 1) {
+            if (typeof request.path !== "string" || !request.path.startsWith("/") || request.path.includes("\0")
+                || Buffer.byteLength(request.path) > 4096)
+                throw new Error("Use one absolute file-dialog path of at most 4096 bytes.");
+            return;
+        }
         if (!("field" in request) || keys.join() !== "field,text" || !Number.isInteger(request.field)
             || request.field < 0 || request.field > 1000 || typeof request.text !== "string"
             || request.text.length > 8192) {
-            throw new Error("Answer a dialog with exactly one button title, or one listed field index and at most 8192 UTF-16 code units of text.");
+            throw new Error("Answer a dialog with exactly one button title, one listed field index and at most 8192 UTF-16 code units of text, or one path.");
         }
         return;
     }

dist/protocol.d.ts

@@ -37,7 +37,7 @@ export type DesktopManagement = {
         height: number;
     };
 };
-/** Press one button by its exact title, or replace one text field's value by its listed index. */
+/** Press one button by its exact title, replace one text field's value by its listed index, or choose a path in a file panel. */
 export type DesktopDialogAnswer = {
     op: "dialog-answer";
     target: DesktopAppTarget | DesktopWindowTarget;
@@ -46,6 +46,8 @@ export type DesktopDialogAnswer = {
 } | {
     field: number;
     text: string;
+} | {
+    path: string;
 });
 /** One status item from desktop_menubar_items: owner process generation, inventory index and its window if it has one. */
 export type DesktopStatusItemTarget = DesktopAppTarget & {

dist/tools.js

@@ -269,7 +269,7 @@ export function desktopTools({ place, app }) {
         {
             name: "desktop_dialog_answer",
             op: "dialog-answer",
-            description: `Answer the one open dialog, alert or sheet of an exact target listed by desktop_dialog, without activating the app. Pass the same application or window target, plus either button with one listed title, or field with one listed text field index and text, which replaces that field's whole value. A button title must match exactly, including case, punctuation and ellipses, and be unique and enabled; there are no aliases such as "default" or "cancel". Native checks read the dialog afresh, bind the press to the process generation, window and button, and send one background Accessibility press. Text is set through Accessibility and read back; secure fields are refused. Buttons such as "Delete" or "Don't Save" discard unsaved work; choose them only deliberately. completed means the native service confirmed the press or the field's new value; the result refreshes the app's window inventory, since a dialog or its window may have closed. In an Open or Save panel the file name field and buttons can be answered, but choosing a folder or path is not supported. Never blindly repeat an interrupted answer; list the dialog again first.`,
+            description: `Answer the one open dialog, alert or sheet of an exact target listed by desktop_dialog. Pass the same application or window target, plus exactly one of: button with one listed title; field with one listed text field index and text, which replaces that field's whole value; or path, for an Open or Save panel only. A button title must match exactly, including case, punctuation and ellipses, and be unique and enabled; there are no aliases such as "default" or "cancel". Native checks read the dialog afresh, bind the press to the process generation, window and button, and send one background Accessibility press. Text is set through Accessibility and read back; secure fields are refused. Buttons such as "Delete" or "Don't Save" discard unsaved work; choose them only deliberately. completed means the native service confirmed the press or the field's new value; the result refreshes the app's window inventory, since a dialog or its window may have closed. Buttons and fields are answered in the background without activating the app. path takes an absolute path and chooses it in the panel without pressing its default button: an existing folder becomes the panel's folder; any other path names a file in an existing folder, which a Save panel takes as its folder and file name and an Open panel selects. This is foreground input: the native service activates the panel's app, verifies the exact panel has keyboard focus, and uses the panel's path field or Go to Folder (Command-Shift-G), so the real keyboard focus changes. completed means the panel read back the folder and, in a Save panel, the name; choosing a file in an Open panel, or a panel that exposes no folder, is unknown, so list the dialog before pressing its default button. Never blindly repeat an interrupted answer; list the dialog again first.`,
             parameters: {
                 type: "object",
                 required: ["target"],
@@ -278,6 +278,7 @@ export function desktopTools({ place, app }) {
                     button: { type: "string", minLength: 1, maxLength: 512 },
                     field: { type: "integer", minimum: 0, maximum: 1000 },
                     text: { type: "string", maxLength: 8192 },
+                    path: { type: "string", minLength: 1, maxLength: 4096 },
                 },
                 additionalProperties: false,
             },
@@ -727,11 +728,12 @@ export function desktopRequest(tool, args) {
         }
     }
     if (tool.op === "dialog-answer") {
-        const { button, field, text } = args;
-        const press = button !== undefined && field === undefined && text === undefined;
-        const enter = button === undefined && field !== undefined && text !== undefined;
-        if (!press && !enter) {
-            throw new Error("Supply one button title, or one field index with its text.");
+        const { button, field, text, path } = args;
+        const press = button !== undefined && field === undefined && text === undefined && path === undefined;
+        const enter = button === undefined && field !== undefined && text !== undefined && path === undefined;
+        const choose = path !== undefined && button === undefined && field === undefined && text === undefined;
+        if (!press && !enter && !choose) {
+            throw new Error("Supply one button title, one field index with its text, or one path.");
         }
     }
     const request = { op: tool.op };

docs/tools.md

@@ -65,7 +65,7 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key |
 | `desktop_system_key` | Press one system-wide shortcut, such as Command-Space or Control-Up, on whatever is frontmost |
 | `desktop_menu` | Run one menu command by its exact title path |
-| `desktop_dialog_answer` | Press one dialog button by its exact title, or replace one dialog text field's value |
+| `desktop_dialog_answer` | Press one dialog button by its exact title, replace one dialog text field's value, or choose a path in an Open or Save panel |
 | `desktop_clipboard_write` | Replace the clipboard with text, an image, or file references |
 
 Pointer input is delivered to the exact window without moving the physical pointer, except for `desktop_hover` and a `desktop_drag` with `to_snapshot`. Hover effects follow the real pointer, so it moves the user's pointer and leaves it there. Before moving, it checks that the snapshot's window is still at the same place and frontmost at the point, and refuses otherwise. Afterward, it checks that the pointer arrived and the window is still frontmost there; if either can't be confirmed, the result is `unknown`. A drag into another window or app can't stay inside one window, so it also uses the real pointer, and pressing in the source may bring its app forward. Pass the destination's snapshot as `to_snapshot`. Both points must be uncovered at their snapshot positions, or the drag is refused before input. The pointer rests on the start point, then presses, pauses, moves, rests over the destination and releases. The result is always `unknown`; inspect both windows to see whether the drop happened. `desktop_press` holds the button through the same exact-window route as a drag, and the host always releases it, even when the call is interrupted or the window changes. When the host turns on the agent cursor, an orange arrow first glides to each click, scroll, drag, hover or press point, about a third of a second before the input is sent. It then shows a ripple for clicks, chevrons for scrolls and a trail for drags. For a press, it stays squeezed until the button is released, then ripples. The cursor fades after a few seconds idle. The cursor never takes clicks or focus and is left out of `desktop_inspect` and `desktop_screen` captures. It is decoration only: if it fails or is slow, input proceeds unchanged after at most 600 ms. Keyboard input, `desktop_type` and `desktop_select` don't show it.
@@ -80,6 +80,8 @@ When the host turns on the window highlight, the window an inspection or action
 
 With `ocr: true`, `desktop_inspect` (in either mode) and `desktop_screen` also run Apple Vision text recognition, at its accurate level and on this Mac, on the screenshot they captured. `recognized_text` lists up to 300 lines in reading order, top to bottom and left to right, with confidence and a box as fractions of the screenshot from its top-left corner; `desktop_screen` adds bounds in global desktop points. A box's center is a pointer point for that inspection's snapshot. Recognition is limited to 8 seconds; the first one after a restart can take longer while Vision loads its model. Recognized text comes from pixels, so it can misread or miss text, and is labeled as such.
 
+`desktop_dialog_answer` with `path` chooses an absolute path in an Open or Save panel without pressing its default button. An existing folder becomes the panel's folder. Any other path names a file in an existing folder: a Save panel takes its folder and file name, and an Open panel selects it. Unlike buttons and fields, this is foreground input: the host activates the panel's app, checks that the exact panel has keyboard focus before each key, and uses the panel's path field or Go to Folder. The result is `completed` only when the panel reads back the folder and, in a Save panel, the name. Choosing a file in an Open panel, or a panel that shows no folder, is `unknown`; list the dialog before pressing its default button.
+
 `desktop_inspect` lists the actions each element offers in `attributes.actions`. A pop-up button has no settable value: click it, then pick the item from the next observation, which puts an open menu's items first.
 
 `desktop_system_key` is for shortcuts that belong to macOS rather than a window: Spotlight (Command-Space), the app switcher (Command-Tab), Mission Control (Control-Up), the screenshot tools (Command-Shift-4 and 5) and the next input source (Control-Space). It takes the same keys and modifiers as `desktop_key`, but no snapshot. The keys are posted as ordinary keyboard input, so they reach whatever is frontmost, and they often move focus, activate an app or open system UI. Nothing reads back what a shortcut did, so the result is `unknown` even after delivery; observe with `desktop_screen`, `desktop_apps` or `desktop_windows`. Any chord with Command and W, Q, H or M is refused before anything is sent, including Command-Shift-Q (log out).
@@ -90,7 +92,6 @@ An action refused because its window was minimized after inspection reports `WIN
 
 - **Insertion outside text fields.** `desktop_insert` needs an observed focused text control with a readable selection.
 - **Confirmed drops between windows.** A drag into another window or app is always reported as `unknown`, because nothing can confirm the drop. Drags are one straight line, with no modifier keys.
-- **Choosing paths in file panels.** `desktop_dialog_answer` can type a file name into a save panel's field, but it can't navigate an open or save panel to a folder.
 - **Automatic retries.** By design, nothing is retried. After a `refused` or `unknown` result, observe again before deciding whether to act.
 - **Remote Macs.** The tools act on the Mac running the host app, on macOS 15 or later.
 

native/patches/README.md

@@ -370,6 +370,22 @@ a fallback here; wait, refuse, and let the caller retry. The SCK gate lock stays
 apps overlap ScreenCaptureKit calls again, which is what it prevents. `ScreenCaptureKitQuarantineGraceTests.swift` covers the
 grace period and the message.
 
+`peekaboo-dialog-path.patch` adds `exactFileDialogPath`, which chooses one absolute path in one exact
+Open or Save panel without pressing its default button. Upstream `dialogHandleFile` finds the panel by a
+fuzzy app name, clicks the panel's center to focus it, and presses the default button in the same call. The
+new operation resolves the panel the way exact forced dismissal does, from an app, PID, or window selector,
+and refuses a dialog that is not a file panel. An existing folder becomes the panel's folder; any other path
+must name a file in an existing folder, which a Save panel (one with `saveAsNameTextField`) takes as its
+folder and name, and an Open panel selects through Go to Folder. Folder and file paths resolve symlinks
+before input, so `/tmp` compares as `/private/tmp`. The host focuses the retained panel's window, checks that the
+panel holds keyboard focus before each key, expands a collapsed panel, sets the expanded panel's
+`PathTextField` and presses Return, and falls back to Go to Folder (Command-Shift-G), whose new field is
+set through Accessibility when it can be and typed otherwise. Nothing clicks the panel. The result is
+`confirmedChange` only when the path field reads back the folder and, in a Save panel, the name field
+reads back the name; otherwise it is `dispatchedUnverified`. Details report the readback. The descriptor,
+permission set, semantic policy and result contract are listed in the upstream inventory tests, and new
+tests cover path resolution and request validation.
+
 `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision,
 and assembles the ordered patch stack in a private Git index. A build compares the checkout
 with each complete ordered prefix, since later patches can change earlier patch contexts.