feat: element actions and value setting for non-text controls

dist/build.js

@@ -46,6 +46,7 @@ const patches = [
     "peekaboo-agent-window.patch",
     "peekaboo-press.patch",
     "peekaboo-layout-keys.patch",
+    "peekaboo-element-action.patch",
 ].map((name) => join(native, "patches", name));
 function spawn(command, env) {
     return spawnSync(command[0], command.slice(1), { encoding: "utf8", env: env ?? process.env });

dist/client.js

@@ -4,7 +4,7 @@ import { lstat, mkdtemp, open, readFile, rm, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { promisify } from "node:util";
-import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js";
+import { DESKTOP_BUTTONS, DESKTOP_CLICKS, DESKTOP_DIRECTIONS, DESKTOP_ELEMENT_ACTIONS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, isDesktopAction, isDesktopKey, isDesktopManagement, } from "./protocol.js";
 const exec = promisify(execFile);
 const MAX_TEXT = 32_000;
 const MAX_IMAGE = 900_000;
@@ -525,6 +525,13 @@ function validateAction(request) {
     if (typeof request.element !== "string" || !request.element || request.element.length > 256) {
         throw new Error("Choose an element ID from the inspected snapshot.");
     }
+    if (request.op === "element-action" && !DESKTOP_ELEMENT_ACTIONS.includes(request.action)) {
+        throw new Error(`Choose one of ${DESKTOP_ELEMENT_ACTIONS.join(", ")}.`);
+    }
+    if (request.op === "set-value" && typeof request.value !== "boolean"
+        && (typeof request.value !== "number" || !Number.isFinite(request.value))) {
+        throw new Error("Set a finite number or true/false; use desktop_type for text.");
+    }
     if (request.op === "type" && (typeof request.text !== "string" || request.text.length > 8192)) {
         throw new Error("Replacement text must contain at most 8192 UTF-16 code units.");
     }
@@ -1019,6 +1026,8 @@ async function execute(native, request, abort) {
             "key",
             "insert",
             "select",
+            "element-action",
+            "set-value",
             "scroll",
             "drag",
             "hover",

dist/protocol.d.ts

@@ -159,6 +159,16 @@ export type DesktopRequest = DesktopManagement | DesktopLaunch | DesktopOpen | D
     op: "insert";
     snapshot: string;
     text: string;
+} | {
+    op: "element-action";
+    snapshot: string;
+    element: string;
+    action: DesktopElementAction;
+} | {
+    op: "set-value";
+    snapshot: string;
+    element: string;
+    value: number | boolean;
 } | {
     op: "select";
     snapshot: string;
@@ -187,13 +197,16 @@ export type DesktopButton = (typeof DESKTOP_BUTTONS)[number];
 export declare const DESKTOP_KEYS: readonly ["enter", "tab", "escape", "backspace", "delete", "up", "down", "left", "right", "space", "home", "end", "pageup", "pagedown", "help", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20", "keypad0", "keypad1", "keypad2", "keypad3", "keypad4", "keypad5", "keypad6", "keypad7", "keypad8", "keypad9", "keypaddecimal", "keypadmultiply", "keypadplus", "keypadminus", "keypaddivide", "keypadequals", "keypadenter", "keypadclear"];
 export declare const DESKTOP_MODIFIERS: readonly ["command", "control", "option", "shift"];
 export declare const DESKTOP_SELECTIONS: readonly ["text", "cursor_before", "cursor_after"];
+/** Named Accessibility actions; AXPress is desktop_click's route and is not listed separately. */
+export declare const DESKTOP_ELEMENT_ACTIONS: readonly ["AXShowMenu", "AXIncrement", "AXDecrement", "AXConfirm", "AXCancel", "AXPick", "AXRaise", "AXScrollToVisible"];
 export type DesktopNamedKey = (typeof DESKTOP_KEYS)[number];
 /** A named key, or the one printable character a key types without Shift or Option, such as `,` or `a`. */
 export type DesktopKey = DesktopNamedKey | (string & {});
 export type DesktopModifier = (typeof DESKTOP_MODIFIERS)[number];
 export type DesktopSelection = (typeof DESKTOP_SELECTIONS)[number];
+export type DesktopElementAction = (typeof DESKTOP_ELEMENT_ACTIONS)[number];
 export type DesktopAction = Extract<DesktopRequest, {
-    op: "click" | "type" | "key" | "insert" | "select" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "menubar-click" | "dock";
+    op: "click" | "type" | "key" | "insert" | "select" | "element-action" | "set-value" | "scroll" | "drag" | "hover" | "press" | "activate" | "launch" | "open" | "quit" | "close" | "focus" | "minimize" | "restore" | "move" | "resize" | "clipboard-write" | "menu" | "dialog-answer" | "hide" | "unhide" | "maximize" | "menubar-click" | "dock";
 }>;
 export type DesktopOutcome = "completed" | "refused" | "unknown";
 export type DesktopImage = {

dist/protocol.js

@@ -60,11 +60,23 @@ export const DESKTOP_KEYS = [
 ];
 export const DESKTOP_MODIFIERS = ["command", "control", "option", "shift"];
 export const DESKTOP_SELECTIONS = ["text", "cursor_before", "cursor_after"];
+/** Named Accessibility actions; AXPress is desktop_click's route and is not listed separately. */
+export const DESKTOP_ELEMENT_ACTIONS = [
+    "AXShowMenu",
+    "AXIncrement",
+    "AXDecrement",
+    "AXConfirm",
+    "AXCancel",
+    "AXPick",
+    "AXRaise",
+    "AXScrollToVisible",
+];
 export function isDesktopAction(request) {
     return request.op === "click" || request.op === "type" || request.op === "key"
         || request.op === "insert" || request.op === "select" || request.op === "scroll"
         || request.op === "drag" || request.op === "hover" || request.op === "press"
         || request.op === "clipboard-write" || request.op === "launch"
+        || request.op === "element-action" || request.op === "set-value"
         || request.op === "open" || request.op === "menu" || request.op === "dialog-answer"
         || request.op === "menubar-click" || request.op === "dock"
         || isDesktopManagement(request);

dist/tools.js

@@ -563,6 +563,45 @@ export function desktopTools({ place, app }) {
                 },
             },
         },
+        {
+            name: "desktop_set_value",
+            op: "set-value",
+            description: `Set the value of one non-text control from a fresh desktop_inspect result and verify it by reading the same element back: a number for a slider, stepper (AXIncrementor), or level indicator, or true/false for a checkbox, switch, or disclosure triangle (true expands it). Pass snapshot_id as snapshot and the literal element ID as element. A number is written directly and must read back exactly, so a control that rounds or clamps it reports unknown; desktop_element_action with AXIncrement or AXDecrement steps a control without a settable value. A toggle with a read-only value that advertises AXPress gets at most one press, and none when it already has the requested state; a mixed state is refused. Use desktop_type for text fields. A pop-up button has no settable value: open it with desktop_click, then choose the item from the fresh observation, which lists an open menu's items first, with desktop_element_action AXPick or desktop_click. The snapshot binds the exact process, window, and element and is single-use. This does not activate the window. Inspect again after an unknown result and never blindly repeat interrupted input.`,
+            parameters: {
+                type: "object",
+                required: ["snapshot", "element", "value"],
+                properties: {
+                    snapshot: id,
+                    element: id,
+                    value: { anyOf: [{ type: "number" }, { type: "boolean" }] },
+                },
+            },
+        },
+        {
+            name: "desktop_element_action",
+            op: "element-action",
+            description: `Perform one named Accessibility action on one element from a fresh desktop_inspect result, without pointer or keyboard input. The element must list the action in its attributes.actions; others are refused before any input. AXShowMenu opens the element's context menu without a right click; AXIncrement and AXDecrement step a slider or stepper by its own increment; AXConfirm and AXCancel accept or dismiss, such as AXCancel on an open menu; AXPick chooses an open menu's item; AXRaise brings a window forward within its app; AXScrollToVisible scrolls the element into view. Use desktop_click to press a button. Pass snapshot_id as snapshot and the literal element ID as element. The snapshot binds the exact process, window, and element and is single-use. Accepted delivery does not prove the effect; inspect again before acting further, and never blindly repeat an action, because each repetition can step, toggle, or open again.`,
+            parameters: {
+                type: "object",
+                required: ["snapshot", "element", "action"],
+                properties: {
+                    snapshot: id,
+                    element: id,
+                    action: {
+                        anyOf: [
+                            { type: "string", const: "AXShowMenu" },
+                            { type: "string", const: "AXIncrement" },
+                            { type: "string", const: "AXDecrement" },
+                            { type: "string", const: "AXConfirm" },
+                            { type: "string", const: "AXCancel" },
+                            { type: "string", const: "AXPick" },
+                            { type: "string", const: "AXRaise" },
+                            { type: "string", const: "AXScrollToVisible" },
+                        ],
+                    },
+                },
+            },
+        },
         {
             name: "desktop_select",
             op: "select",

docs/tools.md

@@ -1,6 +1,6 @@
 # Tools
 
-The [MCP server](mcp.md) and `desktopTools()` expose 33 tools, one per protocol operation. Each tool's description and argument schema in `src/tools.ts` is the full contract; this page is the map.
+The [MCP server](mcp.md) and `desktopTools()` expose 35 tools, one per protocol operation. Each tool's description and argument schema in `src/tools.ts` is the full contract; this page is the map.
 
 Targets are exact: an app target binds a process generation, and a window target binds its window ID and bounds. Input acts on a single-use snapshot from the latest `desktop_inspect`. A stale or changed target is refused rather than redirected. Action results report `completed`, `refused`, or `unknown`, and nothing is retried automatically.
 
@@ -54,6 +54,8 @@ A status item is pressed through Accessibility in its owner process without movi
 | `desktop_press` | Press and hold an element or point for up to 10 seconds, then release in place |
 | `desktop_type` | Replace an editable field's whole value |
 | `desktop_select` | Select literal text in a field, or put the caret before or after it |
+| `desktop_set_value` | Set a slider, stepper or level indicator to a number, or a checkbox, switch or disclosure triangle on or off, and read it back |
+| `desktop_element_action` | Perform one Accessibility action an element lists, such as `AXShowMenu`, `AXIncrement`, `AXDecrement`, `AXPick` or `AXCancel` |
 | `desktop_insert` | Insert text at the caret or over the selection with one temporary paste, then restore the clipboard |
 | `desktop_key` | Press one key with optional Command, Control, Option and Shift: any character key, such as `,` or `/`, or a named key |
 | `desktop_menu` | Run one menu command by its exact title path |
@@ -66,6 +68,8 @@ When the host turns on the window highlight, the window an inspection or action
 
 `desktop_key` takes a character key as the character it types, and finds that key on the current keyboard layout, so `,` is the comma key on any layout. A character that needs Shift or Option, such as `?` or `A`, is refused with the key and modifier to send instead (`/` with `shift` on a US layout); the key press never holds a modifier that wasn't asked for. The named keys are `enter`, `tab`, `escape`, `space`, `backspace`, `delete` (forward delete), `up`, `down`, `left`, `right`, `home`, `end`, `pageup`, `pagedown`, `help` (Insert on PC keyboards), `f1` to `f20`, and the keypad keys `keypad0` to `keypad9`, `keypaddecimal`, `keypadplus`, `keypadminus`, `keypadmultiply`, `keypaddivide`, `keypadequals`, `keypadenter` and `keypadclear`. Dead keys, such as `´` on some layouts, can't be pressed alone; use `desktop_insert` for text.
 
+`desktop_inspect` lists the actions each element offers in `attributes.actions`. A pop-up button has no settable value: click it, then pick the item from the next observation, which puts an open menu's items first.
+
 ## Not supported
 
 - **Keys outside text fields.** `desktop_key` and `desktop_insert` need a focused text control, so Escape, arrows, Return and shortcuts in lists, the Finder or canvases are refused ([#13](https://github.com/githubnext/desktop-tools/issues/13)).

native/patches/README.md

@@ -287,6 +287,25 @@ the needed modifier differs between layouts. The resolved key keeps the characte
 paste, select-all and background shortcut policies still apply. Named keys, their codes and every other
 caller are unchanged. The patch also names F13–F20 and the keypad keys, whose codes are layout-independent.
 
+`peekaboo-element-action.patch` binds named Accessibility actions and value writes to the exact observed
+element, for element actions and non-text values. Both resolved a target string by snapshot ID, then fell back
+to a fuzzy label query, and re-found the live element by a score in which a matching label or frame alone
+could select a neighbor; the only element check afterwards compared that live element with itself. With the
+patch, `performAction` and `setValue` take only a literal element ID from the snapshot, and before input the
+live element must match the observed element's receipt: process, owning window, role, frame, and identifier
+or title. An observed `group-` identifier prefix that observation strips is compared in its native form. A
+mismatch is a typed pre-dispatch `SNAPSHOT_STALE` refusal; the same receipt is checked again immediately
+before the action. Named actions run off MainActor like `AXPress`: `AXShowMenu` reports delivery after the
+existing show-menu grace period while its menu tracks, and every other action keeps the operation lane until
+the native call returns. A failure that may follow delivery stays indeterminate, with no fallback route.
+A checkbox, switch, or disclosure triangle whose value is read-only but which advertises `AXPress` (as
+SwiftUI checkboxes do) is set by at most one press: a state already matching the request sends nothing, a
+mixed or unreadable state refuses, and only a readback of the requested state on the same element confirms
+the change. The `setValue` result contract accepts that press's Accessibility-action delivery. Observation
+also records each element's advertised standard action names, at most 16 of up to 48 ASCII letters, in an
+`actions` attribute; custom actions, which carry app text, are omitted. Tests whose resolver returns a stand-in AX element inject a
+matcher that accepts it; two new tests cover a mismatched live element and a label in place of an element ID.
+
 `desktop-tools-build` resolves only `Package.resolved` versions, checks the pin and checkout revision,
 and assembles the ordered patch stack in a private Git index. A build compares the checkout
 with each complete ordered prefix, since later patches can change earlier patch contexts.