From c0f0b4e81a990291d2ba25fdedd409c2a92c73d8 Mon Sep 17 00:00:00 2001 From: Simon <10131203+gaomeng1900@users.noreply.github.com> Date: Wed, 9 Sep 2026 02:27:28 +0800 Subject: [PATCH 1/3] feat(page-controller): reject misused actions and verify input results --- packages/page-controller/src/actions.ts | 68 +++++++++++++++++++++---- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/page-controller/src/actions.ts b/packages/page-controller/src/actions.ts index 7bb710189..cff44bf93 100644 --- a/packages/page-controller/src/actions.ts +++ b/packages/page-controller/src/actions.ts @@ -62,6 +62,12 @@ function blurLastClickedElement() { * @private Internal method, subject to change at any time. */ export async function clickElement(element: HTMLElement) { + if (isSelectElement(element)) { + throw new Error( + 'Clicking a native ` types whose `value` is not user-typed text. */ +const NON_TEXT_INPUT_TYPES = new Set([ + 'button', + 'checkbox', + 'file', + 'image', + 'radio', + 'reset', + 'submit', +]) + +function assertAcceptsText(element: HTMLElement): void { + const acceptsText = + isTextAreaElement(element) || + element.isContentEditable || + (isInputElement(element) && !NON_TEXT_INPUT_TYPES.has(element.type)) + if (!acceptsText) { + const tag = isInputElement(element) + ? `` + : `<${element.tagName.toLowerCase()}>` + throw new Error(`${tag} does not accept text input.`) + } +} + /** * @private Internal method, subject to change at any time. */ export async function inputTextElement(element: HTMLElement, text: string) { + assertAcceptsText(element) + const isContentEditable = element.isContentEditable - if (!isInputElement(element) && !isTextAreaElement(element) && !isContentEditable) { - throw new Error('Element is not an input, textarea, or contenteditable') - } await clickElement(element) @@ -217,16 +246,26 @@ export async function inputTextElement(element: HTMLElement, text: string) { // Trigger blur for validation element.blur() - } else { - getNativeValueSetter(element as HTMLInputElement | HTMLTextAreaElement).call(element, text) - } - // Only dispatch shared input event for non-contenteditable (contenteditable has its own) - if (!isContentEditable) { - element.dispatchEvent(new Event('input', { bubbles: true })) + await waitFor(0.1) + return } + const input = element as HTMLInputElement | HTMLTextAreaElement + const valueBefore = input.value + getNativeValueSetter(input).call(input, text) + input.dispatchEvent(new Event('input', { bubbles: true })) + await waitFor(0.1) + + // Pages may legitimately reformat the value (masks, normalizers), so a different value + // is not a failure. An unchanged or emptied field means the page discarded the input. + const value = input.value + if (value !== text && (value === valueBefore || value === '')) { + throw new Error( + `The page discarded the input. The field now contains ${JSON.stringify(value)}. Inspect the current page state before retrying.` + ) + } } /** @@ -245,10 +284,19 @@ export async function selectOptionElement(selectElement: HTMLSelectElement, opti throw new Error(`Option with text "${optionText}" not found in select element`) } - selectElement.value = option.value + // Assigning `value` would pick the first option with that value; index targets this exact option. + selectElement.selectedIndex = option.index + selectElement.dispatchEvent(new Event('input', { bubbles: true })) selectElement.dispatchEvent(new Event('change', { bubbles: true })) await waitFor(0.1) // Wait to ensure change event processing completes + + if (selectElement.selectedIndex !== option.index) { + const current = selectElement.selectedOptions.item(0)?.textContent?.trim() ?? '' + throw new Error( + `The page discarded the selection. Expected option ${JSON.stringify(optionText)}, but the current option is ${JSON.stringify(current)}. Inspect the current page state before retrying.` + ) + } } interface ScrollableElement extends Element { From 2f4f0fb0101f81ec5afb4b089e1054b96da058f1 Mon Sep 17 00:00:00 2001 From: Simon <10131203+gaomeng1900@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:06:37 +0800 Subject: [PATCH 2/3] fix(page-controller): report input differences without failing --- docs/agentic-testing/actions.md | 13 ++++++ .../page-controller/src/PageController.ts | 7 ++- packages/page-controller/src/actions.test.ts | 44 +++++++++++++++++++ packages/page-controller/src/actions.ts | 14 ++---- 4 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 packages/page-controller/src/actions.test.ts diff --git a/docs/agentic-testing/actions.md b/docs/agentic-testing/actions.md index 000e4dd81..4d5471b8a 100644 --- a/docs/agentic-testing/actions.md +++ b/docs/agentic-testing/actions.md @@ -5,6 +5,7 @@ 1. Build: `npm run build -w @page-agent/page-controller`. 2. Load `packages/page-controller/dist/lib/page-controller.js` in the page via `import()` (URL or Blob). 3. Call exported actions directly on DOM elements, never native input; do not instantiate `PageController` or modify the bundle. +4. Use computer use to reveal hover-only controls before calling their actions. Resolve elements again after framework renders; a detached element is not the current control. ## Cases @@ -22,6 +23,18 @@ | mui-input | [MUI Autocomplete](https://mui.com/material-ui/react-autocomplete/) — Controlled states | Type, select, clear. | Displayed inputValue/value stay consistent. | | radix-select | [Radix Select](https://www.radix-ui.com/primitives/docs/components/select) | Select, reopen, select another. | Value updates; focus returns to trigger. | +## Input and selection guards + +Use only synthetic test values. Inspect both the action outcome and the rendered value; an error alone does not prove that the input was rejected. + +| ID | Page / demo | Actions | Expected | +| ------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| native-text | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — text / password / textarea | Replace, repeat the same value, clear, and retype. | Values persist; repeated input and clearing succeed. | +| native-select | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — Dropdown (select) | Try `clickElement` and `inputTextElement`; then select `Two` and `Three` with `selectOptionElement`. | Misused actions throw without changing the selection; selecting by visible text succeeds. | +| native-input-guards | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — checkbox / radio / file | Call `inputTextElement` on each control. | Each call throws before changing checked state or opening a file chooser. | +| imask-phone | [IMask](https://imask.js.org/) — phone mask | Input digits, repeat the same digits, clear, and retype. | Formatting succeeds, including when the formatted value already matches the existing value. Clearing succeeds. | +| antd-number | [AntD InputNumber](https://ant.design/components/input-number/) — Formatter | Replace a formatted number, repeat the same number, then blur. | The action outcome agrees with the displayed and retained value. | + ## References - el-menu: [PR #378](https://github.com/alibaba/page-agent/pull/378) diff --git a/packages/page-controller/src/PageController.ts b/packages/page-controller/src/PageController.ts index ad9352853..155d7e30a 100644 --- a/packages/page-controller/src/PageController.ts +++ b/packages/page-controller/src/PageController.ts @@ -276,11 +276,14 @@ export class PageController extends EventTarget { this.assertIndexed() const element = getElementByIndex(this.selectorMap, index) const elemText = this.elementTextMap.get(index) - await inputTextElement(element, text) + const value = await inputTextElement(element, text) return { success: true, - message: `✅ Input text (${text}) into element (${elemText ?? index}).`, + message: + value === text + ? `✅ Input text (${text}) into element (${elemText ?? index}).` + : `✅ Input action completed for element (${elemText ?? index}). Note: requested ${JSON.stringify(text)}; current value is ${JSON.stringify(value)}.`, } } catch (error) { return { diff --git a/packages/page-controller/src/actions.test.ts b/packages/page-controller/src/actions.test.ts new file mode 100644 index 000000000..373f9e451 --- /dev/null +++ b/packages/page-controller/src/actions.test.ts @@ -0,0 +1,44 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { inputTextElement } from './actions' + +describe('inputTextElement', () => { + let input: HTMLInputElement + + beforeEach(() => { + input = document.createElement('input') + document.body.append(input) + vi.spyOn(document, 'elementFromPoint').mockReturnValue(input) + }) + + afterEach(() => { + input.remove() + vi.restoreAllMocks() + }) + + it('returns the normalized value when repeating equivalent input', async () => { + input.addEventListener('input', () => { + input.value = input.value.toUpperCase() + }) + + await expect(inputTextElement(input, 'hello')).resolves.toBe('HELLO') + await expect(inputTextElement(input, 'hello')).resolves.toBe('HELLO') + }) + + it.each(['previous', ''])( + 'reports a page-restored value (%j) without throwing', + async (value) => { + input.value = value + input.addEventListener('input', () => { + input.value = value + }) + + await expect(inputTextElement(input, 'next')).resolves.toBe(value) + } + ) + + it('returns an empty value after clearing the input', async () => { + input.value = 'previous' + await expect(inputTextElement(input, '')).resolves.toBe('') + }) +}) diff --git a/packages/page-controller/src/actions.ts b/packages/page-controller/src/actions.ts index cff44bf93..270a6c207 100644 --- a/packages/page-controller/src/actions.ts +++ b/packages/page-controller/src/actions.ts @@ -158,7 +158,7 @@ function assertAcceptsText(element: HTMLElement): void { /** * @private Internal method, subject to change at any time. */ -export async function inputTextElement(element: HTMLElement, text: string) { +export async function inputTextElement(element: HTMLElement, text: string): Promise { assertAcceptsText(element) const isContentEditable = element.isContentEditable @@ -248,24 +248,16 @@ export async function inputTextElement(element: HTMLElement, text: string) { element.blur() await waitFor(0.1) - return + return element.innerText } const input = element as HTMLInputElement | HTMLTextAreaElement - const valueBefore = input.value getNativeValueSetter(input).call(input, text) input.dispatchEvent(new Event('input', { bubbles: true })) await waitFor(0.1) - // Pages may legitimately reformat the value (masks, normalizers), so a different value - // is not a failure. An unchanged or emptied field means the page discarded the input. - const value = input.value - if (value !== text && (value === valueBefore || value === '')) { - throw new Error( - `The page discarded the input. The field now contains ${JSON.stringify(value)}. Inspect the current page state before retrying.` - ) - } + return input.value } /** From b7ba6eb6f1da418f79d1ad8d23d9fa92ca501dc3 Mon Sep 17 00:00:00 2001 From: Simon <10131203+gaomeng1900@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:15:31 +0800 Subject: [PATCH 3/3] test(page-controller): simplify action coverage --- docs/agentic-testing/actions.md | 14 +------ packages/page-controller/src/actions.test.ts | 44 -------------------- 2 files changed, 1 insertion(+), 57 deletions(-) delete mode 100644 packages/page-controller/src/actions.test.ts diff --git a/docs/agentic-testing/actions.md b/docs/agentic-testing/actions.md index 4d5471b8a..64e6028a8 100644 --- a/docs/agentic-testing/actions.md +++ b/docs/agentic-testing/actions.md @@ -5,7 +5,6 @@ 1. Build: `npm run build -w @page-agent/page-controller`. 2. Load `packages/page-controller/dist/lib/page-controller.js` in the page via `import()` (URL or Blob). 3. Call exported actions directly on DOM elements, never native input; do not instantiate `PageController` or modify the bundle. -4. Use computer use to reveal hover-only controls before calling their actions. Resolve elements again after framework renders; a detached element is not the current control. ## Cases @@ -22,18 +21,7 @@ | el-scroll | [Element Plus Scrollbar](https://element-plus.org/en-US/component/scrollbar.html) — vertical / horizontal | Scroll target container in both directions. | Correct region and direction. | | mui-input | [MUI Autocomplete](https://mui.com/material-ui/react-autocomplete/) — Controlled states | Type, select, clear. | Displayed inputValue/value stay consistent. | | radix-select | [Radix Select](https://www.radix-ui.com/primitives/docs/components/select) | Select, reopen, select another. | Value updates; focus returns to trigger. | - -## Input and selection guards - -Use only synthetic test values. Inspect both the action outcome and the rendered value; an error alone does not prove that the input was rejected. - -| ID | Page / demo | Actions | Expected | -| ------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | -| native-text | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — text / password / textarea | Replace, repeat the same value, clear, and retype. | Values persist; repeated input and clearing succeed. | -| native-select | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — Dropdown (select) | Try `clickElement` and `inputTextElement`; then select `Two` and `Three` with `selectOptionElement`. | Misused actions throw without changing the selection; selecting by visible text succeeds. | -| native-input-guards | [Selenium web form](https://www.selenium.dev/selenium/web/web-form.html) — checkbox / radio / file | Call `inputTextElement` on each control. | Each call throws before changing checked state or opening a file chooser. | -| imask-phone | [IMask](https://imask.js.org/) — phone mask | Input digits, repeat the same digits, clear, and retype. | Formatting succeeds, including when the formatted value already matches the existing value. Clearing succeeds. | -| antd-number | [AntD InputNumber](https://ant.design/components/input-number/) — Formatter | Replace a formatted number, repeat the same number, then blur. | The action outcome agrees with the displayed and retained value. | +| imask-input | [IMask](https://imask.js.org/) — phone mask | Fill, repeat, clear, retype. | Formatting, clearing, and repeated input work. | ## References diff --git a/packages/page-controller/src/actions.test.ts b/packages/page-controller/src/actions.test.ts deleted file mode 100644 index 373f9e451..000000000 --- a/packages/page-controller/src/actions.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -import { inputTextElement } from './actions' - -describe('inputTextElement', () => { - let input: HTMLInputElement - - beforeEach(() => { - input = document.createElement('input') - document.body.append(input) - vi.spyOn(document, 'elementFromPoint').mockReturnValue(input) - }) - - afterEach(() => { - input.remove() - vi.restoreAllMocks() - }) - - it('returns the normalized value when repeating equivalent input', async () => { - input.addEventListener('input', () => { - input.value = input.value.toUpperCase() - }) - - await expect(inputTextElement(input, 'hello')).resolves.toBe('HELLO') - await expect(inputTextElement(input, 'hello')).resolves.toBe('HELLO') - }) - - it.each(['previous', ''])( - 'reports a page-restored value (%j) without throwing', - async (value) => { - input.value = value - input.addEventListener('input', () => { - input.value = value - }) - - await expect(inputTextElement(input, 'next')).resolves.toBe(value) - } - ) - - it('returns an empty value after clearing the input', async () => { - input.value = 'previous' - await expect(inputTextElement(input, '')).resolves.toBe('') - }) -})