From e98e825c3a338ec2d5e30d456067023fcccac273 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Sat, 12 Sep 2026 13:15:10 +0200 Subject: [PATCH 1/2] test: cover webview and app install/uninstall on both platforms The e2e suite never installed or uninstalled an app, and never touched a webview. It also reported far less coverage than it actually had: device commands run inside the daemon, and the daemon was still alive when `go tool covdata` read the counter directory, so none of its work counted. Tests, one group per platform against the playground app: - install and uninstall: uninstall, assert the package is gone from `apps list`, install the release build from disk, then assert every field `apps list` and `apps install` report today, so a dropped `appName` or `versionCode` fails here instead of shipping. - webview: tap into the playground webview screen, then list, url, title, eval, content, query, wait, goto, back, forward and reload, plus negative paths for an unknown webview id and an unknown device. - a separate group launches the system settings app and asserts `webview list` fails on a build the agent cannot attach to. It owns its setup, since `webview list` reads the foreground app and would otherwise break the state the playground group sets up once. `goto`, `back` and `forward` return before the navigation commits, and `wait --state load` can observe the outgoing page's load event, so url assertions poll instead of sleeping. The app restores its last webview url between launches, so the setup navigates back to the sample page rather than assuming a fresh one. Fixes: - Makefile stops the daemon before the suites, so the one they spawn inherits GOCOVERDIR, and again afterwards, so it exits cleanly and flushes its counters before covdata reads them. - `test:ios` ran `--project=simulator`, which made it a silent duplicate of `test:simulator` and left real-device coverage unreachable. The two are now one honest `test:ios-simulator`, and the Makefile runs it instead of the commented-out pair. Coverage: 26.3% -> 43.1% overall, with no change to the product code. commands/webview.go 0% -> 9 of 10 functions at 100%, commands/apps.go 0% -> 44-86%, devices/{android,ios}_webview.go 0% -> 74.6%. --- Makefile | 9 +- test/android.spec.ts | 313 ++++++++++++++++++++++++++++++++++++++--- test/package.json | 3 +- test/playground.ts | 56 ++++++++ test/shapes.ts | 43 ++++++ test/simulator.spec.ts | 309 +++++++++++++++++++++++++++++++++++++++- test/types.ts | 43 ++++++ test/webview.ts | 109 ++++++++++++++ 8 files changed, 854 insertions(+), 31 deletions(-) create mode 100644 test/playground.ts create mode 100644 test/webview.ts diff --git a/Makefile b/Makefile index 96d14470..9f5d981a 100644 --- a/Makefile +++ b/Makefile @@ -24,13 +24,18 @@ test-cover: build-cover test-e2e: build-cover rm -rf test/coverage mkdir -p test/coverage + # device commands run inside the daemon, so its counters are the bulk of the + # coverage. stop any daemon already running: the suites spawn a fresh one that + # inherits GOCOVERDIR, and a -cover binary only writes counters when it exits. + ./mobilecli daemon stop go test ./... -v -race -covermode=atomic -args -test.gocoverdir=$(CURDIR)/test/coverage (cd test && npm run test:server) (cd test && npm run test:daemon) - # (cd test && npm run test:ios) - (cd test && npm run test:simulator) + (cd test && npm run test:ios-simulator) (cd test && npm run test:android) (cd test && npm run test:emulator) + # flushes the daemon's counters into test/coverage before they are read back + GOCOVERDIR=$(CURDIR)/test/coverage ./mobilecli daemon stop go tool covdata textfmt -i=test/coverage -o coverage.out go tool cover -html=coverage.out -o coverage.html go tool cover -func=coverage.out diff --git a/test/android.spec.ts b/test/android.spec.ts index dcf0c110..2cc8039c 100644 --- a/test/android.spec.ts +++ b/test/android.spec.ts @@ -3,23 +3,45 @@ import {execFileSync, spawn} from 'child_process'; import * as path from 'path'; import * as fs from 'fs'; import * as os from 'os'; -import type {UIElement, UIDumpResponse, ForegroundAppResponse} from './types'; +import type {InstalledApp, Point, UIElement, UIDumpResponse, ForegroundAppResponse} from './types'; import {coverageEnv} from './coverage'; import { expectAppShape, expectFsListingShape, expectForegroundAppShape, + expectErrorEnvelope, + expectInstallResultShape, + expectInstalledAppShape, expectOkEnvelope, expectUIDumpShape, } from './shapes'; +import { + centerOf, + expectWebViewShape, + expectWebViewUrlToBecome, + findWebViewButton, + flattenElements, + WEBVIEW_COMMANDS_TAKING_AN_ID, + WEBVIEW_DONE_GREETING, + WEBVIEW_DONE_URL, + WEBVIEW_MISSING_ID, + WEBVIEW_SAMPLE_TITLE, + WEBVIEW_SAMPLE_URL, +} from './webview'; +import type {WebViewInfo, WebViewQueryResult} from './webview'; +import { + downloadPlayground, + PLAYGROUND_APP_NAME, + PLAYGROUND_APP_VERSION, + PLAYGROUND_APP_VERSION_CODE, + PLAYGROUND_PACKAGE, +} from './playground'; const mobilecliBinary = path.join(__dirname, '..', 'mobilecli'); // settings is present on every android image, unlike chrome or play store const SETTINGS_PACKAGE = 'com.android.settings'; -const PLAYGROUND_PACKAGE = 'com.mobilenext.playground'; - // the settings search ui is a separate package that joins the settings task via // taskAffinity=com.android.settings.root. force-stopping settings alone leaves its // activity on top of that task, so `am start` resumes a task settings no longer owns @@ -43,11 +65,6 @@ type Dimensions = { height: number; }; -type Point = { - x: number; - y: number; -}; - // this spec is written against an emulator: it force-stops apps, writes to // /sdcard, and leaves the device on arbitrary screens. a physical android device // reports type "real" and is left alone. @@ -127,7 +144,7 @@ test.describe('Android Tests', () => { const apps = listApps(device!.id); apps.forEach(expectAppShape); - expect(apps.map((a: any) => a.packageName)).toContain(SETTINGS_PACKAGE); + expect(apps.map(app => app.packageName)).toContain(SETTINGS_PACKAGE); }); test('should launch Settings app and verify it is in foreground', async () => { @@ -259,6 +276,188 @@ test.describe('Android Tests', () => { } }); + test.describe('install and uninstall playground', () => { + // installs an app we own rather than anything already on the image, so the + // uninstall half is safe to run for real. also leaves playground installed + // for the app-container fs group below. + test.skip(({deviceType}) => deviceType === 'real', 'leaves a physical phone modified'); + + let apkPath: string; + + test.beforeAll(async () => { + apkPath = await downloadPlayground('android'); + }); + + test('should uninstall playground and no longer list it', () => { + test.skip(!device, 'No Android device found'); + + uninstallPlaygroundIfPresent(device!.id); + expect(installedPackageNames(device!.id)).not.toContain(PLAYGROUND_PACKAGE); + }); + + test('should install playground from a local apk', () => { + test.skip(!device, 'No Android device found'); + + const result: unknown = mobilecliJson(['apps', 'install', apkPath, '--device', device!.id]).data; + expectInstallResultShape(result); + expect(result.app.packageName).toBe(PLAYGROUND_PACKAGE); + expect(result.app.version).toBe(PLAYGROUND_APP_VERSION); + expect(result.app.versionCode).toBe(PLAYGROUND_APP_VERSION_CODE); + }); + + test('should list playground with every field it reports today', () => { + test.skip(!device, 'No Android device found'); + + const app = findInstalledApp(device!.id, PLAYGROUND_PACKAGE); + expect(app, `${PLAYGROUND_PACKAGE} missing after install`).toBeDefined(); + expectInstalledAppShape(app); + expect(app!.appName).toBe(PLAYGROUND_APP_NAME); + expect(app!.version).toBe(PLAYGROUND_APP_VERSION); + expect(app!.versionCode).toBe(PLAYGROUND_APP_VERSION_CODE); + }); + }); + + test.describe('webview', () => { + // the playground webview screen is the one embedded webview we control on + // both platforms. a real handset would be left on an arbitrary screen. + test.skip(({deviceType}) => deviceType === 'real', 'leaves a physical phone modified'); + + let webViewId: string; + + test.beforeAll(async () => { + if (!device) return; + await openPlaygroundWebViewScreen(device.id); + await sleep(3000); + webViewId = firstWebView(device.id).id; + + // the app restores whatever url the webview last showed, so start every + // run from the sample page instead of inheriting the previous run's state + webViewGoto(device.id, webViewId, WEBVIEW_SAMPLE_URL); + webViewWait(device.id, webViewId, 'load'); + }); + + test('should list the playground webview', () => { + test.skip(!device, 'No Android device found'); + + const webView = firstWebView(device!.id); + expect(webView.url).toBe(WEBVIEW_SAMPLE_URL); + expect(webView.title).toBe(WEBVIEW_SAMPLE_TITLE); + expect(webView.bundleId).toBe(PLAYGROUND_PACKAGE); + expect(webView.isVisible).toBe(true); + }); + + test('should report the url and title of the webview', () => { + test.skip(!device, 'No Android device found'); + + expect(webViewUrl(device!.id, webViewId)).toBe(WEBVIEW_SAMPLE_URL); + expect(webViewTitle(device!.id, webViewId)).toBe(WEBVIEW_SAMPLE_TITLE); + }); + + test('should evaluate javascript inside the webview', () => { + test.skip(!device, 'No Android device found'); + + expect(webViewEval(device!.id, webViewId, 'document.title')).toBe(WEBVIEW_SAMPLE_TITLE); + }); + + test('should dump the html content of the webview', () => { + test.skip(!device, 'No Android device found'); + + const html = webViewContent(device!.id, webViewId); + expect(html).toContain('
{ + test.skip(!device, 'No Android device found'); + + const inputs = webViewQuery(device!.id, webViewId, 'input#name'); + expect(inputs.length).toBe(1); + expect(inputs[0].tag).toBe('input'); + expect(inputs[0].id).toBe('name'); + }); + + test('should wait for the webview to finish loading', () => { + test.skip(!device, 'No Android device found'); + + webViewWait(device!.id, webViewId, 'domcontentloaded'); + webViewWait(device!.id, webViewId, 'load'); + }); + + test('should navigate the webview to another url', async () => { + test.skip(!device, 'No Android device found'); + + webViewGoto(device!.id, webViewId, WEBVIEW_DONE_URL); + webViewWait(device!.id, webViewId, 'load'); + + await expectWebViewUrlToBecome(() => webViewUrl(device!.id, webViewId), WEBVIEW_DONE_URL); + expect(webViewQuery(device!.id, webViewId, 'h1')[0].text).toBe(WEBVIEW_DONE_GREETING); + }); + + test('should go back to the page it navigated away from', async () => { + test.skip(!device, 'No Android device found'); + + webViewGoBack(device!.id, webViewId); + + await expectWebViewUrlToBecome(() => webViewUrl(device!.id, webViewId), WEBVIEW_SAMPLE_URL); + }); + + test('should go forward again', async () => { + test.skip(!device, 'No Android device found'); + + webViewGoForward(device!.id, webViewId); + + await expectWebViewUrlToBecome(() => webViewUrl(device!.id, webViewId), WEBVIEW_DONE_URL); + }); + + test('should report an error for every command given an unknown webview id', () => { + test.skip(!device, 'No Android device found'); + + for (const [subcommand, ...args] of WEBVIEW_COMMANDS_TAKING_AN_ID) { + const message = webViewCommandError(device!.id, [subcommand, WEBVIEW_MISSING_ID, ...args]); + expect(message, `${subcommand} accepted an unknown webview id`).toContain(WEBVIEW_MISSING_ID); + } + }); + + test('should report an error when the device does not exist', () => { + test.skip(!device, 'No Android device found'); + + for (const [subcommand, ...args] of WEBVIEW_COMMANDS_TAKING_AN_ID) { + const message = webViewCommandError('no-such-device', [subcommand, WEBVIEW_MISSING_ID, ...args]); + expect(message, `${subcommand} accepted an unknown device`).toContain('error finding device'); + } + expect(webViewCommandError('no-such-device', ['list'])).toContain('error finding device'); + }); + + test('should reload the webview and stay on the same url', async () => { + test.skip(!device, 'No Android device found'); + + webViewReload(device!.id, webViewId); + webViewWait(device!.id, webViewId, 'load'); + + await expectWebViewUrlToBecome(() => webViewUrl(device!.id, webViewId), WEBVIEW_DONE_URL); + }); + + }); + + // its own describe, not a test inside the playground group above: `webview list` + // reads the foreground app, so this launches a different app and would break the + // shared state the playground tests set up once in their beforeAll + test.describe('webview on an app that cannot be inspected', () => { + test.beforeAll(async () => { + if (!device) return; + launchApp(device.id, SETTINGS_PACKAGE); + await sleep(3000); + }); + + test('should fail to list webviews in an app that is not debuggable', () => { + test.skip(!device, 'No Android device found'); + + const message = webViewCommandError(device!.id, ['list']); + expect(message).toContain('webview list failed'); + expect(message).toContain(SETTINGS_PACKAGE); + }); + }); + test.describe('fs operations on app container (com.mobilenext.playground)', () => { // reading an app sandbox needs a debuggable build installed, which is // guaranteed on an emulator image but not on someone's phone @@ -379,8 +578,26 @@ function sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } -function listApps(deviceId: string): any[] { - return mobilecliJson(['apps', 'list', '--device', deviceId]).data; +function listApps(deviceId: string): InstalledApp[] { + return mobilecliJson(['apps', 'list', '--device', deviceId]).data as InstalledApp[]; +} + +function installedPackageNames(deviceId: string): string[] { + return listApps(deviceId).map(app => app.packageName); +} + +function findInstalledApp(deviceId: string, packageName: string): InstalledApp | undefined { + return listApps(deviceId).find(app => app.packageName === packageName); +} + +// uninstalling an app that is not installed is not an error worth failing on: the +// point of this call is only to reach a known-clean starting state +function uninstallPlaygroundIfPresent(deviceId: string): void { + try { + mobilecliJson(['apps', 'uninstall', PLAYGROUND_PACKAGE, '--device', deviceId]); + } catch { + // already absent + } } function launchApp(deviceId: string, packageName: string): void { @@ -438,11 +655,6 @@ function pressButton(deviceId: string, button: string): void { mobilecli(['io', 'button', button, '--device', deviceId]); } -// android returns the view hierarchy as a nested tree, so flatten it before searching -function flattenElements(elements: UIElement[]): UIElement[] { - return elements.flatMap(element => [element, ...flattenElements(element.children ?? [])]); -} - function findElementByText(uiDump: UIDumpResponse, text: string): UIElement { const element = flattenElements(uiDump.data.elements).find(el => el.text === text); if (!element) { @@ -461,11 +673,70 @@ function allTextsIn(uiDump: UIDumpResponse): string[] { return flattenElements(uiDump.data.elements).map(el => el.text).filter(Boolean) as string[]; } -function centerOf(element: UIElement): Point { - return { - x: element.rect.x + Math.floor(element.rect.width / 2), - y: element.rect.y + Math.floor(element.rect.height / 2), - }; +async function openPlaygroundWebViewScreen(deviceId: string): Promise { + launchApp(deviceId, PLAYGROUND_PACKAGE); + await sleep(3000); + const button = findWebViewButton(dumpUI(deviceId)); + tap(deviceId, centerOf(button).x, centerOf(button).y); +} + +// runs a webview command that is expected to fail and returns the error message +function webViewCommandError(deviceId: string, args: string[]): string { + try { + mobilecliJson(['webview', ...args, '--device', deviceId]); + } catch (error: unknown) { + const stdout = (error as {stdout?: string}).stdout ?? ''; + return expectErrorEnvelope(JSON.parse(stdout)); + } + + throw new Error(`webview ${args.join(' ')} unexpectedly succeeded`); +} + +function firstWebView(deviceId: string): WebViewInfo { + const webViews = mobilecliJson(['webview', 'list', '--device', deviceId]).data as unknown[]; + expect(webViews.length, 'no webview reported by the playground app').toBeGreaterThan(0); + expectWebViewShape(webViews[0]); + return webViews[0]; +} + +function webViewUrl(deviceId: string, webViewId: string): string { + return mobilecliJson(['webview', 'url', webViewId, '--device', deviceId]).data as string; +} + +function webViewTitle(deviceId: string, webViewId: string): string { + return mobilecliJson(['webview', 'title', webViewId, '--device', deviceId]).data as string; +} + +function webViewContent(deviceId: string, webViewId: string): string { + return mobilecliJson(['webview', 'content', webViewId, '--device', deviceId]).data as string; +} + +function webViewEval(deviceId: string, webViewId: string, expression: string): unknown { + return mobilecliJson(['webview', 'eval', webViewId, expression, '--device', deviceId]).data; +} + +function webViewQuery(deviceId: string, webViewId: string, selector: string): WebViewQueryResult[] { + return mobilecliJson(['webview', 'query', webViewId, selector, '--device', deviceId]).data as WebViewQueryResult[]; +} + +function webViewGoto(deviceId: string, webViewId: string, url: string): void { + mobilecliJson(['webview', 'goto', webViewId, url, '--device', deviceId]); +} + +function webViewReload(deviceId: string, webViewId: string): void { + mobilecliJson(['webview', 'reload', webViewId, '--device', deviceId]); +} + +function webViewGoBack(deviceId: string, webViewId: string): void { + mobilecliJson(['webview', 'back', webViewId, '--device', deviceId]); +} + +function webViewGoForward(deviceId: string, webViewId: string): void { + mobilecliJson(['webview', 'forward', webViewId, '--device', deviceId]); +} + +function webViewWait(deviceId: string, webViewId: string, state: string): void { + mobilecliJson(['webview', 'wait', webViewId, '--state', state, '--timeout', '15000', '--device', deviceId]); } function getAppContainerPath(deviceId: string, packageName: string): string { diff --git a/test/package.json b/test/package.json index a3b9ac20..801971fe 100644 --- a/test/package.json +++ b/test/package.json @@ -7,8 +7,7 @@ "test": "playwright test", "test:server": "playwright test --project=server", "test:daemon": "playwright test --project=daemon", - "test:ios": "playwright test --project=simulator", - "test:simulator": "playwright test --project=simulator", + "test:ios-simulator": "playwright test --project=simulator", "test:emulator": "playwright test --project=emulator", "test:android": "playwright test --project=android" }, diff --git a/test/playground.ts b/test/playground.ts new file mode 100644 index 00000000..b8eb48dd --- /dev/null +++ b/test/playground.ts @@ -0,0 +1,56 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +// the playground app is the one app both specs can install and uninstall freely: +// it is ours, it is not part of any system image, and its bundle id is identical +// on both platforms. bump this one constant to move every spec to a new build. +export const PLAYGROUND_VERSION = 'v1.0.6'; +export const PLAYGROUND_PACKAGE = 'com.mobilenext.playground'; + +// values the release is built with, asserted so a regression that drops a field +// from `apps list` or `apps install` fails here instead of shipping +export const PLAYGROUND_APP_NAME = 'Playground'; +export const PLAYGROUND_APP_VERSION = '1.0'; +export const PLAYGROUND_APP_VERSION_CODE = '1'; + +// android installs an apk; the ios simulator takes a zipped .app bundle +type PlaygroundPlatform = 'android' | 'ios'; + +const ARTIFACT_EXTENSION: Record = { + android: 'apk', + ios: 'zip', +}; + +function artifactUrl(platform: PlaygroundPlatform): string { + const version = PLAYGROUND_VERSION.replace(/^v/, ''); + const file = `Playground-${version}.${ARTIFACT_EXTENSION[platform]}`; + return `https://github.com/mobile-next/playground/releases/download/${PLAYGROUND_VERSION}/${file}`; +} + +// cached under the version, so a rerun on the same build skips the download and a +// version bump can never install a stale artifact left behind by an earlier run +function artifactPath(platform: PlaygroundPlatform): string { + const version = PLAYGROUND_VERSION.replace(/^v/, ''); + return path.join(os.tmpdir(), `mobilecli-playground-${version}.${ARTIFACT_EXTENSION[platform]}`); +} + +export async function downloadPlayground(platform: PlaygroundPlatform): Promise { + const localPath = artifactPath(platform); + if (fs.existsSync(localPath) && fs.statSync(localPath).size > 0) { + return localPath; + } + + const url = artifactUrl(platform); + const response = await fetch(url); + if (!response.ok) { + throw new Error(`failed to download ${url}: ${response.status} ${response.statusText}`); + } + + // written to a temp name first, so an interrupted run cannot leave a truncated + // file that the next run would happily treat as cached + const partialPath = `${localPath}.partial`; + fs.writeFileSync(partialPath, Buffer.from(await response.arrayBuffer())); + fs.renameSync(partialPath, localPath); + return localPath; +} diff --git a/test/shapes.ts b/test/shapes.ts index a50e88a1..fa3712bb 100644 --- a/test/shapes.ts +++ b/test/shapes.ts @@ -1,4 +1,5 @@ import {expect} from '@playwright/test'; +import type {InstallResult, InstalledApp, UninstallResult} from './types'; // Shared wire-format assertions. // @@ -100,3 +101,45 @@ export function expectOkEnvelope(response: any): any { expect(response.data, 'ok envelope must carry data').toBeDefined(); return response.data; } + +// `apps list` reports all four fields on both platforms today. expectAppShape stays +// lenient for arbitrary system apps; this one is the strict contract, asserted +// against an app we ship ourselves, so dropping a field is caught as a regression. +export function expectInstalledAppShape(app: unknown): asserts app is InstalledApp { + const fields: (keyof InstalledApp)[] = ['packageName', 'appName', 'version', 'versionCode']; + const record = app as Record; + for (const field of fields) { + expect(typeof record?.[field], `app.${field}: ${JSON.stringify(app)}`).toBe('string'); + expect((record[field] as string).length, `app.${field} is empty`).toBeGreaterThan(0); + } +} + +// `apps install` echoes a human-readable message plus the metadata it read out of +// the artifact. it reports no appName, unlike `apps list`. +export function expectInstallResultShape(data: unknown): asserts data is InstallResult { + const result = data as {message?: unknown; app?: Record}; + expect(typeof result?.message, `install message: ${JSON.stringify(data)}`).toBe('string'); + expect((result.message as string).length).toBeGreaterThan(0); + + const fields: (keyof InstallResult['app'])[] = ['packageName', 'version', 'versionCode']; + for (const field of fields) { + expect(typeof result?.app?.[field], `install app.${field}: ${JSON.stringify(data)}`).toBe('string'); + expect((result.app![field] as string).length, `install app.${field} is empty`).toBeGreaterThan(0); + } +} + +// `apps uninstall` answers with just the bundle id it removed +export function expectUninstallResultShape(data: unknown): asserts data is UninstallResult { + const result = data as {packageName?: unknown}; + expect(typeof result?.packageName, `uninstall result: ${JSON.stringify(data)}`).toBe('string'); + expect((result.packageName as string).length).toBeGreaterThan(0); +} + +// the error half of the envelope: a failing command exits non-zero and prints +// this instead of `data` +export function expectErrorEnvelope(response: unknown): string { + const envelope = response as {status?: unknown; error?: unknown}; + expect(envelope?.status, `expected status error, got: ${JSON.stringify(response)}`).toBe('error'); + expect(typeof envelope.error, 'error envelope must carry a message').toBe('string'); + return envelope.error as string; +} diff --git a/test/simulator.spec.ts b/test/simulator.spec.ts index 1a54d31d..3b4ecd94 100644 --- a/test/simulator.spec.ts +++ b/test/simulator.spec.ts @@ -14,10 +14,33 @@ import { expectDeviceShape, expectFsListingShape, expectForegroundAppShape, + expectErrorEnvelope, + expectInstallResultShape, + expectInstalledAppShape, expectOkEnvelope, expectUIDumpShape, } from './shapes'; -import type {UIElement, UIDumpResponse, DeviceInfoResponse, ForegroundAppResponse} from './types'; +import { + centerOf, + expectWebViewShape, + expectWebViewUrlToBecome, + findWebViewButton, + WEBVIEW_COMMANDS_TAKING_AN_ID, + WEBVIEW_DONE_GREETING, + WEBVIEW_DONE_URL, + WEBVIEW_MISSING_ID, + WEBVIEW_SAMPLE_TITLE, + WEBVIEW_SAMPLE_URL, +} from './webview'; +import type {WebViewInfo, WebViewQueryResult} from './webview'; +import { + downloadPlayground, + PLAYGROUND_APP_NAME, + PLAYGROUND_APP_VERSION, + PLAYGROUND_APP_VERSION_CODE, + PLAYGROUND_PACKAGE, +} from './playground'; +import type {AppsListResponse, InstalledApp, UIElement, UIDumpResponse, DeviceInfoResponse, ForegroundAppResponse} from './types'; type Dimensions = { width: number; @@ -26,6 +49,9 @@ type Dimensions = { const TEST_SERVER_URL = 'http://localhost:12001'; +// ships on every simulator image and is never a debug build +const IOS_SETTINGS_BUNDLE_ID = 'com.apple.Preferences'; + test.describe('iOS Simulator Tests', () => { [/*'16',*/ /*'17', '18',*/ '26'].forEach((iosVersion) => { test.describe(`iOS ${iosVersion}`, () => { @@ -293,8 +319,186 @@ test.describe('iOS Simulator Tests', () => { verifyRawViewtreeDump(rawDump); }); + test.describe('install and uninstall playground', () => { + // installs an app we own rather than anything on the simulator image, so + // the uninstall half is safe. also leaves playground installed for the + // app-container fs group below. + let zipPath: string; + + test.beforeAll(async () => { + zipPath = await downloadPlayground('ios'); + }); + + test('should uninstall playground and no longer list it', async () => { + test.skip(!simulatorId, 'simulator not found'); + + uninstallPlaygroundIfPresent(simulatorId); + expect(installedPackageNames(simulatorId)).not.toContain(PLAYGROUND_PACKAGE); + }); + + test('should install playground from a local zip', async () => { + test.skip(!simulatorId, 'simulator not found'); + + const result: unknown = mobilecli(['apps', 'install', zipPath, '--device', simulatorId]).data; + expectInstallResultShape(result); + expect(result.app.packageName).toBe(PLAYGROUND_PACKAGE); + expect(result.app.version).toBe(PLAYGROUND_APP_VERSION); + expect(result.app.versionCode).toBe(PLAYGROUND_APP_VERSION_CODE); + }); + + test('should list playground with every field it reports today', async () => { + test.skip(!simulatorId, 'simulator not found'); + + const app = findInstalledApp(simulatorId, PLAYGROUND_PACKAGE); + expect(app, `${PLAYGROUND_PACKAGE} missing after install`).toBeDefined(); + expectInstalledAppShape(app); + expect(app!.appName).toBe(PLAYGROUND_APP_NAME); + expect(app!.version).toBe(PLAYGROUND_APP_VERSION); + expect(app!.versionCode).toBe(PLAYGROUND_APP_VERSION_CODE); + }); + }); + + test.describe('webview', () => { + // the playground webview screen is the one embedded webview we control on + // both platforms. a real handset would be left on an arbitrary screen. + let webViewId: string; + + test.beforeAll(async () => { + if (!simulatorId) return; + await openPlaygroundWebViewScreen(simulatorId); + await sleep(3000); + webViewId = firstWebView(simulatorId).id; + + // the app restores whatever url the webview last showed, so start every + // run from the sample page instead of inheriting the previous run's state + webViewGoto(simulatorId, webViewId, WEBVIEW_SAMPLE_URL); + webViewWait(simulatorId, webViewId, 'load'); + }); + + test('should list the playground webview', () => { + test.skip(!simulatorId, 'simulator not found'); + + const webView = firstWebView(simulatorId); + expect(webView.url).toBe(WEBVIEW_SAMPLE_URL); + expect(webView.title).toBe(WEBVIEW_SAMPLE_TITLE); + // ios reports no owning bundle for an inspected webview + expect(webView.bundleId).toBe(''); + expect(webView.isVisible).toBe(true); + }); + + test('should report the url and title of the webview', () => { + test.skip(!simulatorId, 'simulator not found'); + + expect(webViewUrl(simulatorId, webViewId)).toBe(WEBVIEW_SAMPLE_URL); + expect(webViewTitle(simulatorId, webViewId)).toBe(WEBVIEW_SAMPLE_TITLE); + }); + + test('should evaluate javascript inside the webview', () => { + test.skip(!simulatorId, 'simulator not found'); + + expect(webViewEval(simulatorId, webViewId, 'document.title')).toBe(WEBVIEW_SAMPLE_TITLE); + }); + + test('should dump the html content of the webview', () => { + test.skip(!simulatorId, 'simulator not found'); + + const html = webViewContent(simulatorId, webViewId); + expect(html).toContain(' { + test.skip(!simulatorId, 'simulator not found'); + + const inputs = webViewQuery(simulatorId, webViewId, 'input#name'); + expect(inputs.length).toBe(1); + expect(inputs[0].tag).toBe('input'); + expect(inputs[0].id).toBe('name'); + }); + + test('should wait for the webview to finish loading', () => { + test.skip(!simulatorId, 'simulator not found'); + + webViewWait(simulatorId, webViewId, 'domcontentloaded'); + webViewWait(simulatorId, webViewId, 'load'); + }); + + test('should navigate the webview to another url', async () => { + test.skip(!simulatorId, 'simulator not found'); + + webViewGoto(simulatorId, webViewId, WEBVIEW_DONE_URL); + webViewWait(simulatorId, webViewId, 'load'); + + await expectWebViewUrlToBecome(() => webViewUrl(simulatorId, webViewId), WEBVIEW_DONE_URL); + expect(webViewQuery(simulatorId, webViewId, 'h1')[0].text).toBe(WEBVIEW_DONE_GREETING); + }); + + test('should go back to the page it navigated away from', async () => { + test.skip(!simulatorId, 'simulator not found'); + + webViewGoBack(simulatorId, webViewId); + await sleep(2000); + + await expectWebViewUrlToBecome(() => webViewUrl(simulatorId, webViewId), WEBVIEW_SAMPLE_URL); + }); + + test('should go forward again', async () => { + test.skip(!simulatorId, 'simulator not found'); + + webViewGoForward(simulatorId, webViewId); + await sleep(2000); + + await expectWebViewUrlToBecome(() => webViewUrl(simulatorId, webViewId), WEBVIEW_DONE_URL); + }); + + test('should report an error for every command given an unknown webview id', () => { + test.skip(!simulatorId, 'simulator not found'); + + for (const [subcommand, ...args] of WEBVIEW_COMMANDS_TAKING_AN_ID) { + const message = webViewCommandError(simulatorId, [subcommand, WEBVIEW_MISSING_ID, ...args]); + expect(message, `${subcommand} accepted an unknown webview id`).toContain(WEBVIEW_MISSING_ID); + } + }); + + test('should report an error when the device does not exist', () => { + test.skip(!simulatorId, 'simulator not found'); + + const message = webViewCommandError('no-such-device', ['list']); + expect(message).toContain('error finding device'); + }); + + test('should reload the webview and stay on the same url', async () => { + test.skip(!simulatorId, 'simulator not found'); + + webViewReload(simulatorId, webViewId); + webViewWait(simulatorId, webViewId, 'load'); + + await expectWebViewUrlToBecome(() => webViewUrl(simulatorId, webViewId), WEBVIEW_DONE_URL); + }); + + }); + + // its own describe, not a test inside the playground group above: `webview list` + // reads the foreground app, so this launches a different app and would break the + // shared state the playground tests set up once in their beforeAll + test.describe('webview on an app that cannot be inspected', () => { + test.beforeAll(async () => { + if (!simulatorId) return; + launchApp(simulatorId, IOS_SETTINGS_BUNDLE_ID); + await sleep(3000); + }); + + test('should fail to list webviews in an app that is not debuggable', () => { + test.skip(!simulatorId, 'simulator not found'); + + const message = webViewCommandError(simulatorId, ['list']); + expect(message).toContain('webview list failed'); + expect(message).toContain(IOS_SETTINGS_BUNDLE_ID); + }); + }); + test.describe('fs operations on app container (com.mobilenext.playground)', () => { - const packageName = 'com.mobilenext.playground'; + const packageName = PLAYGROUND_PACKAGE; let containerPath: string; let remoteDir: string; let remoteFile: string; @@ -453,13 +657,35 @@ function verifyDeviceInfo(info: DeviceInfoResponse, simulatorId: string): void { expect(info.data.device.state).toBe('online'); } -function listApps(simulatorId: string): any { - return mobilecli(['apps', 'list', '--device', simulatorId]); +function listApps(simulatorId: string): AppsListResponse { + return mobilecli(['apps', 'list', '--device', simulatorId]) as AppsListResponse; } -function verifyAppsListContainsSafari(response: any): void { +function verifyAppsListContainsSafari(response: AppsListResponse): void { response.data.forEach(expectAppShape); - expect(response.data.map((a: any) => a.packageName)).toContain('com.apple.mobilesafari'); + expect(response.data.map(app => app.packageName)).toContain('com.apple.mobilesafari'); +} + +function installedApps(simulatorId: string): InstalledApp[] { + return listApps(simulatorId).data as InstalledApp[]; +} + +function installedPackageNames(simulatorId: string): string[] { + return installedApps(simulatorId).map(app => app.packageName); +} + +function findInstalledApp(simulatorId: string, packageName: string): InstalledApp | undefined { + return installedApps(simulatorId).find(app => app.packageName === packageName); +} + +// uninstalling an app that is not installed is not an error worth failing on: the +// point of this call is only to reach a known-clean starting state +function uninstallPlaygroundIfPresent(simulatorId: string): void { + try { + mobilecli(['apps', 'uninstall', PLAYGROUND_PACKAGE, '--device', simulatorId]); + } catch { + // already absent + } } function launchApp(simulatorId: string, packageName: string): void { @@ -659,6 +885,77 @@ function verifyRawViewtreeDump(response: any): void { expect(Array.isArray(rawData.children)).toBe(true); } +async function openPlaygroundWebViewScreen(simulatorId: string): Promise { + launchApp(simulatorId, PLAYGROUND_PACKAGE); + await sleep(3000); + const button = findWebViewButton(dumpUI(simulatorId)); + tap(simulatorId, centerOf(button).x, centerOf(button).y); +} + +// runs a webview command that is expected to fail and returns the error message +function webViewCommandError(simulatorId: string, args: string[]): string { + try { + mobilecli(['webview', ...args, '--device', simulatorId]); + } catch (error: unknown) { + const stdout = (error as {stdout?: string}).stdout ?? ''; + return expectErrorEnvelope(JSON.parse(stdout)); + } + + throw new Error(`webview ${args.join(' ')} unexpectedly succeeded`); +} + +function firstWebView(simulatorId: string): WebViewInfo { + const webViews = mobilecli(['webview', 'list', '--device', simulatorId]).data as unknown[]; + expect(webViews.length, 'no webview reported by the playground app').toBeGreaterThan(0); + expectWebViewShape(webViews[0]); + return webViews[0]; +} + +function webViewUrl(simulatorId: string, webViewId: string): string { + return mobilecli(['webview', 'url', webViewId, '--device', simulatorId]).data as string; +} + +function webViewTitle(simulatorId: string, webViewId: string): string { + return mobilecli(['webview', 'title', webViewId, '--device', simulatorId]).data as string; +} + +function webViewContent(simulatorId: string, webViewId: string): string { + return mobilecli(['webview', 'content', webViewId, '--device', simulatorId]).data as string; +} + +function webViewEval(simulatorId: string, webViewId: string, expression: string): unknown { + return mobilecli(['webview', 'eval', webViewId, expression, '--device', simulatorId]).data; +} + +function webViewQuery(simulatorId: string, webViewId: string, selector: string): WebViewQueryResult[] { + return mobilecli(['webview', 'query', webViewId, selector, '--device', simulatorId]).data as WebViewQueryResult[]; +} + +function webViewGoto(simulatorId: string, webViewId: string, url: string): void { + mobilecli(['webview', 'goto', webViewId, url, '--device', simulatorId]); +} + +function webViewReload(simulatorId: string, webViewId: string): void { + mobilecli(['webview', 'reload', webViewId, '--device', simulatorId]); +} + +function webViewGoBack(simulatorId: string, webViewId: string): void { + mobilecli(['webview', 'back', webViewId, '--device', simulatorId]); +} + +function webViewGoForward(simulatorId: string, webViewId: string): void { + mobilecli(['webview', 'forward', webViewId, '--device', simulatorId]); +} + +function webViewWait(simulatorId: string, webViewId: string, state: string): void { + mobilecli(['webview', 'wait', webViewId, '--state', state, '--timeout', '15000', '--device', simulatorId]); +} + +// playwright has no sleep of its own and these waits are for device settling +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + function getAppContainerPath(simulatorId: string, packageName: string): string { return mobilecli(['apps', 'path', packageName, '--device', simulatorId]).data.path; } diff --git a/test/types.ts b/test/types.ts index 1a97598e..24d2ad59 100644 --- a/test/types.ts +++ b/test/types.ts @@ -15,6 +15,11 @@ export interface UIElement { children?: UIElement[]; } +export interface Point { + x: number; + y: number; +} + export interface UIDumpResponse { status: string; data: { @@ -49,3 +54,41 @@ export interface ForegroundAppResponse { version: string; }; } + +// one entry of `apps list`. every field is reported on both platforms today; the +// shape assertions in shapes.ts are what keep that true. +export interface InstalledApp { + packageName: string; + appName: string; + version: string; + versionCode: string; +} + +export interface AppsListResponse { + status: string; + data: InstalledApp[]; +} + +// `apps install` reads the metadata out of the artifact, so it has no appName +export interface InstallResult { + message: string; + app: { + packageName: string; + version: string; + versionCode: string; + }; +} + +export interface AppsInstallResponse { + status: string; + data: InstallResult; +} + +export interface UninstallResult { + packageName: string; +} + +export interface AppsUninstallResponse { + status: string; + data: UninstallResult; +} diff --git a/test/webview.ts b/test/webview.ts new file mode 100644 index 00000000..08e2ad57 --- /dev/null +++ b/test/webview.ts @@ -0,0 +1,109 @@ +import {expect} from '@playwright/test'; +import type {Point, UIDumpResponse, UIElement} from './types'; + +// the playground webview loads this page, and the page reads its own query string: +// `?source=webview` shows a login form, `?done=` shows a greeting instead. +// that gives every navigation command a visible, assertable effect. +export const WEBVIEW_SAMPLE_URL = 'https://mobilewright.dev/samples/webview/?source=webview'; +export const WEBVIEW_DONE_NAME = 'mobilecli'; +export const WEBVIEW_DONE_URL = `https://mobilewright.dev/samples/webview/?done=${WEBVIEW_DONE_NAME}`; +export const WEBVIEW_SAMPLE_TITLE = 'Sample Login'; +export const WEBVIEW_DONE_GREETING = `Hello there, ${WEBVIEW_DONE_NAME}. You are still in the webview!`; + +// the button that opens the webview screen from the playground main menu. android +// labels it through a text node, ios through the button's accessibility label. +export const WEB_VIEW_BUTTON_LABEL = 'Web View'; + +// both platforms return a nested tree, so flatten it before searching +export function flattenElements(elements: UIElement[]): UIElement[] { + return elements.flatMap(element => [element, ...flattenElements(element.children ?? [])]); +} + +export function centerOf(element: UIElement): Point { + return { + x: element.rect.x + Math.floor(element.rect.width / 2), + y: element.rect.y + Math.floor(element.rect.height / 2), + }; +} + +// android puts the caption in a child text node of the row, ios puts it on the +// button itself, so match on any element that carries the label and take the one +// with a tappable area +export function findWebViewButton(uiDump: UIDumpResponse): UIElement { + const matches = flattenElements(uiDump.data.elements).filter(element => + [element.text, element.label, element.name].includes(WEB_VIEW_BUTTON_LABEL)); + + const button = matches.find(element => element.rect.width > 0 && element.rect.height > 0); + if (!button) { + throw new Error(`no tappable "${WEB_VIEW_BUTTON_LABEL}" button in the playground menu`); + } + + return button; +} + +// `webview list` reports the same fields on both platforms, but ios leaves +// bundleId and processName empty, so those are asserted as present, not filled +export function expectWebViewShape(webView: unknown): asserts webView is WebViewInfo { + const view = webView as Record; + + for (const field of ['id', 'url', 'title', 'bundleId', 'processName']) { + expect(typeof view?.[field], `webview.${field}: ${JSON.stringify(webView)}`).toBe('string'); + } + expect((view.id as string).length, 'webview.id is empty').toBeGreaterThan(0); + expect(typeof view.isVisible).toBe('boolean'); + + for (const field of ['x', 'y', 'width', 'height']) { + expect(typeof (view.bounds as Record)?.[field], `webview.bounds.${field}`).toBe('number'); + } +} + +export interface WebViewInfo { + id: string; + url: string; + title: string; + bundleId: string; + processName: string; + bounds: { + x: number; + y: number; + width: number; + height: number; + }; + isVisible: boolean; +} + +export interface WebViewQueryResult { + tag: string; + id: string | null; + class: string | null; + href: string | null; + text: string | null; + value: string | null; +} + +// `goto`, `back` and `forward` return as soon as the navigation is requested, and +// `wait` can observe the load event of the page still on screen. polling the url +// is what actually proves the navigation landed, without a guessed sleep. +export async function expectWebViewUrlToBecome(readUrl: () => string, expected: string): Promise { + await expect.poll(readUrl, { + timeout: 15000, + message: `webview never navigated to ${expected}`, + }).toBe(expected); +} + +// no webview ever has this id, so every command that takes one fails on it +export const WEBVIEW_MISSING_ID = 'no-such-webview'; + +// each subcommand's arguments after the webview id, so one test can walk them all +export const WEBVIEW_COMMANDS_TAKING_AN_ID: ReadonlyArray = [ + ['url'], + ['title'], + ['content'], + ['reload'], + ['back'], + ['forward'], + ['wait'], + ['goto', WEBVIEW_SAMPLE_URL], + ['query', 'body'], + ['eval', 'document.title'], +]; From 73738cc8e53c10808730c22a585e61a97c8e7f11 Mon Sep 17 00:00:00 2001 From: gmegidish Date: Sat, 12 Sep 2026 14:13:04 +0200 Subject: [PATCH 2/2] test: poll the device instead of sleeping, and address review The android settings tests slept for a fixed 3 to 5 seconds and then asserted. On a slow emulator that guess was wrong often enough that unmodified main failed 1, 2 and 3 of them across three consecutive runs. test/poll.ts reads the device up to ten times, a second apart, and returns as soon as it agrees. Applied to every place that was sleeping on a guess: waiting for an app to reach the foreground, for the launcher to come back, for text to appear on screen, for the playground menu to draw, and for a webview to be reported. That made the suite both stable and faster: the emulator project went from about 1.1 minutes to 34 seconds, and it exposed a real race the fixed sleeps had been papering over, where a webview is listed before its page has a title. The webview setup now settles the page before any test reads it. Review fixes: - The Makefile ran the suites as separate recipe lines, so a failing suite skipped the daemon stop and lost its counters. They now run in one shell under an EXIT trap. - webViewCommandError parsed stdout as json unconditionally, so a timeout or a missing binary surfaced as "Unexpected end of JSON input" instead of the real failure. Empty stdout now rethrows the original. - The uninspectable-app group skips real devices and clears the settings task afterwards on android, and terminates settings on ios, so neither leaves a device in that state. - The two groups that walk a device through an ordered sequence are marked serial, so a failure retries the group rather than one step out of context. Not taken: replacing the mobilewright.dev sample page with a local fixture. The playground app hardcodes that url and loads it itself, so the dependency exists with or without these tests. --- Makefile | 14 +++--- test/android.spec.ts | 111 ++++++++++++++++++++++++++++++----------- test/poll.ts | 25 ++++++++++ test/simulator.spec.ts | 53 +++++++++++++++++--- 4 files changed, 162 insertions(+), 41 deletions(-) create mode 100644 test/poll.ts diff --git a/Makefile b/Makefile index 9f5d981a..16be689f 100644 --- a/Makefile +++ b/Makefile @@ -29,13 +29,15 @@ test-e2e: build-cover # inherits GOCOVERDIR, and a -cover binary only writes counters when it exits. ./mobilecli daemon stop go test ./... -v -race -covermode=atomic -args -test.gocoverdir=$(CURDIR)/test/coverage - (cd test && npm run test:server) - (cd test && npm run test:daemon) - (cd test && npm run test:ios-simulator) - (cd test && npm run test:android) + # one shell with a trap, so a failing suite still stops the daemon and flushes + # its counters into test/coverage before they are read back + set -e; \ + trap 'GOCOVERDIR=$(CURDIR)/test/coverage ./mobilecli daemon stop >/dev/null 2>&1 || true' EXIT; \ + (cd test && npm run test:server); \ + (cd test && npm run test:daemon); \ + (cd test && npm run test:ios-simulator); \ + (cd test && npm run test:android); \ (cd test && npm run test:emulator) - # flushes the daemon's counters into test/coverage before they are read back - GOCOVERDIR=$(CURDIR)/test/coverage ./mobilecli daemon stop go tool covdata textfmt -i=test/coverage -o coverage.out go tool cover -html=coverage.out -o coverage.html go tool cover -func=coverage.out diff --git a/test/android.spec.ts b/test/android.spec.ts index 2cc8039c..b66ae2ce 100644 --- a/test/android.spec.ts +++ b/test/android.spec.ts @@ -5,6 +5,7 @@ import * as fs from 'fs'; import * as os from 'os'; import type {InstalledApp, Point, UIElement, UIDumpResponse, ForegroundAppResponse} from './types'; import {coverageEnv} from './coverage'; +import {eventually} from './poll'; import { expectAppShape, expectFsListingShape, @@ -152,9 +153,8 @@ test.describe('Android Tests', () => { clearSettingsTask(device!.id); launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(3000); - expect(getForegroundApp(device!.id).data.packageName).toBe(SETTINGS_PACKAGE); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); }); test('should terminate Settings app and verify launcher is in foreground', async () => { @@ -166,41 +166,34 @@ test.describe('Android Tests', () => { // force-stop returns to whatever task sits below the app, so start from the // launcher — otherwise an app left running by an earlier test surfaces instead pressButton(device!.id, 'HOME'); - await sleep(2000); + await expectLauncherToBeInForeground(device!.id); launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(3000); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); clearSettingsTask(device!.id); - await sleep(3000); - - expect(getForegroundApp(device!.id).data.packageName).toMatch(LAUNCHER_PACKAGE_PATTERN); + await expectLauncherToBeInForeground(device!.id); }); test('should handle launching app twice (idempotency)', async () => { test.skip(!device, 'No Android device found'); launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(3000); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); // launching again should resume the app, not fail launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(3000); - - expect(getForegroundApp(device!.id).data.packageName).toBe(SETTINGS_PACKAGE); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); }); test('should press HOME button and return to launcher from Settings', async () => { test.skip(!device, 'No Android device found'); launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(3000); - expect(getForegroundApp(device!.id).data.packageName).toBe(SETTINGS_PACKAGE); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); pressButton(device!.id, 'HOME'); - await sleep(3000); - - expect(getForegroundApp(device!.id).data.packageName).toMatch(LAUNCHER_PACKAGE_PATTERN); + await expectLauncherToBeInForeground(device!.id); }); test('should tap on Network & internet in Settings and navigate to that screen', async ({deviceType}) => { @@ -212,13 +205,13 @@ test.describe('Android Tests', () => { // land on the settings root screen rather than wherever a previous test left it clearSettingsTask(device!.id); launchApp(device!.id, SETTINGS_PACKAGE); - await sleep(5000); + await expectForegroundAppToBecome(device!.id, SETTINGS_PACKAGE); + await expectTextOnScreen(device!.id, 'Network & internet'); const entry = findElementByText(dumpUI(device!.id), 'Network & internet'); tap(device!.id, centerOf(entry).x, centerOf(entry).y); - await sleep(3000); - verifyElementWithTextExists(dumpUI(device!.id), 'Airplane mode'); + await expectTextOnScreen(device!.id, 'Airplane mode'); }); test.describe('fs operations on /sdcard/Download', () => { @@ -276,7 +269,7 @@ test.describe('Android Tests', () => { } }); - test.describe('install and uninstall playground', () => { + test.describe.serial('install and uninstall playground', () => { // installs an app we own rather than anything already on the image, so the // uninstall half is safe to run for real. also leaves playground installed // for the app-container fs group below. @@ -317,7 +310,7 @@ test.describe('Android Tests', () => { }); }); - test.describe('webview', () => { + test.describe.serial('webview', () => { // the playground webview screen is the one embedded webview we control on // both platforms. a real handset would be left on an arbitrary screen. test.skip(({deviceType}) => deviceType === 'real', 'leaves a physical phone modified'); @@ -326,14 +319,22 @@ test.describe('Android Tests', () => { test.beforeAll(async () => { if (!device) return; - await openPlaygroundWebViewScreen(device.id); - await sleep(3000); - webViewId = firstWebView(device.id).id; + const deviceId = device.id; + + await openPlaygroundWebViewScreen(deviceId); + await expectWebViewToAppear(deviceId); + webViewId = firstWebView(deviceId).id; // the app restores whatever url the webview last showed, so start every // run from the sample page instead of inheriting the previous run's state - webViewGoto(device.id, webViewId, WEBVIEW_SAMPLE_URL); - webViewWait(device.id, webViewId, 'load'); + webViewGoto(deviceId, webViewId, WEBVIEW_SAMPLE_URL); + webViewWait(deviceId, webViewId, 'load'); + + // the webview is listed as soon as it exists, before its page has a title, + // so settle it here rather than leaving every test to race the load + await expectWebViewUrlToBecome(() => webViewUrl(deviceId, webViewId), WEBVIEW_SAMPLE_URL); + await eventually(() => webViewTitle(deviceId, webViewId), 'sample page never finished loading') + .toBe(WEBVIEW_SAMPLE_TITLE); }); test('should list the playground webview', () => { @@ -443,10 +444,17 @@ test.describe('Android Tests', () => { // reads the foreground app, so this launches a different app and would break the // shared state the playground tests set up once in their beforeAll test.describe('webview on an app that cannot be inspected', () => { + test.skip(({deviceType}) => deviceType === 'real', 'leaves a physical phone modified'); + test.beforeAll(async () => { if (!device) return; launchApp(device.id, SETTINGS_PACKAGE); - await sleep(3000); + await expectForegroundAppToBecome(device.id, SETTINGS_PACKAGE); + }); + + test.afterAll(() => { + if (!device) return; + clearSettingsTask(device.id); }); test('should fail to list webviews in an app that is not debuggable', () => { @@ -675,25 +683,53 @@ function allTextsIn(uiDump: UIDumpResponse): string[] { async function openPlaygroundWebViewScreen(deviceId: string): Promise { launchApp(deviceId, PLAYGROUND_PACKAGE); - await sleep(3000); + + // wait for the main menu to draw before looking for the button on it + await eventually(() => hasWebViewButton(deviceId), 'playground menu never appeared').toBe(true); + const button = findWebViewButton(dumpUI(deviceId)); tap(deviceId, centerOf(button).x, centerOf(button).y); } +function hasWebViewButton(deviceId: string): boolean { + try { + findWebViewButton(dumpUI(deviceId)); + return true; + } catch { + return false; + } +} + +// tapping the menu entry starts the webview activity, which loads its page before +// the agent can report it +async function expectWebViewToAppear(deviceId: string): Promise { + await eventually(() => listWebViews(deviceId).length, 'no webview appeared in the playground app') + .toBeGreaterThan(0); +} + // runs a webview command that is expected to fail and returns the error message function webViewCommandError(deviceId: string, args: string[]): string { try { mobilecliJson(['webview', ...args, '--device', deviceId]); } catch (error: unknown) { + // a timeout or a missing binary fails without printing an envelope, and + // parsing that as json would bury the real cause const stdout = (error as {stdout?: string}).stdout ?? ''; + if (stdout.trim() === '') { + throw error; + } return expectErrorEnvelope(JSON.parse(stdout)); } throw new Error(`webview ${args.join(' ')} unexpectedly succeeded`); } +function listWebViews(deviceId: string): unknown[] { + return mobilecliJson(['webview', 'list', '--device', deviceId]).data as unknown[]; +} + function firstWebView(deviceId: string): WebViewInfo { - const webViews = mobilecliJson(['webview', 'list', '--device', deviceId]).data as unknown[]; + const webViews = listWebViews(deviceId); expect(webViews.length, 'no webview reported by the playground app').toBeGreaterThan(0); expectWebViewShape(webViews[0]); return webViews[0]; @@ -739,6 +775,23 @@ function webViewWait(deviceId: string, webViewId: string, state: string): void { mobilecliJson(['webview', 'wait', webViewId, '--state', state, '--timeout', '15000', '--device', deviceId]); } +// the device settles on its own schedule, so these read it until it agrees +// rather than sleeping for a guessed duration +async function expectForegroundAppToBecome(deviceId: string, packageName: string): Promise { + await eventually(() => getForegroundApp(deviceId).data.packageName, + `${packageName} never came to the foreground`).toBe(packageName); +} + +async function expectLauncherToBeInForeground(deviceId: string): Promise { + await eventually(() => getForegroundApp(deviceId).data.packageName, + 'launcher never came to the foreground').toMatch(LAUNCHER_PACKAGE_PATTERN); +} + +async function expectTextOnScreen(deviceId: string, text: string): Promise { + await eventually(() => allTextsIn(dumpUI(deviceId)), + `"${text}" never appeared on screen`).toContain(text); +} + function getAppContainerPath(deviceId: string, packageName: string): string { return mobilecliJson(['apps', 'path', packageName, '--device', deviceId]).data.path; } diff --git a/test/poll.ts b/test/poll.ts new file mode 100644 index 00000000..3a4ff866 --- /dev/null +++ b/test/poll.ts @@ -0,0 +1,25 @@ +import {expect} from '@playwright/test'; + +// A device does not settle on demand: an app takes time to come to the +// foreground, a webview takes time to commit a navigation, and a screen takes +// time to draw. A fixed sleep either guesses too low and fails on a slow +// emulator, or guesses too high and pads every run. +// +// Ten attempts, one second apart, is long enough for the slowest of these and +// returns immediately when the device is ready. +const ATTEMPTS = 10; +const INTERVAL_MS = 1000; + +const POLL_OPTIONS = { + timeout: ATTEMPTS * INTERVAL_MS, + intervals: new Array(ATTEMPTS).fill(INTERVAL_MS), +}; + +// Reads the device repeatedly until the assertion chained onto it passes: +// +// await eventually(() => foregroundPackage(id), 'settings never came up').toBe(SETTINGS_PACKAGE); +// +// The returned value is a normal expect matcher, so any assertion works. +export function eventually(read: () => T, message: string) { + return expect.poll(read, {...POLL_OPTIONS, message}); +} diff --git a/test/simulator.spec.ts b/test/simulator.spec.ts index 3b4ecd94..ac3fc0f0 100644 --- a/test/simulator.spec.ts +++ b/test/simulator.spec.ts @@ -9,6 +9,7 @@ import { } from './simctl'; import {randomUUID} from "node:crypto"; import {coverageEnv} from './coverage'; +import {eventually} from './poll'; import { expectAppShape, expectDeviceShape, @@ -319,7 +320,7 @@ test.describe('iOS Simulator Tests', () => { verifyRawViewtreeDump(rawDump); }); - test.describe('install and uninstall playground', () => { + test.describe.serial('install and uninstall playground', () => { // installs an app we own rather than anything on the simulator image, so // the uninstall half is safe. also leaves playground installed for the // app-container fs group below. @@ -358,7 +359,7 @@ test.describe('iOS Simulator Tests', () => { }); }); - test.describe('webview', () => { + test.describe.serial('webview', () => { // the playground webview screen is the one embedded webview we control on // both platforms. a real handset would be left on an arbitrary screen. let webViewId: string; @@ -366,13 +367,19 @@ test.describe('iOS Simulator Tests', () => { test.beforeAll(async () => { if (!simulatorId) return; await openPlaygroundWebViewScreen(simulatorId); - await sleep(3000); + await expectWebViewToAppear(simulatorId); webViewId = firstWebView(simulatorId).id; // the app restores whatever url the webview last showed, so start every // run from the sample page instead of inheriting the previous run's state webViewGoto(simulatorId, webViewId, WEBVIEW_SAMPLE_URL); webViewWait(simulatorId, webViewId, 'load'); + + // the webview is listed as soon as it exists, before its page has a title, + // so settle it here rather than leaving every test to race the load + await expectWebViewUrlToBecome(() => webViewUrl(simulatorId, webViewId), WEBVIEW_SAMPLE_URL); + await eventually(() => webViewTitle(simulatorId, webViewId), 'sample page never finished loading') + .toBe(WEBVIEW_SAMPLE_TITLE); }); test('should list the playground webview', () => { @@ -485,7 +492,13 @@ test.describe('iOS Simulator Tests', () => { test.beforeAll(async () => { if (!simulatorId) return; launchApp(simulatorId, IOS_SETTINGS_BUNDLE_ID); - await sleep(3000); + await eventually(() => getForegroundApp(simulatorId).data.packageName, + 'settings never came to the foreground').toBe(IOS_SETTINGS_BUNDLE_ID); + }); + + test.afterAll(() => { + if (!simulatorId) return; + terminateApp(simulatorId, IOS_SETTINGS_BUNDLE_ID); }); test('should fail to list webviews in an app that is not debuggable', () => { @@ -887,25 +900,53 @@ function verifyRawViewtreeDump(response: any): void { async function openPlaygroundWebViewScreen(simulatorId: string): Promise { launchApp(simulatorId, PLAYGROUND_PACKAGE); - await sleep(3000); + + // wait for the main menu to draw before looking for the button on it + await eventually(() => hasWebViewButton(simulatorId), 'playground menu never appeared').toBe(true); + const button = findWebViewButton(dumpUI(simulatorId)); tap(simulatorId, centerOf(button).x, centerOf(button).y); } +function hasWebViewButton(simulatorId: string): boolean { + try { + findWebViewButton(dumpUI(simulatorId)); + return true; + } catch { + return false; + } +} + +// tapping the menu entry starts the webview activity, which loads its page before +// the agent can report it +async function expectWebViewToAppear(simulatorId: string): Promise { + await eventually(() => listWebViews(simulatorId).length, 'no webview appeared in the playground app') + .toBeGreaterThan(0); +} + // runs a webview command that is expected to fail and returns the error message function webViewCommandError(simulatorId: string, args: string[]): string { try { mobilecli(['webview', ...args, '--device', simulatorId]); } catch (error: unknown) { + // a timeout or a missing binary fails without printing an envelope, and + // parsing that as json would bury the real cause const stdout = (error as {stdout?: string}).stdout ?? ''; + if (stdout.trim() === '') { + throw error; + } return expectErrorEnvelope(JSON.parse(stdout)); } throw new Error(`webview ${args.join(' ')} unexpectedly succeeded`); } +function listWebViews(simulatorId: string): unknown[] { + return mobilecli(['webview', 'list', '--device', simulatorId]).data as unknown[]; +} + function firstWebView(simulatorId: string): WebViewInfo { - const webViews = mobilecli(['webview', 'list', '--device', simulatorId]).data as unknown[]; + const webViews = listWebViews(simulatorId); expect(webViews.length, 'no webview reported by the playground app').toBeGreaterThan(0); expectWebViewShape(webViews[0]); return webViews[0];