diff --git a/docs/src/api/class-locator.md b/docs/src/api/class-locator.md index ad37165471462..5d8a763480206 100644 --- a/docs/src/api/class-locator.md +++ b/docs/src/api/class-locator.md @@ -244,6 +244,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to When specified, limits the depth of the snapshot. +### option: Locator.ariaSnapshot.boxes +* since: v1.60 +- `boxes` <[boolean]> + +When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + ## async method: Locator.blur * since: v1.28 diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 47e5626b725d8..b20c766651a90 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -4253,6 +4253,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption: including e When specified, limits the depth of the snapshot. +### option: Page.ariaSnapshot.boxes +* since: v1.60 +- `boxes` <[boolean]> + +When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + ## async method: Page.tap * since: v1.8 * discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md). diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts index 08e23d585ef6a..17b438dc13616 100644 --- a/packages/injected/src/ariaSnapshot.ts +++ b/packages/injected/src/ariaSnapshot.ts @@ -41,6 +41,7 @@ export type AriaTreeOptions = { refPrefix?: string; doNotRenderActive?: boolean; depth?: number; + boxes?: boolean; }; type InternalOptions = { @@ -51,9 +52,11 @@ type InternalOptions = { renderCursorPointer?: boolean, renderActive?: boolean, renderStringsAsRegex?: boolean, + renderBoxes?: boolean, }; function toInternalOptions(options: AriaTreeOptions): InternalOptions { + const renderBoxes = options.boxes; if (options.mode === 'ai') { // For AI consumption. return { @@ -63,18 +66,19 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions { includeGenericRole: true, renderActive: !options.doNotRenderActive, renderCursorPointer: true, + renderBoxes, }; } if (options.mode === 'autoexpect') { // To auto-generate assertions on visible elements. - return { visibility: 'ariaAndVisible', refs: 'none' }; + return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes }; } if (options.mode === 'codegen') { // To generate aria assertion with regex heurisitcs. - return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true }; + return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes }; } // To match aria snapshot. - return { visibility: 'aria', refs: 'none' }; + return { visibility: 'aria', refs: 'none', renderBoxes }; } export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot { @@ -624,6 +628,13 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr if (renderCursorPointer && aria.hasPointerCursor(ariaNode)) key += ' [cursor=pointer]'; } + if (options.renderBoxes) { + const element = ariaNodeElement(ariaNode); + if (element) { + const r = element.getBoundingClientRect(); + key += ` [box=${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}]`; + } + } return key; }; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 30e475fcdea33..452fbe353a4cf 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2051,6 +2051,11 @@ export interface Page { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ @@ -13057,6 +13062,11 @@ export interface Locator { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ diff --git a/packages/playwright-core/src/client/locator.ts b/packages/playwright-core/src/client/locator.ts index 9de0f2b29528e..8dda1580c156e 100644 --- a/packages/playwright-core/src/client/locator.ts +++ b/packages/playwright-core/src/client/locator.ts @@ -329,8 +329,8 @@ export class Locator implements api.Locator { return ref ?? null; } - async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise { - const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth }); + async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean } = {}): Promise { + const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth, boxes: options.boxes }); return result.snapshot; } diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 2d519d8095c62..1dc9817d36204 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -866,8 +866,8 @@ export class Page extends ChannelOwner implements api.Page return result.pdf; } - async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, _track?: string } = {}): Promise { - const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth }); + async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean, _track?: string } = {}): Promise { + const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth, boxes: options.boxes }); return result.snapshot; } diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 3c24734e63a40..d9eaec067b8da 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -1660,6 +1660,7 @@ scheme.FrameAriaSnapshotParams = tObject({ track: tOptional(tString), selector: tOptional(tString), depth: tOptional(tInt), + boxes: tOptional(tBoolean), timeout: tFloat, }); scheme.FrameAriaSnapshotResult = tObject({ diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 90fafe6282f98..a298a0df92773 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1774,14 +1774,14 @@ export class Frame extends SdkObject { return { ref }; } - async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> { + async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number, boxes?: boolean } = {}): Promise<{ snapshot: string }> { if (options.selector && options.track) throw new Error('Cannot specify both selector and track options'); if (options.selector && options.mode !== 'ai') { // Non-ai locator snapshot is auto-waiting and does not include iframes. const snapshot = await this._retryWithProgressIfNotConnected(progress, options.selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => { - return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth })); + return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth, boxes: options.boxes })); }); return { snapshot }; } diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 257b657ba3d00..281ad0a6a1922 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -1061,7 +1061,7 @@ export class InitScript extends DisposableObject { } } -export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number } = {}): Promise<{ full: string[], incremental?: string[] }> { +export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number, boxes?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> { // Only await the topmost navigations, inner frames will be empty when racing. const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async (progress, continuePolling) => { try { @@ -1085,6 +1085,7 @@ export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Fra doNotRenderActive: options.doNotRenderActive, info: options.info, depth: options.depth, + boxes: options.boxes, })); if (snapshotOrRetry === true) return continuePolling; diff --git a/packages/playwright-core/src/tools/backend/response.ts b/packages/playwright-core/src/tools/backend/response.ts index 6b83adc25ae6b..eda678a8468e2 100644 --- a/packages/playwright-core/src/tools/backend/response.ts +++ b/packages/playwright-core/src/tools/backend/response.ts @@ -50,6 +50,7 @@ export class Response { private _includeSnapshotFileName: string | undefined; private _includeSnapshotRoot: playwright.Locator | undefined; private _includeSnapshotDepth: number | undefined; + private _includeSnapshotBoxes: boolean | undefined; private _isClose: boolean = false; readonly toolName: string; @@ -142,10 +143,11 @@ export class Response { this._includeSnapshot = this._context.config.snapshot?.mode ?? 'full'; } - setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number) { + setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number, boxes?: boolean) { this._includeSnapshot = 'explicit'; this._includeSnapshotFileName = includeSnapshotFileName; this._includeSnapshotDepth = depth; + this._includeSnapshotBoxes = boxes; this._includeSnapshotRoot = root; } @@ -242,7 +244,7 @@ export class Response { addSection('Ran Playwright code', this._code, 'js'); // Render tab titles upon changes or when more than one tab. - const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._clientWorkspace) : undefined; + const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._includeSnapshotBoxes, this._clientWorkspace) : undefined; const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot())); if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) { if (tabHeaders.length !== 1) diff --git a/packages/playwright-core/src/tools/backend/snapshot.ts b/packages/playwright-core/src/tools/backend/snapshot.ts index 30730b33b4f00..516b17da43921 100644 --- a/packages/playwright-core/src/tools/backend/snapshot.ts +++ b/packages/playwright-core/src/tools/backend/snapshot.ts @@ -42,6 +42,7 @@ const snapshot = defineTabTool({ target: z.string().optional().describe(elementTargetDescription), filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), depth: z.number().optional().describe('Limit the depth of the snapshot tree'), + boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot'), }), type: 'readOnly', }, @@ -50,7 +51,7 @@ const snapshot = defineTabTool({ let resolved: { locator: playwright.Locator | undefined, resolved: string } = { locator: undefined, resolved: '' }; if (params.target) resolved = await tab.targetLocator({ target: params.target }); - response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth); + response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth, params.boxes); }, }); diff --git a/packages/playwright-core/src/tools/backend/tab.ts b/packages/playwright-core/src/tools/backend/tab.ts index 0284509303422..e409fa344d07c 100644 --- a/packages/playwright-core/src/tools/backend/tab.ts +++ b/packages/playwright-core/src/tools/backend/tab.ts @@ -383,13 +383,13 @@ export class Tab extends EventEmitter { this._requests.length = 0; } - async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, relativeTo: string | undefined): Promise { + async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, boxes: boolean | undefined, relativeTo: string | undefined): Promise { await this._initializedPromise; let tabSnapshot: TabSnapshot | undefined; const modalStates = await this._raceAgainstModalStates(async () => { const ariaSnapshot = root - ? await root.ariaSnapshot({ mode: 'ai', depth }) - : await this.page.ariaSnapshot({ mode: 'ai', depth }); + ? await root.ariaSnapshot({ mode: 'ai', depth, boxes }) + : await this.page.ariaSnapshot({ mode: 'ai', depth, boxes }); tabSnapshot = { ariaSnapshot, modalStates: [], diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md index 66b5ad4acd108..9e70ca9322b40 100644 --- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md +++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md @@ -256,6 +256,9 @@ playwright-cli snapshot "#main" # limit snapshot depth for efficiency, take a partial snapshot afterwards playwright-cli snapshot --depth=4 playwright-cli snapshot e34 + +# include each element's bounding box as [box=x,y,width,height] +playwright-cli snapshot --boxes ``` ## Targeting elements diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index d703773e874b5..594778e874b0f 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -367,9 +367,10 @@ const snapshot = declareCommand({ options: z.object({ filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'), depth: numberArg.optional().describe('Limit snapshot depth, unlimited by default.'), + boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot.'), }), toolName: 'browser_snapshot', - toolParams: ({ filename, target, depth }) => ({ filename, target, depth }), + toolParams: ({ filename, target, depth, boxes }) => ({ filename, target, depth, boxes }), }); const generateLocator = declareCommand({ diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 30e475fcdea33..452fbe353a4cf 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2051,6 +2051,11 @@ export interface Page { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ @@ -13057,6 +13062,11 @@ export interface Locator { * @param options */ ariaSnapshot(options?: { + /** + * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`. + */ + boxes?: boolean; + /** * When specified, limits the depth of the snapshot. */ diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index d03bbcdd712ec..8125a83d924c8 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -2927,6 +2927,7 @@ export type FrameAriaSnapshotParams = { track?: string, selector?: string, depth?: number, + boxes?: boolean, timeout: number, }; export type FrameAriaSnapshotOptions = { @@ -2934,6 +2935,7 @@ export type FrameAriaSnapshotOptions = { track?: string, selector?: string, depth?: number, + boxes?: boolean, }; export type FrameAriaSnapshotResult = { snapshot: string, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 45970c3cc14d6..4ec62cb7f1c84 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -2324,6 +2324,7 @@ Frame: track: string? selector: string? depth: int? + boxes: boolean? timeout: float returns: snapshot: string diff --git a/tests/mcp/cli-core.spec.ts b/tests/mcp/cli-core.spec.ts index 89c472b887a09..24ef136ca5059 100644 --- a/tests/mcp/cli-core.spec.ts +++ b/tests/mcp/cli-core.spec.ts @@ -310,6 +310,20 @@ test('snapshot depth', async ({ cli, server }) => { - button "Cancel" [ref=e6]`); }); +test('snapshot --boxes', async ({ cli, server }) => { + server.setContent('/', ` + + + `, 'text/html'); + await cli('open', server.PREFIX); + + const { inlineSnapshot } = await cli('snapshot', '--boxes'); + expect(inlineSnapshot).toContain(`- button "click" [ref=e1] [box=100,50,80,40]`); + + const { inlineSnapshot: plain } = await cli('snapshot'); + expect(plain).not.toMatch(/\[box=/); +}); + test('eval --raw', async ({ cli, server }) => { await cli('open', server.HELLO_WORLD); const { output } = await cli('eval', '--raw', '() => document.title'); diff --git a/tests/mcp/core.spec.ts b/tests/mcp/core.spec.ts index 42490d8fd88c6..06423357b550d 100644 --- a/tests/mcp/core.spec.ts +++ b/tests/mcp/core.spec.ts @@ -276,6 +276,31 @@ test('snapshot depth', async ({ client, server }) => { }); }); +test('snapshot with boxes', async ({ client, server }) => { + server.setContent('/', ` + + + `, 'text/html'); + + await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.PREFIX }, + }); + + expect(await client.callTool({ + name: 'browser_snapshot', + arguments: { boxes: true }, + })).toHaveResponse({ + inlineSnapshot: expect.stringContaining(`- button "click" [ref=e1] [box=100,50,80,40]`), + }); + + expect(await client.callTool({ + name: 'browser_snapshot', + })).toHaveResponse({ + inlineSnapshot: expect.not.stringMatching(/\[box=/), + }); +}); + test('snapshot by ref', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-cli/issues/347' } }, async ({ client, server }) => { server.setContent('/', `
    diff --git a/tests/page/page-aria-snapshot.spec.ts b/tests/page/page-aria-snapshot.spec.ts index 8b6bcf19e1e05..da9708d56c4dd 100644 --- a/tests/page/page-aria-snapshot.spec.ts +++ b/tests/page/page-aria-snapshot.spec.ts @@ -748,3 +748,29 @@ it('should snapshot a locator inside an iframe', async ({ page }) => { - listitem: Item 2 `); }); + +it('should snapshot with box from page', async ({ page }) => { + await page.setContent(` + + `); + + const snapshot = await page.ariaSnapshot({ boxes: true }); + expect(snapshot).toBe(`- button "click" [box=100,50,80,40]`); +}); + +it('should snapshot with box from locator', async ({ page }) => { + await page.setContent(` +
    + +
    + `); + + const snapshot = await page.locator('div').ariaSnapshot({ boxes: true }); + expect(snapshot).toBe(`- button "ok" [box=15,25,60,30]`); +}); + +it('should not include box when option is omitted', async ({ page }) => { + await page.setContent(``); + const snapshot = await page.ariaSnapshot(); + expect(snapshot).not.toMatch(/\[box=/); +});