feat(aria-snapshot): add boxes option to include bounding boxes

docs/src/api/class-locator.md

@@ -244,6 +244,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption. Defaults to
 
 When specified, limits the depth of the snapshot.
 
+### option: Locator.ariaSnapshot.boxes
+* since: v1.60
+- `boxes` <[boolean]>
+
+When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+
 ## async method: Locator.blur
 * since: v1.28
 

docs/src/api/class-page.md

@@ -4253,6 +4253,12 @@ When set to `"ai"`, returns a snapshot optimized for AI consumption: including e
 
 When specified, limits the depth of the snapshot.
 
+### option: Page.ariaSnapshot.boxes
+* since: v1.60
+- `boxes` <[boolean]>
+
+When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+
 ## async method: Page.tap
 * since: v1.8
 * discouraged: Use locator-based [`method: Locator.tap`] instead. Read more about [locators](../locators.md).

packages/injected/src/ariaSnapshot.ts

@@ -41,6 +41,7 @@ export type AriaTreeOptions = {
   refPrefix?: string;
   doNotRenderActive?: boolean;
   depth?: number;
+  boxes?: boolean;
 };
 
 type InternalOptions = {
@@ -51,9 +52,11 @@ type InternalOptions = {
   renderCursorPointer?: boolean,
   renderActive?: boolean,
   renderStringsAsRegex?: boolean,
+  renderBoxes?: boolean,
 };
 
 function toInternalOptions(options: AriaTreeOptions): InternalOptions {
+  const renderBoxes = options.boxes;
   if (options.mode === 'ai') {
     // For AI consumption.
     return {
@@ -63,18 +66,19 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
       includeGenericRole: true,
       renderActive: !options.doNotRenderActive,
       renderCursorPointer: true,
+      renderBoxes,
     };
   }
   if (options.mode === 'autoexpect') {
     // To auto-generate assertions on visible elements.
-    return { visibility: 'ariaAndVisible', refs: 'none' };
+    return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes };
   }
   if (options.mode === 'codegen') {
     // To generate aria assertion with regex heurisitcs.
-    return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true };
+    return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes };
   }
   // To match aria snapshot.
-  return { visibility: 'aria', refs: 'none' };
+  return { visibility: 'aria', refs: 'none', renderBoxes };
 }
 
 export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {
@@ -624,6 +628,13 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTr
       if (renderCursorPointer && aria.hasPointerCursor(ariaNode))
         key += ' [cursor=pointer]';
     }
+    if (options.renderBoxes) {
+      const element = ariaNodeElement(ariaNode);
+      if (element) {
+        const r = element.getBoundingClientRect();
+        key += ` [box=${Math.round(r.x)},${Math.round(r.y)},${Math.round(r.width)},${Math.round(r.height)}]`;
+      }
+    }
     return key;
   };
 

packages/playwright-client/types/types.d.ts

@@ -2051,6 +2051,11 @@ export interface Page {
    * @param options
    */
   ariaSnapshot(options?: {
+    /**
+     * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+     */
+    boxes?: boolean;
+
     /**
      * When specified, limits the depth of the snapshot.
      */
@@ -13057,6 +13062,11 @@ export interface Locator {
    * @param options
    */
   ariaSnapshot(options?: {
+    /**
+     * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+     */
+    boxes?: boolean;
+
     /**
      * When specified, limits the depth of the snapshot.
      */

packages/playwright-core/src/client/locator.ts

@@ -329,8 +329,8 @@ export class Locator implements api.Locator {
     return ref ?? null;
   }
 
-  async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise<string> {
-    const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth });
+  async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean } = {}): Promise<string> {
+    const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth, boxes: options.boxes });
     return result.snapshot;
   }
 

packages/playwright-core/src/client/page.ts

@@ -866,8 +866,8 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
     return result.pdf;
   }
 
-  async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, _track?: string } = {}): Promise<string> {
-    const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth });
+  async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number, boxes?: boolean, _track?: string } = {}): Promise<string> {
+    const result = await this.mainFrame()._channel.ariaSnapshot({ timeout: this._timeoutSettings.timeout(options), track: options._track, mode: options.mode, depth: options.depth, boxes: options.boxes });
     return result.snapshot;
   }
 

packages/playwright-core/src/protocol/validator.ts

@@ -1660,6 +1660,7 @@ scheme.FrameAriaSnapshotParams = tObject({
   track: tOptional(tString),
   selector: tOptional(tString),
   depth: tOptional(tInt),
+  boxes: tOptional(tBoolean),
   timeout: tFloat,
 });
 scheme.FrameAriaSnapshotResult = tObject({

packages/playwright-core/src/server/frames.ts

@@ -1774,14 +1774,14 @@ export class Frame extends SdkObject<FrameEventMap> {
     return { ref };
   }
 
-  async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> {
+  async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number, boxes?: boolean } = {}): Promise<{ snapshot: string }> {
     if (options.selector && options.track)
       throw new Error('Cannot specify both selector and track options');
 
     if (options.selector && options.mode !== 'ai') {
       // Non-ai locator snapshot is auto-waiting and does not include iframes.
       const snapshot = await this._retryWithProgressIfNotConnected(progress, options.selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => {
-        return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth }));
+        return await progress.race(handle.evaluateInUtility(([injected, element, opts]) => injected.ariaSnapshot(element, opts), { mode: 'default' as const, depth: options.depth, boxes: options.boxes }));
       });
       return { snapshot };
     }

packages/playwright-core/src/server/page.ts

@@ -1061,7 +1061,7 @@ export class InitScript extends DisposableObject {
   }
 }
 
-export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number } = {}): Promise<{ full: string[], incremental?: string[] }> {
+export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Frame, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, info?: SelectorInfo, depth?: number, boxes?: boolean } = {}): Promise<{ full: string[], incremental?: string[] }> {
   // Only await the topmost navigations, inner frames will be empty when racing.
   const snapshot = await frame.retryWithProgressAndTimeouts(progress, [1000, 2000, 4000, 8000], async (progress, continuePolling) => {
     try {
@@ -1085,6 +1085,7 @@ export async function ariaSnapshotForFrame(progress: Progress, frame: frames.Fra
         doNotRenderActive: options.doNotRenderActive,
         info: options.info,
         depth: options.depth,
+        boxes: options.boxes,
       }));
       if (snapshotOrRetry === true)
         return continuePolling;

packages/playwright-core/src/tools/backend/response.ts

@@ -50,6 +50,7 @@ export class Response {
   private _includeSnapshotFileName: string | undefined;
   private _includeSnapshotRoot: playwright.Locator | undefined;
   private _includeSnapshotDepth: number | undefined;
+  private _includeSnapshotBoxes: boolean | undefined;
   private _isClose: boolean = false;
 
   readonly toolName: string;
@@ -142,10 +143,11 @@ export class Response {
     this._includeSnapshot = this._context.config.snapshot?.mode ?? 'full';
   }
 
-  setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number) {
+  setIncludeFullSnapshot(includeSnapshotFileName?: string, root?: playwright.Locator, depth?: number, boxes?: boolean) {
     this._includeSnapshot = 'explicit';
     this._includeSnapshotFileName = includeSnapshotFileName;
     this._includeSnapshotDepth = depth;
+    this._includeSnapshotBoxes = boxes;
     this._includeSnapshotRoot = root;
   }
 
@@ -242,7 +244,7 @@ export class Response {
       addSection('Ran Playwright code', this._code, 'js');
 
     // Render tab titles upon changes or when more than one tab.
-    const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._clientWorkspace) : undefined;
+    const tabSnapshot = this._context.currentTab() ? await this._context.currentTabOrDie().captureSnapshot(this._includeSnapshotRoot, this._includeSnapshotDepth, this._includeSnapshotBoxes, this._clientWorkspace) : undefined;
     const tabHeaders = await Promise.all(this._context.tabs().map(tab => tab.headerSnapshot()));
     if (this._includeSnapshot !== 'none' || tabHeaders.some(header => header.changed)) {
       if (tabHeaders.length !== 1)

packages/playwright-core/src/tools/backend/snapshot.ts

@@ -42,6 +42,7 @@ const snapshot = defineTabTool({
       target: z.string().optional().describe(elementTargetDescription),
       filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),
       depth: z.number().optional().describe('Limit the depth of the snapshot tree'),
+      boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot'),
     }),
     type: 'readOnly',
   },
@@ -50,7 +51,7 @@ const snapshot = defineTabTool({
     let resolved: { locator: playwright.Locator | undefined, resolved: string } = { locator: undefined, resolved: '' };
     if (params.target)
       resolved = await tab.targetLocator({ target: params.target });
-    response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth);
+    response.setIncludeFullSnapshot(params.filename, resolved.locator, params.depth, params.boxes);
   },
 });
 

packages/playwright-core/src/tools/backend/tab.ts

@@ -383,13 +383,13 @@ export class Tab extends EventEmitter<TabEventsInterface> {
     this._requests.length = 0;
   }
 
-  async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {
+  async captureSnapshot(root: playwright.Locator | undefined, depth: number | undefined, boxes: boolean | undefined, relativeTo: string | undefined): Promise<TabSnapshot> {
     await this._initializedPromise;
     let tabSnapshot: TabSnapshot | undefined;
     const modalStates = await this._raceAgainstModalStates(async () => {
       const ariaSnapshot = root
-        ? await root.ariaSnapshot({ mode: 'ai', depth })
-        : await this.page.ariaSnapshot({ mode: 'ai', depth });
+        ? await root.ariaSnapshot({ mode: 'ai', depth, boxes })
+        : await this.page.ariaSnapshot({ mode: 'ai', depth, boxes });
       tabSnapshot = {
         ariaSnapshot,
         modalStates: [],

packages/playwright-core/src/tools/cli-client/skill/SKILL.md

@@ -256,6 +256,9 @@ playwright-cli snapshot "#main"
 # limit snapshot depth for efficiency, take a partial snapshot afterwards
 playwright-cli snapshot --depth=4
 playwright-cli snapshot e34
+
+# include each element's bounding box as [box=x,y,width,height]
+playwright-cli snapshot --boxes
 ```
 
 ## Targeting elements

packages/playwright-core/src/tools/cli-daemon/commands.ts

@@ -367,9 +367,10 @@ const snapshot = declareCommand({
   options: z.object({
     filename: z.string().optional().describe('Save snapshot to markdown file instead of returning it in the response.'),
     depth: numberArg.optional().describe('Limit snapshot depth, unlimited by default.'),
+    boxes: z.boolean().optional().describe('Include each element\'s bounding box as [box=x,y,width,height] in the snapshot.'),
   }),
   toolName: 'browser_snapshot',
-  toolParams: ({ filename, target, depth }) => ({ filename, target, depth }),
+  toolParams: ({ filename, target, depth, boxes }) => ({ filename, target, depth, boxes }),
 });
 
 const generateLocator = declareCommand({

packages/playwright-core/types/types.d.ts

@@ -2051,6 +2051,11 @@ export interface Page {
    * @param options
    */
   ariaSnapshot(options?: {
+    /**
+     * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+     */
+    boxes?: boolean;
+
     /**
      * When specified, limits the depth of the snapshot.
      */
@@ -13057,6 +13062,11 @@ export interface Locator {
    * @param options
    */
   ariaSnapshot(options?: {
+    /**
+     * When `true`, appends each element's bounding box as `[box=x,y,width,height]` to the snapshot. Defaults to `false`.
+     */
+    boxes?: boolean;
+
     /**
      * When specified, limits the depth of the snapshot.
      */

packages/protocol/src/channels.d.ts

@@ -2927,13 +2927,15 @@ export type FrameAriaSnapshotParams = {
   track?: string,
   selector?: string,
   depth?: number,
+  boxes?: boolean,
   timeout: number,
 };
 export type FrameAriaSnapshotOptions = {
   mode?: 'ai' | 'default',
   track?: string,
   selector?: string,
   depth?: number,
+  boxes?: boolean,
 };
 export type FrameAriaSnapshotResult = {
   snapshot: string,

packages/protocol/src/protocol.yml

@@ -2324,6 +2324,7 @@ Frame:
         track: string?
         selector: string?
         depth: int?
+        boxes: boolean?
         timeout: float
       returns:
         snapshot: string

tests/mcp/cli-core.spec.ts

@@ -310,6 +310,20 @@ test('snapshot depth', async ({ cli, server }) => {
     - button "Cancel" [ref=e6]`);
 });
 
+test('snapshot --boxes', async ({ cli, server }) => {
+  server.setContent('/', `
+    <style>body { margin: 0; }</style>
+    <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+  `, 'text/html');
+  await cli('open', server.PREFIX);
+
+  const { inlineSnapshot } = await cli('snapshot', '--boxes');
+  expect(inlineSnapshot).toContain(`- button "click" [ref=e1] [box=100,50,80,40]`);
+
+  const { inlineSnapshot: plain } = await cli('snapshot');
+  expect(plain).not.toMatch(/\[box=/);
+});
+
 test('eval --raw', async ({ cli, server }) => {
   await cli('open', server.HELLO_WORLD);
   const { output } = await cli('eval', '--raw', '() => document.title');

tests/mcp/core.spec.ts

@@ -276,6 +276,31 @@ test('snapshot depth', async ({ client, server }) => {
   });
 });
 
+test('snapshot with boxes', async ({ client, server }) => {
+  server.setContent('/', `
+    <style>body { margin: 0; }</style>
+    <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+  `, 'text/html');
+
+  await client.callTool({
+    name: 'browser_navigate',
+    arguments: { url: server.PREFIX },
+  });
+
+  expect(await client.callTool({
+    name: 'browser_snapshot',
+    arguments: { boxes: true },
+  })).toHaveResponse({
+    inlineSnapshot: expect.stringContaining(`- button "click" [ref=e1] [box=100,50,80,40]`),
+  });
+
+  expect(await client.callTool({
+    name: 'browser_snapshot',
+  })).toHaveResponse({
+    inlineSnapshot: expect.not.stringMatching(/\[box=/),
+  });
+});
+
 test('snapshot by ref', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-cli/issues/347' } }, async ({ client, server }) => {
   server.setContent('/', `
     <ul>

tests/page/page-aria-snapshot.spec.ts

@@ -748,3 +748,29 @@ it('should snapshot a locator inside an iframe', async ({ page }) => {
       - listitem: Item 2
   `);
 });
+
+it('should snapshot with box from page', async ({ page }) => {
+  await page.setContent(`
+    <button style="position:absolute;left:100px;top:50px;width:80px;height:40px;margin:0;padding:0;border:0;">click</button>
+  `);
+
+  const snapshot = await page.ariaSnapshot({ boxes: true });
+  expect(snapshot).toBe(`- button "click" [box=100,50,80,40]`);
+});
+
+it('should snapshot with box from locator', async ({ page }) => {
+  await page.setContent(`
+    <div style="position:absolute;left:10px;top:20px;width:200px;height:100px;">
+      <button style="position:absolute;left:5px;top:5px;width:60px;height:30px;margin:0;padding:0;border:0;">ok</button>
+    </div>
+  `);
+
+  const snapshot = await page.locator('div').ariaSnapshot({ boxes: true });
+  expect(snapshot).toBe(`- button "ok" [box=15,25,60,30]`);
+});
+
+it('should not include box when option is omitted', async ({ page }) => {
+  await page.setContent(`<button>click</button>`);
+  const snapshot = await page.ariaSnapshot();
+  expect(snapshot).not.toMatch(/\[box=/);
+});