From 00fe2fe467cd8f516d16c42a14ff431f32ce4d48 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Mon, 19 Jan 2026 11:26:58 +0000 Subject: [PATCH] chore: page.agent timeouts sorted For all agent calls, timeout limits the outer `agent.xyz()` call, not individual actions inside. When running the loop for generation, timeout is disabled entirely. This means, generation is only limited by the test timeout. When generating, all individual actions inside are run as a one-shot, with all auto-waiting disabled. This effectively makes them instant, so timeout has no effect on actions. Instead, timeout effectively limits the model processing time. For `agent.perform()` and `agent.extract()`, timeout is taken from the first defined: - `timeout` option for the call; - `page.setDefaultTimeout()` or `browserContext.setDefaultTimeout()`; - `config.actionTimeout`; - 0 (no timeout) if nothing above is defined. For `agent.expect()`, timeout is taken from the first defined: - `timeout` option for the call; - `page.agent({ expect: { timeout } })` option for the agent; - `config.expect.timeout`; - 5000 if nothing above is defined. --- docs/src/api/class-page.md | 5 ++++ docs/src/api/class-pageagent.md | 20 +++++++++++++ docs/src/api/params.md | 7 ----- packages/injected/src/injectedScript.ts | 3 +- packages/playwright-client/types/types.d.ts | 18 +++++++++-- packages/playwright-core/src/client/page.ts | 4 ++- .../playwright-core/src/client/pageAgent.ts | 22 +++++++------- .../src/server/agent/actionRunner.ts | 27 +++++++---------- .../src/server/agent/pageAgent.ts | 6 ++-- packages/playwright-core/src/server/frames.ts | 18 +++++------ .../playwright-core/src/server/progress.ts | 6 +++- packages/playwright-core/types/types.d.ts | 18 +++++++++-- packages/playwright/src/index.ts | 3 ++ packages/protocol/src/progress.d.ts | 1 + ...timeout-during-run-from-agent-options.json | 30 +++++++++++++++++++ ...rary-agent-expect-expectTitle-success.json | 6 ++-- ...-expect-expectTitle-wrong-title-error.json | 6 ++-- ...ibrary-agent-expect-expectURL-success.json | 18 +++++------ ...ent-expect-expectURL-with-regex-error.json | 6 ++-- ...ary-agent-expect-expectURL-with-regex.json | 2 +- ...gent-expect-expectURL-wrong-URL-error.json | 12 ++++---- ...rform-run-timeout-inherited-from-page.json | 30 +++++++++++++++++++ tests/library/agent-expect.spec.ts | 26 ++++++++++++++-- tests/library/agent-helpers.ts | 3 +- tests/library/agent-perform.spec.ts | 22 ++++++++++++++ 25 files changed, 232 insertions(+), 87 deletions(-) create mode 100644 tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json create mode 100644 tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 5c2b7521971d8..55399a4d678f8 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache. - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. +### option: Page.agent.expect +* since: v1.58 +- `expect` <[Object]> + - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms. + ### option: Page.agent.limits * since: v1.58 - `limits` <[Object]> diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md index 26242e8c4428d..0f04e2b04d851 100644 --- a/docs/src/api/class-pageagent.md +++ b/docs/src/api/class-pageagent.md @@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported'); Expectation to assert. +### option: PageAgent.expect.timeout +* since: v1.58 +- `timeout` <[float]> + +Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout. + ### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -68,6 +74,13 @@ Task to perform using agentic loop. * since: v1.58 - `schema` <[z.ZodSchema]> +### option: PageAgent.extract.timeout +* since: v1.58 +- `timeout` <[float]> + +Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 @@ -94,6 +107,13 @@ await agent.perform('Click submit button'); Task to perform using agentic loop. +### option: PageAgent.perform.timeout +* since: v1.58 +- `timeout` <[float]> + +Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or +[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout. + ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 9b7bf8f3d4af4..69257b93db23d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property. -## page-agent-timeout -* since: v1.58 -- `timeout` <[int]> - -Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. - ## page-agent-call-options-v1.58 - %%-page-agent-cache-key-%% - %%-page-agent-max-tokens-%% - %%-page-agent-max-actions-%% - %%-page-agent-max-action-retries-%% -- %%-page-agent-timeout-%% ## fetch-param-url - `url` <[string]> diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts index ce18bf46aa252..2be0082e4d79a 100644 --- a/packages/injected/src/injectedScript.ts +++ b/packages/injected/src/injectedScript.ts @@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript'; export type FrameExpectParams = Omit & { expectedValue?: any; timeoutForLogs?: number; - explicitTimeout?: number; - noPreChecks?: boolean; + noAutoWaiting?: boolean; }; export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable'; diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index e7c027f2a8941..adc7654de9599 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -865,7 +865,9 @@ export class Page extends ChannelOwner implements api.Page systemPrompt: options.systemPrompt, }; const { agent } = await this._channel.agent(params); - return PageAgent.from(agent); + const pageAgent = PageAgent.from(agent); + pageAgent._expectTimeout = options?.expect?.timeout; + return pageAgent; } async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> { diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts index 2837f5bbb11f3..ec6a16cc9aeab 100644 --- a/packages/playwright-core/src/client/pageAgent.ts +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -22,14 +22,9 @@ import { Page } from './page'; import type * as api from '../../types/types'; import type * as channels from '@protocol/channels'; -type PageAgentOptions = { - maxTokens?: number; - maxTurns?: number; - cacheKey?: string; -}; - export class PageAgent extends ChannelOwner implements api.PageAgent { private _page: Page; + _expectTimeout?: number; static from(channel: channels.PageAgentChannel): PageAgent { return (channel as any)._object; @@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner implement this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params)); } - async expect(expectation: string, options: PageAgentOptions = {}) { - await this._channel.expect({ expectation, ...options }); + async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) { + const timeout = options.timeout ?? this._expectTimeout ?? 5000; + await this._channel.expect({ expectation, ...options, timeout }); } - async perform(task: string, options: PageAgentOptions = {}) { - const { usage } = await this._channel.perform({ task, ...options }); + async perform(task: string, options: channels.PageAgentPerformOptions = {}) { + const timeout = this._page._timeoutSettings.timeout(options); + const { usage } = await this._channel.perform({ task, ...options, timeout }); return { usage }; } - async extract(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { - const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options }); + async extract(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> { + const timeout = this._page._timeoutSettings.timeout(options); + const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout }); return { result, usage }; } diff --git a/packages/playwright-core/src/server/agent/actionRunner.ts b/packages/playwright-core/src/server/agent/actionRunner.ts index 9f25d3fec282e..9773c29b55a03 100644 --- a/packages/playwright-core/src/server/agent/actionRunner.ts +++ b/packages/playwright-core/src/server/agent/actionRunner.ts @@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page } async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) { - // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress. - const timeout = expectTimeout(mode); const result = await frame.expect(progress, selector, { ...options, - timeoutForLogs: timeout, - explicitTimeout: timeout, - // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway. - noPreChecks: mode === 'generate', + // When generating, we want the expect to pass or fail immediately and give feedback to the model. + noAutoWaiting: mode === 'generate', + timeoutForLogs: mode === 'generate' ? undefined : progress.timeout, }); if (!result.matches === !options.isNot) { const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received; @@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r expectation, locator: selector ? asLocatorDescription('javascript', selector) : undefined, timedOut: result.timedOut, - timeout, + timeout: mode === 'generate' ? undefined : progress.timeout, printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`, printedReceived: result.errorMessage ? '' : `Received: ${received}`, errorMessage: result.errorMessage, @@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.value', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Value', params }; } else if (action.type === 'checkbox' || action.type === 'radio') { @@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.checked', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Checked', params }; } else { @@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, selector: action.selector, expression: 'to.be.visible', isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Visible', params }; } @@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.match.snapshot', expectedText: [], isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params }; } @@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.url', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect URL', params }; } @@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action, expression: 'to.have.title', expectedText, isNot: !!action.isNot, - timeout: expectTimeout(mode), + timeout, }; return { type: 'Frame', method: 'expect', title: 'Expect Title', params }; } @@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions }; return callMetadata; } - -function expectTimeout(mode: 'generate' | 'run') { - return mode === 'generate' ? 0 : 5000; -} diff --git a/packages/playwright-core/src/server/agent/pageAgent.ts b/packages/playwright-core/src/server/agent/pageAgent.ts index 120f611390f5c..7b22731aa58e0 100644 --- a/packages/playwright-core/src/server/agent/pageAgent.ts +++ b/packages/playwright-core/src/server/agent/pageAgent.ts @@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use ### Task ${userTask} `; - + progress.disableTimeout(); await runLoop(progress, context, performTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe ### Expectation ${expectation} `; - + progress.disableTimeout(); await runLoop(progress, context, expectTools, task, undefined, callParams); await updateCache(context, cacheKey); } @@ -101,7 +101,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To const apiCacheTextBefore = context.agentParams.apiCacheFile ? await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}'; - const apiCacheBefore = JSON.parse(apiCacheTextBefore); + const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}'); const loop = new Loop({ api: context.agentParams.api as any, diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index 5431ea6a40e26..a09b41d050cff 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -1389,14 +1389,10 @@ export class Frame extends SdkObject { progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } }; }; try { - // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort. - const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined; - timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ }); - // Step 1: perform locator handlers checkpoint with a specified timeout. if (selector) progress.log(`waiting for ${this._asLocator(selector)}`); - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); // Step 2: perform one-shot expect check without a timeout. @@ -1404,19 +1400,19 @@ export class Frame extends SdkObject { // that should succeed when the locator is already visible. try { const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true); - if (resultOneShot.matches !== options.isNot) + if (options.noAutoWaiting || resultOneShot.matches !== options.isNot) return resultOneShot; } catch (e) { - if (this.isNonRetriableError(e)) + if (options.noAutoWaiting || this.isNonRetriableError(e)) throw e; // Ignore any other errors from one-shot, we'll handle them during retries. } // Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time. const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => { - if (!options.noPreChecks) + if (!options.noAutoWaiting) await this._page.performActionPreChecks(progress); - const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise); + const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false); if (matches === options.isNot) { // Keep waiting in these cases: // expect(locator).conditionThatDoesNotMatch @@ -1446,9 +1442,9 @@ export class Frame extends SdkObject { } } - private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise) { + private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) { // The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted. - const race = (p: Promise) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p)); + const race = (p: Promise) => noAbort ? p : progress.race(p); const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined; const { frame, info } = selectorInFrame || { frame: this, info: undefined }; diff --git a/packages/playwright-core/src/server/progress.ts b/packages/playwright-core/src/server/progress.ts index b6cb8201fb27d..07b580bff6376 100644 --- a/packages/playwright-core/src/server/progress.ts +++ b/packages/playwright-core/src/server/progress.ts @@ -61,10 +61,14 @@ export class ProgressController { const deadline = timeout ? monotonicTime() + timeout : 0; assert(this._state === 'before'); this._state = 'running'; + let timer: NodeJS.Timeout | undefined; const progress: Progress = { timeout: timeout ?? 0, deadline, + disableTimeout: () => { + clearTimeout(timer); + }, log: message => { if (this._state === 'running') this.metadata.log.push(message); @@ -87,10 +91,10 @@ export class ProgressController { signal: this._controller.signal, }; - let timer: NodeJS.Timeout | undefined; if (deadline) { const timeoutError = new TimeoutError(`Timeout ${timeout}ms exceeded.`); timer = setTimeout(() => { + // TODO: migrate this to "progress.disableTimeout()". if (this.metadata.pauseStartTime && !this.metadata.pauseEndTime) return; if (this._state === 'running') { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 6e2a58a63d4d3..50dd490708e13 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -2112,6 +2112,13 @@ export interface Page { cacheOutFile?: string; }; + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + /** * Limits to use for the agentic loop. */ @@ -5440,7 +5447,10 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. */ timeout?: number; }): Promise; @@ -5482,7 +5492,11 @@ export interface PageAgent { maxTokens?: number; /** - * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout. + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. */ timeout?: number; }): Promise<{ diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 82b978fb8caf2..874ba50ef2d00 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -477,6 +477,9 @@ const playwrightFixtures: Fixtures = ({ limits: agentOptions?.limits, secrets: agentOptions?.secrets, systemPrompt: agentOptions?.systemPrompt, + expect: { + timeout: testInfoImpl._projectInternal.expect?.timeout, + }, }); await use(agent); diff --git a/packages/protocol/src/progress.d.ts b/packages/protocol/src/progress.d.ts index 8e389e0c31ea4..f7abd096ac5b9 100644 --- a/packages/protocol/src/progress.d.ts +++ b/packages/protocol/src/progress.d.ts @@ -37,6 +37,7 @@ import type { CallMetadata } from './callMetadata'; export interface Progress { timeout: number; deadline: number; + disableTimeout(): void; log(message: string): void; race(promise: Promise | Promise[]): Promise; wait(timeout: number): Promise; // timeout = 0 here means "wait 0 ms", not forever. diff --git a/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json new file mode 100644 index 0000000000000..1109e7715d4e2 --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-expect-expect-timeout-during-run-from-agent-options.json @@ -0,0 +1,30 @@ +{ + "44bfbaa6fe0cb038adf37547ecb000d4f39e243f": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll verify that the submit button is visible on the page." + }, + { + "type": "tool_call", + "name": "browser_expect_visible", + "arguments": { + "role": "button", + "accessibleName": "Submit", + "_is_done": true + }, + "id": "toolu_012RgUS8buCjrQRgt4w2sYFf" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 2203, + "output": 107 + } + } +} \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json index bb23b43b9d30c..0f0ff37ab65b6 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-success.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to assert this condition." + "text": "I need to verify that the page title is \"My Page Title\". I'll use the browser_expect_title tool to check this." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "My Page Title", "_is_done": true }, - "id": "toolu_01VvWdf4VVFNQuDaEtTsAu3K" + "id": "toolu_01R1QyAwfxk2KSz3tsAx9RCA" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2195, - "output": 106 + "output": 105 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json index e14e18a7d207d..a4ae141362c51 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectTitle-wrong-title-error.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page title is \"Other Title\" using the browser_expect_title tool." + "text": "I need to verify that the page title is \"Other Title\" using one of the browser_expect_* tools." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "title": "Other Title", "_is_done": true }, - "id": "toolu_017UT5LEe8JRoB53R1YGKrYD" + "id": "toolu_017ZvXqNbcab9Uyjzw3VuByK" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2194, - "output": 97 + "output": 100 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json index c3f3b6c22520b..56fadb43dc1e4 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-success.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page URL is /counter.html using the browser_expect_url tool." + "text": "I'll verify that the page URL is /counter.html." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "url": "/counter.html", "_is_done": true }, - "id": "toolu_017jkU3FcrQpAVJgtjrzYovK" + "id": "toolu_01PkjsZaM5eEGtjLtMixyJUk" } ], "stopReason": { @@ -23,25 +23,25 @@ }, "usage": { "input": 2240, - "output": 99 + "output": 91 } }, - "f6d9bc80fd5bad48f6a886d7ec77c8d5889521b1": { + "71a7a4f51b3994a200f239eda9b505454825a79f": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue - the actual URL includes a server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I need to use a regex pattern to match the URL since it includes a server prefix. Let me update the expectation to match the actual URL pattern." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/counter\\.html$/", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_01JQuC1CJR7W6LiwAnvjaiDX" + "id": "toolu_01Wwq7Pbe2DthqpmskgGfCzk" } ], "stopReason": { @@ -49,8 +49,8 @@ } }, "usage": { - "input": 2441, - "output": 109 + "input": 2433, + "output": 111 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json index f034545e132aa..3f76a92de99db 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex-error.json @@ -5,7 +5,7 @@ "content": [ { "type": "text", - "text": "I'll verify that the page URL matches the `/counter` pattern using the browser_expect_url tool with a regex pattern." + "text": "I'll verify that the page URL matches the /counter pattern using the browser_expect_url tool with a regex pattern." }, { "type": "tool_call", @@ -14,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_01J38E1DkiNA7JsvQtGx3otW" + "id": "toolu_01JxsK661QmxpScYeTMCfWTZ" } ], "stopReason": { @@ -23,7 +23,7 @@ }, "usage": { "input": 2239, - "output": 103 + "output": 102 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json index 19a6a08b333e3..c8ae66692ec94 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-with-regex.json @@ -14,7 +14,7 @@ "regex": "/counter/", "_is_done": true }, - "id": "toolu_01Qen21mVwEVb2JsEVavTWbf" + "id": "toolu_01YVkbby1j3FBXk2o5Bi4cjP" } ], "stopReason": { diff --git a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json index 43b309b180056..b88a2b2d35cfc 100644 --- a/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json +++ b/tests/library/__llm_cache__/library-agent-expect-expectURL-wrong-URL-error.json @@ -14,7 +14,7 @@ "url": "/counter.html", "_is_done": true }, - "id": "toolu_01B3LSMGJoDrDfCZKwa3neZ9" + "id": "toolu_01BrgLhGzD2dYWeu75KGgxKv" } ], "stopReason": { @@ -26,22 +26,22 @@ "output": 99 } }, - "99f45f7c71e64f612bfc0bcb148faf235ccfe006": { + "8315bc81a0dfc915cb8be72ad00102f627e370dd": { "result": { "role": "assistant", "content": [ { "type": "text", - "text": "I see the issue. The actual URL includes the server prefix. Let me use a regex pattern to match the URL ending with /counter.html." + "text": "I see the issue. The actual URL includes the server portion. Let me use a regex pattern to match the URL ending with /counter.html." }, { "type": "tool_call", "name": "browser_expect_url", "arguments": { - "regex": "/counter\\.html$/", + "regex": "/\\/counter\\.html$/", "_is_done": true }, - "id": "toolu_01Ufux7hKwV9Djp9jT1JgYwJ" + "id": "toolu_01KtWVz2EdJjx2V9viYeqs71" } ], "stopReason": { @@ -50,7 +50,7 @@ }, "usage": { "input": 2441, - "output": 109 + "output": 110 } } } \ No newline at end of file diff --git a/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json new file mode 100644 index 0000000000000..1cd1ae76339cf --- /dev/null +++ b/tests/library/__llm_cache__/library-agent-perform-perform-run-timeout-inherited-from-page.json @@ -0,0 +1,30 @@ +{ + "554366043f6477d30a36655c10cbb4ec0eaa21d4": { + "result": { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll click the Fox button for you." + }, + { + "type": "tool_call", + "name": "browser_click", + "arguments": { + "element": "Fox button", + "ref": "e3", + "_is_done": true + }, + "id": "toolu_01TBMe2oPqGUpNuPnFnDeZJe" + } + ], + "stopReason": { + "code": "ok" + } + }, + "usage": { + "input": 3066, + "output": 101 + } + } +} \ No newline at end of file diff --git a/tests/library/agent-expect.spec.ts b/tests/library/agent-expect.spec.ts index c7befd887ab86..f301a6c5ab1b9 100644 --- a/tests/library/agent-expect.spec.ts +++ b/tests/library/agent-expect.spec.ts @@ -196,16 +196,36 @@ test('expect timeout during run', async ({ context }) => { { const { page, agent } = await runAgent(context); await page.setContent(``); - const error = await agent.expect('submit button is visible', { timeout: 10000 }).catch(e => e); + const error = await agent.expect('submit button is visible', { timeout: 3000 }).catch(e => e); expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed Locator: getByRole('button', { name: 'Submit' }) Expected: visible -Timeout: 5000ms +Timeout: 3000ms Error: element(s) not found Call log: - - Expect Visible with timeout 5000ms`); + - Expect Visible with timeout 3000ms`); + } +}); + +test('expect timeout during run from agent options', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(``); + await agent.expect('submit button is visible'); + } + expect(await cacheObject()).toEqual({ + 'submit button is visible': { + actions: [expect.objectContaining({ method: 'expectVisible' })], + }, + }); + { + const { page, agent } = await runAgent(context, { expect: { timeout: 3000 } }); + await page.setContent(``); + const error = await agent.expect('submit button is visible').catch(e => e); + expect(stripAnsi(error.message)).toContain(`pageAgent.expect: expect(locator).toBeVisible() failed`); + expect(stripAnsi(error.message)).toContain(`Expect Visible with timeout 3000ms`); } }); diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts index 88492bca9607d..5a718082df48d 100644 --- a/tests/library/agent-helpers.ts +++ b/tests/library/agent-helpers.ts @@ -55,9 +55,10 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio return { page, agent }; } -export async function runAgent(context: BrowserContext, options: { secrets?: Record } = {}) { +export async function runAgent(context: BrowserContext, options: AgentOptions = {}) { const page = await context.newPage(); const agent = await page.agent({ + ...options, cache: { cacheFile: cacheFile() }, ...{ _doNotRenderActive: true }, }); diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts index c7f80dd3ec9f0..9a14b409a20df 100644 --- a/tests/library/agent-perform.spec.ts +++ b/tests/library/agent-perform.spec.ts @@ -174,6 +174,28 @@ test('perform run timeout', async ({ context }) => { } }); +test('perform run timeout inherited from page', async ({ context }) => { + { + const { page, agent } = await generateAgent(context); + await page.setContent(` + + + `); + await agent.perform('click the Fox button'); + } + { + const { page, agent } = await runAgent(context); + await page.setContent(` + + + `); + page.setDefaultTimeout(3000); + const error = await agent.perform('click the Fox button').catch(e => e); + expect(error.message).toContain('Timeout 3000ms exceeded.'); + expect(error.message).toContain(`waiting for getByRole('button', { name: 'Fox' })`); + } +}); + test('invalid cache file throws error', async ({ context }) => { await setCacheObject({ 'some key': {