From eeca541dde17ef1bfd44fb0c0498cd58b116618a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 12 Jan 2026 17:31:49 +0000 Subject: [PATCH] feat(browser): expand browser control surface --- src/agents/tools/browser-tool.ts | 5 +- src/browser/client-actions-core.ts | 28 +- src/browser/client-actions-observe.ts | 108 ++++- src/browser/client-actions-state.ts | 307 +++++++++++++ src/browser/client-actions-types.ts | 2 + src/browser/client-actions.ts | 1 + src/browser/client.ts | 29 ++ src/browser/pw-ai.ts | 19 + src/browser/pw-role-snapshot.ts | 22 + src/browser/pw-session.ts | 111 ++++- src/browser/pw-tools-core.ts | 631 +++++++++++++++++++++++--- src/browser/routes/agent.ts | 549 +++++++++++++++++++++- 12 files changed, 1747 insertions(+), 65 deletions(-) create mode 100644 src/browser/client-actions-state.ts diff --git a/src/agents/tools/browser-tool.ts b/src/agents/tools/browser-tool.ts index ffea30a85..3b33bbac4 100644 --- a/src/agents/tools/browser-tool.ts +++ b/src/agents/tools/browser-tool.ts @@ -125,6 +125,7 @@ const BrowserToolSchema = Type.Object({ compact: Type.Optional(Type.Boolean()), depth: Type.Optional(Type.Number()), selector: Type.Optional(Type.String()), + frame: Type.Optional(Type.String()), fullPage: Type.Optional(Type.Boolean()), ref: Type.Optional(Type.String()), element: Type.Optional(Type.String()), @@ -354,16 +355,18 @@ export function createBrowserTool(opts?: { typeof params.selector === "string" ? params.selector.trim() : undefined; + const frame = + typeof params.frame === "string" ? params.frame.trim() : undefined; const snapshot = await browserSnapshot(baseUrl, { format, targetId, limit, ...(resolvedMaxChars ? { maxChars: resolvedMaxChars } : {}), - ...(resolvedMaxChars ? { maxChars: resolvedMaxChars } : {}), interactive, compact, depth, selector, + frame, profile, }); if (snapshot.format === "ai") { diff --git a/src/browser/client-actions-core.ts b/src/browser/client-actions-core.ts index c1bd15d91..80d21a890 100644 --- a/src/browser/client-actions-core.ts +++ b/src/browser/client-actions-core.ts @@ -23,6 +23,7 @@ export type BrowserActRequest = doubleClick?: boolean; button?: string; modifiers?: string[]; + timeoutMs?: number; } | { kind: "type"; @@ -31,15 +32,29 @@ export type BrowserActRequest = targetId?: string; submit?: boolean; slowly?: boolean; + timeoutMs?: number; + } + | { kind: "press"; key: string; targetId?: string; delayMs?: number } + | { kind: "hover"; ref: string; targetId?: string; timeoutMs?: number } + | { + kind: "drag"; + startRef: string; + endRef: string; + targetId?: string; + timeoutMs?: number; + } + | { + kind: "select"; + ref: string; + values: string[]; + targetId?: string; + timeoutMs?: number; } - | { kind: "press"; key: string; targetId?: string } - | { kind: "hover"; ref: string; targetId?: string } - | { kind: "drag"; startRef: string; endRef: string; targetId?: string } - | { kind: "select"; ref: string; values: string[]; targetId?: string } | { kind: "fill"; fields: BrowserFormField[]; targetId?: string; + timeoutMs?: number; } | { kind: "resize"; width: number; height: number; targetId?: string } | { @@ -47,7 +62,12 @@ export type BrowserActRequest = timeMs?: number; text?: string; textGone?: string; + selector?: string; + url?: string; + loadState?: "load" | "domcontentloaded" | "networkidle"; + fn?: string; targetId?: string; + timeoutMs?: number; } | { kind: "evaluate"; fn: string; ref?: string; targetId?: string } | { kind: "close"; targetId?: string }; diff --git a/src/browser/client-actions-observe.ts b/src/browser/client-actions-observe.ts index bd34e2b44..b4db4aad2 100644 --- a/src/browser/client-actions-observe.ts +++ b/src/browser/client-actions-observe.ts @@ -1,6 +1,13 @@ -import type { BrowserActionPathResult } from "./client-actions-types.js"; +import type { + BrowserActionPathResult, + BrowserActionTargetOk, +} from "./client-actions-types.js"; import { fetchBrowserJson } from "./client-fetch.js"; -import type { BrowserConsoleMessage } from "./pw-session.js"; +import type { + BrowserConsoleMessage, + BrowserNetworkRequest, + BrowserPageError, +} from "./pw-session.js"; function buildProfileQuery(profile?: string): string { return profile ? `?profile=${encodeURIComponent(profile)}` : ""; @@ -34,3 +41,100 @@ export async function browserPdfSave( timeoutMs: 20000, }); } + +export async function browserPageErrors( + baseUrl: string, + opts: { targetId?: string; clear?: boolean; profile?: string } = {}, +): Promise<{ ok: true; targetId: string; errors: BrowserPageError[] }> { + const q = new URLSearchParams(); + if (opts.targetId) q.set("targetId", opts.targetId); + if (typeof opts.clear === "boolean") q.set("clear", String(opts.clear)); + if (opts.profile) q.set("profile", opts.profile); + const suffix = q.toString() ? `?${q.toString()}` : ""; + return await fetchBrowserJson<{ + ok: true; + targetId: string; + errors: BrowserPageError[]; + }>(`${baseUrl}/errors${suffix}`, { timeoutMs: 20000 }); +} + +export async function browserRequests( + baseUrl: string, + opts: { + targetId?: string; + filter?: string; + clear?: boolean; + profile?: string; + } = {}, +): Promise<{ ok: true; targetId: string; requests: BrowserNetworkRequest[] }> { + const q = new URLSearchParams(); + if (opts.targetId) q.set("targetId", opts.targetId); + if (opts.filter) q.set("filter", opts.filter); + if (typeof opts.clear === "boolean") q.set("clear", String(opts.clear)); + if (opts.profile) q.set("profile", opts.profile); + const suffix = q.toString() ? `?${q.toString()}` : ""; + return await fetchBrowserJson<{ + ok: true; + targetId: string; + requests: BrowserNetworkRequest[]; + }>(`${baseUrl}/requests${suffix}`, { timeoutMs: 20000 }); +} + +export async function browserTraceStart( + baseUrl: string, + opts: { + targetId?: string; + screenshots?: boolean; + snapshots?: boolean; + sources?: boolean; + profile?: string; + } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/trace/start${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + screenshots: opts.screenshots, + snapshots: opts.snapshots, + sources: opts.sources, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserTraceStop( + baseUrl: string, + opts: { targetId?: string; path?: string; profile?: string } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/trace/stop${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, path: opts.path }), + timeoutMs: 20000, + }, + ); +} + +export async function browserHighlight( + baseUrl: string, + opts: { ref: string; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/highlight${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, ref: opts.ref }), + timeoutMs: 20000, + }, + ); +} diff --git a/src/browser/client-actions-state.ts b/src/browser/client-actions-state.ts new file mode 100644 index 000000000..a84631718 --- /dev/null +++ b/src/browser/client-actions-state.ts @@ -0,0 +1,307 @@ +import type { + BrowserActionOk, + BrowserActionTargetOk, +} from "./client-actions-types.js"; +import { fetchBrowserJson } from "./client-fetch.js"; + +function buildProfileQuery(profile?: string): string { + return profile ? `?profile=${encodeURIComponent(profile)}` : ""; +} + +export async function browserCookies( + baseUrl: string, + opts: { targetId?: string; profile?: string } = {}, +): Promise<{ ok: true; targetId: string; cookies: unknown[] }> { + const q = new URLSearchParams(); + if (opts.targetId) q.set("targetId", opts.targetId); + if (opts.profile) q.set("profile", opts.profile); + const suffix = q.toString() ? `?${q.toString()}` : ""; + return await fetchBrowserJson<{ + ok: true; + targetId: string; + cookies: unknown[]; + }>(`${baseUrl}/cookies${suffix}`, { timeoutMs: 20000 }); +} + +export async function browserCookiesSet( + baseUrl: string, + opts: { + cookie: Record; + targetId?: string; + profile?: string; + }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/cookies/set${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, cookie: opts.cookie }), + timeoutMs: 20000, + }, + ); +} + +export async function browserCookiesClear( + baseUrl: string, + opts: { targetId?: string; profile?: string } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/cookies/clear${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId }), + timeoutMs: 20000, + }, + ); +} + +export async function browserStorageGet( + baseUrl: string, + opts: { + kind: "local" | "session"; + key?: string; + targetId?: string; + profile?: string; + }, +): Promise<{ ok: true; targetId: string; values: Record }> { + const q = new URLSearchParams(); + if (opts.targetId) q.set("targetId", opts.targetId); + if (opts.key) q.set("key", opts.key); + if (opts.profile) q.set("profile", opts.profile); + const suffix = q.toString() ? `?${q.toString()}` : ""; + return await fetchBrowserJson<{ + ok: true; + targetId: string; + values: Record; + }>(`${baseUrl}/storage/${opts.kind}${suffix}`, { timeoutMs: 20000 }); +} + +export async function browserStorageSet( + baseUrl: string, + opts: { + kind: "local" | "session"; + key: string; + value: string; + targetId?: string; + profile?: string; + }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/storage/${opts.kind}/set${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + key: opts.key, + value: opts.value, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserStorageClear( + baseUrl: string, + opts: { kind: "local" | "session"; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/storage/${opts.kind}/clear${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetOffline( + baseUrl: string, + opts: { offline: boolean; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/offline${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, offline: opts.offline }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetHeaders( + baseUrl: string, + opts: { + headers: Record; + targetId?: string; + profile?: string; + }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/headers${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, headers: opts.headers }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetHttpCredentials( + baseUrl: string, + opts: { + username?: string; + password?: string; + clear?: boolean; + targetId?: string; + profile?: string; + } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/credentials${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + username: opts.username, + password: opts.password, + clear: opts.clear, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetGeolocation( + baseUrl: string, + opts: { + latitude?: number; + longitude?: number; + accuracy?: number; + origin?: string; + clear?: boolean; + targetId?: string; + profile?: string; + } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/geolocation${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + latitude: opts.latitude, + longitude: opts.longitude, + accuracy: opts.accuracy, + origin: opts.origin, + clear: opts.clear, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetMedia( + baseUrl: string, + opts: { + colorScheme: "dark" | "light" | "no-preference" | "none"; + targetId?: string; + profile?: string; + }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/media${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + colorScheme: opts.colorScheme, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetTimezone( + baseUrl: string, + opts: { timezoneId: string; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/timezone${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + timezoneId: opts.timezoneId, + }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetLocale( + baseUrl: string, + opts: { locale: string; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/locale${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, locale: opts.locale }), + timeoutMs: 20000, + }, + ); +} + +export async function browserSetDevice( + baseUrl: string, + opts: { name: string; targetId?: string; profile?: string }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/device${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, name: opts.name }), + timeoutMs: 20000, + }, + ); +} + +export async function browserClearPermissions( + baseUrl: string, + opts: { targetId?: string; profile?: string } = {}, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson( + `${baseUrl}/set/geolocation${q}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ targetId: opts.targetId, clear: true }), + timeoutMs: 20000, + }, + ); +} diff --git a/src/browser/client-actions-types.ts b/src/browser/client-actions-types.ts index c1c765ff7..9ad0d820d 100644 --- a/src/browser/client-actions-types.ts +++ b/src/browser/client-actions-types.ts @@ -12,3 +12,5 @@ export type BrowserActionPathResult = { targetId: string; url?: string; }; + +export type BrowserActionTargetOk = { ok: true; targetId: string }; diff --git a/src/browser/client-actions.ts b/src/browser/client-actions.ts index 24fd38be9..c495f5d01 100644 --- a/src/browser/client-actions.ts +++ b/src/browser/client-actions.ts @@ -1,3 +1,4 @@ export * from "./client-actions-core.js"; export * from "./client-actions-observe.js"; +export * from "./client-actions-state.js"; export * from "./client-actions-types.js"; diff --git a/src/browser/client.ts b/src/browser/client.ts index c87d4dfc1..279ac5edf 100644 --- a/src/browser/client.ts +++ b/src/browser/client.ts @@ -72,6 +72,13 @@ export type SnapshotResult = url: string; snapshot: string; truncated?: boolean; + refs?: Record; + stats?: { + lines: number; + chars: number; + refs: number; + interactive: number; + }; }; export function resolveBrowserControlUrl(overrideUrl?: string) { @@ -243,6 +250,26 @@ export async function browserCloseTab( ); } +export async function browserTabAction( + baseUrl: string, + opts: { + action: "list" | "new" | "close" | "select"; + index?: number; + profile?: string; + }, +): Promise { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson(`${baseUrl}/tabs/action${q}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action: opts.action, + index: opts.index, + }), + timeoutMs: 10_000, + }); +} + export async function browserSnapshot( baseUrl: string, opts: { @@ -254,6 +281,7 @@ export async function browserSnapshot( compact?: boolean; depth?: number; selector?: string; + frame?: string; profile?: string; }, ): Promise { @@ -270,6 +298,7 @@ export async function browserSnapshot( if (typeof opts.depth === "number" && Number.isFinite(opts.depth)) q.set("depth", String(opts.depth)); if (opts.selector?.trim()) q.set("selector", opts.selector.trim()); + if (opts.frame?.trim()) q.set("frame", opts.frame.trim()); if (opts.profile) q.set("profile", opts.profile); return await fetchBrowserJson( `${baseUrl}/snapshot?${q.toString()}`, diff --git a/src/browser/pw-ai.ts b/src/browser/pw-ai.ts index 026190fe4..6ea78c72e 100644 --- a/src/browser/pw-ai.ts +++ b/src/browser/pw-ai.ts @@ -12,20 +12,39 @@ export { armFileUploadViaPlaywright, clickViaPlaywright, closePageViaPlaywright, + cookiesClearViaPlaywright, + cookiesGetViaPlaywright, + cookiesSetViaPlaywright, dragViaPlaywright, + emulateMediaViaPlaywright, evaluateViaPlaywright, fillFormViaPlaywright, getConsoleMessagesViaPlaywright, + getNetworkRequestsViaPlaywright, + getPageErrorsViaPlaywright, + highlightViaPlaywright, hoverViaPlaywright, navigateViaPlaywright, pdfViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, selectOptionViaPlaywright, + setDeviceViaPlaywright, + setExtraHTTPHeadersViaPlaywright, + setGeolocationViaPlaywright, + setHttpCredentialsViaPlaywright, setInputFilesViaPlaywright, + setLocaleViaPlaywright, + setOfflineViaPlaywright, + setTimezoneViaPlaywright, snapshotAiViaPlaywright, snapshotRoleViaPlaywright, + storageClearViaPlaywright, + storageGetViaPlaywright, + storageSetViaPlaywright, takeScreenshotViaPlaywright, + traceStartViaPlaywright, + traceStopViaPlaywright, typeViaPlaywright, waitForViaPlaywright, } from "./pw-tools-core.js"; diff --git a/src/browser/pw-role-snapshot.ts b/src/browser/pw-role-snapshot.ts index 9ddbda251..e39fb392e 100644 --- a/src/browser/pw-role-snapshot.ts +++ b/src/browser/pw-role-snapshot.ts @@ -7,6 +7,13 @@ export type RoleRef = { export type RoleRefMap = Record; +export type RoleSnapshotStats = { + lines: number; + chars: number; + refs: number; + interactive: number; +}; + export type RoleSnapshotOptions = { /** Only include interactive elements (buttons, links, inputs, etc.). */ interactive?: boolean; @@ -70,6 +77,21 @@ const STRUCTURAL_ROLES = new Set([ "none", ]); +export function getRoleSnapshotStats( + snapshot: string, + refs: RoleRefMap, +): RoleSnapshotStats { + const interactive = Object.values(refs).filter((r) => + INTERACTIVE_ROLES.has(r.role), + ).length; + return { + lines: snapshot.split("\n").length, + chars: snapshot.length, + refs: Object.keys(refs).length, + interactive, + }; +} + function getIndentLevel(line: string): number { const match = line.match(/^(\s*)/); return match ? Math.floor(match[1].length / 2) : 0; diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index 30278e34d..b3a1227f8 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -3,6 +3,8 @@ import type { BrowserContext, ConsoleMessage, Page, + Request, + Response, } from "playwright-core"; import { chromium } from "playwright-core"; import { formatErrorMessage } from "../infra/errors.js"; @@ -15,6 +17,24 @@ export type BrowserConsoleMessage = { location?: { url?: string; lineNumber?: number; columnNumber?: number }; }; +export type BrowserPageError = { + message: string; + name?: string; + stack?: string; + timestamp: string; +}; + +export type BrowserNetworkRequest = { + id: string; + timestamp: string; + method: string; + url: string; + resourceType?: string; + status?: number; + ok?: boolean; + failureText?: string; +}; + type SnapshotForAIResult = { full: string; incremental?: string }; type SnapshotForAIOptions = { timeout?: number; track?: string }; @@ -37,6 +57,10 @@ type ConnectedBrowser = { type PageState = { console: BrowserConsoleMessage[]; + errors: BrowserPageError[]; + requests: BrowserNetworkRequest[]; + requestIds: WeakMap; + nextRequestId: number; armIdUpload: number; armIdDialog: number; /** @@ -44,13 +68,21 @@ type PageState = { * These refs are NOT Playwright's `aria-ref` values. */ roleRefs?: Record; + roleRefsFrameSelector?: string; +}; + +type ContextState = { + traceActive: boolean; }; const pageStates = new WeakMap(); +const contextStates = new WeakMap(); const observedContexts = new WeakSet(); const observedPages = new WeakSet(); const MAX_CONSOLE_MESSAGES = 500; +const MAX_PAGE_ERRORS = 200; +const MAX_NETWORK_REQUESTS = 500; let cached: ConnectedBrowser | null = null; let connecting: Promise | null = null; @@ -65,6 +97,10 @@ export function ensurePageState(page: Page): PageState { const state: PageState = { console: [], + errors: [], + requests: [], + requestIds: new WeakMap(), + nextRequestId: 0, armIdUpload: 0, armIdDialog: 0, }; @@ -82,6 +118,59 @@ export function ensurePageState(page: Page): PageState { state.console.push(entry); if (state.console.length > MAX_CONSOLE_MESSAGES) state.console.shift(); }); + page.on("pageerror", (err: Error) => { + state.errors.push({ + message: err?.message ? String(err.message) : String(err), + name: err?.name ? String(err.name) : undefined, + stack: err?.stack ? String(err.stack) : undefined, + timestamp: new Date().toISOString(), + }); + if (state.errors.length > MAX_PAGE_ERRORS) state.errors.shift(); + }); + page.on("request", (req: Request) => { + state.nextRequestId += 1; + const id = `r${state.nextRequestId}`; + state.requestIds.set(req, id); + state.requests.push({ + id, + timestamp: new Date().toISOString(), + method: req.method(), + url: req.url(), + resourceType: req.resourceType(), + }); + if (state.requests.length > MAX_NETWORK_REQUESTS) state.requests.shift(); + }); + page.on("response", (resp: Response) => { + const req = resp.request(); + const id = state.requestIds.get(req); + if (!id) return; + let rec: BrowserNetworkRequest | undefined; + for (let i = state.requests.length - 1; i >= 0; i -= 1) { + const candidate = state.requests[i]; + if (candidate && candidate.id === id) { + rec = candidate; + break; + } + } + if (!rec) return; + rec.status = resp.status(); + rec.ok = resp.ok(); + }); + page.on("requestfailed", (req: Request) => { + const id = state.requestIds.get(req); + if (!id) return; + let rec: BrowserNetworkRequest | undefined; + for (let i = state.requests.length - 1; i >= 0; i -= 1) { + const candidate = state.requests[i]; + if (candidate && candidate.id === id) { + rec = candidate; + break; + } + } + if (!rec) return; + rec.failureText = req.failure()?.errorText; + rec.ok = false; + }); page.on("close", () => { pageStates.delete(page); observedPages.delete(page); @@ -94,11 +183,20 @@ export function ensurePageState(page: Page): PageState { function observeContext(context: BrowserContext) { if (observedContexts.has(context)) return; observedContexts.add(context); + ensureContextState(context); for (const page of context.pages()) ensurePageState(page); context.on("page", (page) => ensurePageState(page)); } +export function ensureContextState(context: BrowserContext): ContextState { + const existing = contextStates.get(context); + if (existing) return existing; + const state: ContextState = { traceActive: false }; + contextStates.set(context, state); + return state; +} + function observeBrowser(browser: Browser) { for (const context of browser.contexts()) observeContext(context); } @@ -208,9 +306,18 @@ export function refLocator(page: Page, ref: string) { `Unknown ref "${normalized}". Run a new snapshot and use a ref from that snapshot.`, ); } + const scope = state?.roleRefsFrameSelector + ? page.frameLocator(state.roleRefsFrameSelector) + : page; + const locAny = scope as unknown as { + getByRole: ( + role: never, + opts?: { name?: string; exact?: boolean }, + ) => ReturnType; + }; const locator = info.name - ? page.getByRole(info.role as never, { name: info.name, exact: true }) - : page.getByRole(info.role as never); + ? locAny.getByRole(info.role as never, { name: info.name, exact: true }) + : locAny.getByRole(info.role as never); return info.nth !== undefined ? locator.nth(info.nth) : locator; } diff --git a/src/browser/pw-tools-core.ts b/src/browser/pw-tools-core.ts index bf23f888a..c16ba3ac6 100644 --- a/src/browser/pw-tools-core.ts +++ b/src/browser/pw-tools-core.ts @@ -1,11 +1,17 @@ +import type { CDPSession, Page } from "playwright-core"; +import { devices as playwrightDevices } from "playwright-core"; import type { BrowserFormField } from "./client-actions-core.js"; import { buildRoleSnapshotFromAriaSnapshot, + getRoleSnapshotStats, parseRoleRef, type RoleSnapshotOptions, } from "./pw-role-snapshot.js"; import { type BrowserConsoleMessage, + type BrowserNetworkRequest, + type BrowserPageError, + ensureContextState, ensurePageState, getPageForTargetId, refLocator, @@ -23,6 +29,42 @@ function requireRef(value: unknown): string { return ref; } +function toAIFriendlyError(error: unknown, selector: string): Error { + const message = error instanceof Error ? error.message : String(error); + + if (message.includes("strict mode violation")) { + const countMatch = message.match(/resolved to (\d+) elements/); + const count = countMatch ? countMatch[1] : "multiple"; + return new Error( + `Selector "${selector}" matched ${count} elements. ` + + `Run a new snapshot to get updated refs, or use a different ref.`, + ); + } + + if ( + (message.includes("Timeout") || message.includes("waiting for")) && + (message.includes("to be visible") || message.includes("not visible")) + ) { + return new Error( + `Element "${selector}" not found or not visible. ` + + `Run a new snapshot to see current page elements.`, + ); + } + + if ( + message.includes("intercepts pointer events") || + message.includes("not visible") || + message.includes("not receive pointer events") + ) { + return new Error( + `Element "${selector}" is not interactable (hidden or covered). ` + + `Try scrolling it into view, closing overlays, or re-snapshotting.`, + ); + } + + return error instanceof Error ? error : new Error(message); +} + export async function snapshotAiViaPlaywright(opts: { cdpUrl: string; targetId?: string; @@ -66,17 +108,28 @@ export async function snapshotRoleViaPlaywright(opts: { cdpUrl: string; targetId?: string; selector?: string; + frameSelector?: string; options?: RoleSnapshotOptions; -}): Promise<{ snapshot: string }> { +}): Promise<{ + snapshot: string; + refs: Record; + stats: { lines: number; chars: number; refs: number; interactive: number }; +}> { const page = await getPageForTargetId({ cdpUrl: opts.cdpUrl, targetId: opts.targetId, }); const state = ensurePageState(page); - const locator = opts.selector?.trim() - ? page.locator(opts.selector.trim()) - : page.locator(":root"); + const frameSelector = opts.frameSelector?.trim() || ""; + const selector = opts.selector?.trim() || ""; + const locator = frameSelector + ? selector + ? page.frameLocator(frameSelector).locator(selector) + : page.frameLocator(frameSelector).locator(":root") + : selector + ? page.locator(selector) + : page.locator(":root"); const ariaSnapshot = await locator.ariaSnapshot(); const built = buildRoleSnapshotFromAriaSnapshot( @@ -84,7 +137,95 @@ export async function snapshotRoleViaPlaywright(opts: { opts.options, ); state.roleRefs = built.refs; - return { snapshot: built.snapshot }; + state.roleRefsFrameSelector = frameSelector || undefined; + return { + snapshot: built.snapshot, + refs: built.refs, + stats: getRoleSnapshotStats(built.snapshot, built.refs), + }; +} + +export async function getPageErrorsViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + clear?: boolean; +}): Promise<{ errors: BrowserPageError[] }> { + const page = await getPageForTargetId(opts); + const state = ensurePageState(page); + const errors = [...state.errors]; + if (opts.clear) state.errors = []; + return { errors }; +} + +export async function getNetworkRequestsViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + filter?: string; + clear?: boolean; +}): Promise<{ requests: BrowserNetworkRequest[] }> { + const page = await getPageForTargetId(opts); + const state = ensurePageState(page); + const raw = [...state.requests]; + const filter = typeof opts.filter === "string" ? opts.filter.trim() : ""; + const requests = filter ? raw.filter((r) => r.url.includes(filter)) : raw; + if (opts.clear) { + state.requests = []; + state.requestIds = new WeakMap(); + } + return { requests }; +} + +export async function highlightViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + ref: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const ref = requireRef(opts.ref); + try { + await refLocator(page, ref).highlight(); + } catch (err) { + throw toAIFriendlyError(err, ref); + } +} + +export async function traceStartViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + screenshots?: boolean; + snapshots?: boolean; + sources?: boolean; +}): Promise { + const page = await getPageForTargetId(opts); + const context = page.context(); + const ctxState = ensureContextState(context); + if (ctxState.traceActive) { + throw new Error( + "Trace already running. Stop the current trace before starting a new one.", + ); + } + await context.tracing.start({ + screenshots: opts.screenshots ?? true, + snapshots: opts.snapshots ?? true, + sources: opts.sources ?? false, + }); + ctxState.traceActive = true; +} + +export async function traceStopViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + path: string; +}): Promise { + const page = await getPageForTargetId(opts); + const context = page.context(); + const ctxState = ensureContextState(context); + if (!ctxState.traceActive) { + throw new Error("No active trace. Start a trace before stopping it."); + } + await context.tracing.stop({ path: opts.path }); + ctxState.traceActive = false; } export async function clickViaPlaywright(opts: { @@ -101,23 +242,28 @@ export async function clickViaPlaywright(opts: { targetId: opts.targetId, }); ensurePageState(page); - const locator = refLocator(page, requireRef(opts.ref)); + const ref = requireRef(opts.ref); + const locator = refLocator(page, ref); const timeout = Math.max( 500, Math.min(60_000, Math.floor(opts.timeoutMs ?? 8000)), ); - if (opts.doubleClick) { - await locator.dblclick({ - timeout, - button: opts.button, - modifiers: opts.modifiers, - }); - } else { - await locator.click({ - timeout, - button: opts.button, - modifiers: opts.modifiers, - }); + try { + if (opts.doubleClick) { + await locator.dblclick({ + timeout, + button: opts.button, + modifiers: opts.modifiers, + }); + } else { + await locator.click({ + timeout, + button: opts.button, + modifiers: opts.modifiers, + }); + } + } catch (err) { + throw toAIFriendlyError(err, ref); } } @@ -130,9 +276,13 @@ export async function hoverViaPlaywright(opts: { const ref = requireRef(opts.ref); const page = await getPageForTargetId(opts); ensurePageState(page); - await refLocator(page, ref).hover({ - timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), - }); + try { + await refLocator(page, ref).hover({ + timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), + }); + } catch (err) { + throw toAIFriendlyError(err, ref); + } } export async function dragViaPlaywright(opts: { @@ -147,9 +297,13 @@ export async function dragViaPlaywright(opts: { if (!startRef || !endRef) throw new Error("startRef and endRef are required"); const page = await getPageForTargetId(opts); ensurePageState(page); - await refLocator(page, startRef).dragTo(refLocator(page, endRef), { - timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), - }); + try { + await refLocator(page, startRef).dragTo(refLocator(page, endRef), { + timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), + }); + } catch (err) { + throw toAIFriendlyError(err, `${startRef} -> ${endRef}`); + } } export async function selectOptionViaPlaywright(opts: { @@ -163,9 +317,13 @@ export async function selectOptionViaPlaywright(opts: { if (!opts.values?.length) throw new Error("values are required"); const page = await getPageForTargetId(opts); ensurePageState(page); - await refLocator(page, ref).selectOption(opts.values, { - timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), - }); + try { + await refLocator(page, ref).selectOption(opts.values, { + timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)), + }); + } catch (err) { + throw toAIFriendlyError(err, ref); + } } export async function pressKeyViaPlaywright(opts: { @@ -183,6 +341,330 @@ export async function pressKeyViaPlaywright(opts: { }); } +export async function cookiesGetViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; +}): Promise<{ cookies: unknown[] }> { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const cookies = await page.context().cookies(); + return { cookies }; +} + +export async function cookiesSetViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + cookie: { + name: string; + value: string; + url?: string; + domain?: string; + path?: string; + expires?: number; + httpOnly?: boolean; + secure?: boolean; + sameSite?: "Lax" | "None" | "Strict"; + }; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const cookie = opts.cookie; + if (!cookie.name || cookie.value === undefined) { + throw new Error("cookie name and value are required"); + } + const hasUrl = typeof cookie.url === "string" && cookie.url.trim(); + const hasDomainPath = + typeof cookie.domain === "string" && + cookie.domain.trim() && + typeof cookie.path === "string" && + cookie.path.trim(); + if (!hasUrl && !hasDomainPath) { + throw new Error("cookie requires url, or domain+path"); + } + await page.context().addCookies([cookie]); +} + +export async function cookiesClearViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + await page.context().clearCookies(); +} + +type StorageKind = "local" | "session"; + +export async function storageGetViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + kind: StorageKind; + key?: string; +}): Promise<{ values: Record }> { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const kind = opts.kind; + const key = typeof opts.key === "string" ? opts.key : undefined; + const values = await page.evaluate( + ({ kind: kind2, key: key2 }) => { + const store = + kind2 === "session" ? window.sessionStorage : window.localStorage; + if (key2) { + const value = store.getItem(key2); + return value === null ? {} : { [key2]: value }; + } + const out: Record = {}; + for (let i = 0; i < store.length; i += 1) { + const k = store.key(i); + if (!k) continue; + const v = store.getItem(k); + if (v !== null) out[k] = v; + } + return out; + }, + { kind, key }, + ); + return { values: values ?? {} }; +} + +export async function storageSetViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + kind: StorageKind; + key: string; + value: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const key = String(opts.key ?? ""); + if (!key) throw new Error("key is required"); + await page.evaluate( + ({ kind, key: k, value }) => { + const store = + kind === "session" ? window.sessionStorage : window.localStorage; + store.setItem(k, value); + }, + { kind: opts.kind, key, value: String(opts.value ?? "") }, + ); +} + +export async function storageClearViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + kind: StorageKind; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + await page.evaluate( + ({ kind }) => { + const store = + kind === "session" ? window.sessionStorage : window.localStorage; + store.clear(); + }, + { kind: opts.kind }, + ); +} + +export async function setOfflineViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + offline: boolean; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + await page.context().setOffline(Boolean(opts.offline)); +} + +export async function setExtraHTTPHeadersViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + headers: Record; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + await page.context().setExtraHTTPHeaders(opts.headers); +} + +export async function setHttpCredentialsViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + username?: string; + password?: string; + clear?: boolean; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + if (opts.clear) { + await page.context().setHTTPCredentials(null); + return; + } + const username = String(opts.username ?? ""); + const password = String(opts.password ?? ""); + if (!username) throw new Error("username is required (or set clear=true)"); + await page.context().setHTTPCredentials({ username, password }); +} + +export async function setGeolocationViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + latitude?: number; + longitude?: number; + accuracy?: number; + origin?: string; + clear?: boolean; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const context = page.context(); + if (opts.clear) { + await context.setGeolocation(null); + await context.clearPermissions().catch(() => {}); + return; + } + if (typeof opts.latitude !== "number" || typeof opts.longitude !== "number") { + throw new Error("latitude and longitude are required (or set clear=true)"); + } + await context.setGeolocation({ + latitude: opts.latitude, + longitude: opts.longitude, + accuracy: typeof opts.accuracy === "number" ? opts.accuracy : undefined, + }); + const origin = + opts.origin?.trim() || + (() => { + try { + return new URL(page.url()).origin; + } catch { + return ""; + } + })(); + if (origin) { + await context.grantPermissions(["geolocation"], { origin }).catch(() => {}); + } +} + +export async function emulateMediaViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + colorScheme: "dark" | "light" | "no-preference" | null; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + await page.emulateMedia({ colorScheme: opts.colorScheme }); +} + +async function withCdpSession( + page: Page, + fn: (session: CDPSession) => Promise, +): Promise { + const session = await page.context().newCDPSession(page); + try { + return await fn(session); + } finally { + await session.detach().catch(() => {}); + } +} + +export async function setLocaleViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + locale: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const locale = String(opts.locale ?? "").trim(); + if (!locale) throw new Error("locale is required"); + await withCdpSession(page, async (session) => { + try { + await session.send("Emulation.setLocaleOverride", { locale }); + } catch (err) { + if ( + String(err).includes("Another locale override is already in effect") + ) { + return; + } + throw err; + } + }); +} + +export async function setTimezoneViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + timezoneId: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const timezoneId = String(opts.timezoneId ?? "").trim(); + if (!timezoneId) throw new Error("timezoneId is required"); + await withCdpSession(page, async (session) => { + try { + await session.send("Emulation.setTimezoneOverride", { timezoneId }); + } catch (err) { + const msg = String(err); + if (msg.includes("Timezone override is already in effect")) return; + if (msg.includes("Invalid timezone")) + throw new Error(`Invalid timezone ID: ${timezoneId}`); + throw err; + } + }); +} + +export async function setDeviceViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + name: string; +}): Promise { + const page = await getPageForTargetId(opts); + ensurePageState(page); + const name = String(opts.name ?? "").trim(); + if (!name) throw new Error("device name is required"); + const descriptor = (playwrightDevices as Record)[name] as + | { + userAgent?: string; + viewport?: { width: number; height: number }; + deviceScaleFactor?: number; + isMobile?: boolean; + hasTouch?: boolean; + locale?: string; + } + | undefined; + if (!descriptor) { + throw new Error(`Unknown device "${name}".`); + } + + if (descriptor.viewport) { + await page.setViewportSize({ + width: descriptor.viewport.width, + height: descriptor.viewport.height, + }); + } + + await withCdpSession(page, async (session) => { + if (descriptor.userAgent || descriptor.locale) { + await session.send("Emulation.setUserAgentOverride", { + userAgent: descriptor.userAgent ?? "", + acceptLanguage: descriptor.locale ?? undefined, + }); + } + if (descriptor.viewport) { + await session.send("Emulation.setDeviceMetricsOverride", { + mobile: Boolean(descriptor.isMobile), + width: descriptor.viewport.width, + height: descriptor.viewport.height, + deviceScaleFactor: descriptor.deviceScaleFactor ?? 1, + screenWidth: descriptor.viewport.width, + screenHeight: descriptor.viewport.height, + }); + } + if (descriptor.hasTouch) { + await session.send("Emulation.setTouchEmulationEnabled", { + enabled: true, + }); + } + }); +} + export async function typeViaPlaywright(opts: { cdpUrl: string; targetId?: string; @@ -195,16 +677,21 @@ export async function typeViaPlaywright(opts: { const text = String(opts.text ?? ""); const page = await getPageForTargetId(opts); ensurePageState(page); - const locator = refLocator(page, requireRef(opts.ref)); + const ref = requireRef(opts.ref); + const locator = refLocator(page, ref); const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)); - if (opts.slowly) { - await locator.click({ timeout }); - await locator.type(text, { timeout, delay: 75 }); - } else { - await locator.fill(text, { timeout }); - } - if (opts.submit) { - await locator.press("Enter", { timeout }); + try { + if (opts.slowly) { + await locator.click({ timeout }); + await locator.type(text, { timeout, delay: 75 }); + } else { + await locator.fill(text, { timeout }); + } + if (opts.submit) { + await locator.press("Enter", { timeout }); + } + } catch (err) { + throw toAIFriendlyError(err, ref); } } @@ -212,9 +699,11 @@ export async function fillFormViaPlaywright(opts: { cdpUrl: string; targetId?: string; fields: BrowserFormField[]; + timeoutMs?: number; }): Promise { const page = await getPageForTargetId(opts); ensurePageState(page); + const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)); for (const field of opts.fields) { const ref = field.ref.trim(); const type = field.type.trim(); @@ -233,10 +722,18 @@ export async function fillFormViaPlaywright(opts: { rawValue === 1 || rawValue === "1" || rawValue === "true"; - await locator.setChecked(checked); + try { + await locator.setChecked(checked, { timeout }); + } catch (err) { + throw toAIFriendlyError(err, ref); + } continue; } - await locator.fill(value); + try { + await locator.fill(value, { timeout }); + } catch (err) { + throw toAIFriendlyError(err, ref); + } } } @@ -359,7 +856,11 @@ export async function setInputFilesViaPlaywright(opts: { ? refLocator(page, inputRef) : page.locator(element).first(); - await locator.setInputFiles(opts.paths); + try { + await locator.setInputFiles(opts.paths); + } catch (err) { + throw toAIFriendlyError(err, inputRef || element); + } try { const handle = await locator.elementHandle(); if (handle) { @@ -421,30 +922,54 @@ export async function waitForViaPlaywright(opts: { timeMs?: number; text?: string; textGone?: string; + selector?: string; + url?: string; + loadState?: "load" | "domcontentloaded" | "networkidle"; + fn?: string; timeoutMs?: number; }): Promise { const page = await getPageForTargetId(opts); ensurePageState(page); + const timeout = Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)); + if (typeof opts.timeMs === "number" && Number.isFinite(opts.timeMs)) { await page.waitForTimeout(Math.max(0, opts.timeMs)); } if (opts.text) { - await page - .getByText(opts.text) - .first() - .waitFor({ - state: "visible", - timeout: Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)), - }); + await page.getByText(opts.text).first().waitFor({ + state: "visible", + timeout, + }); } if (opts.textGone) { - await page - .getByText(opts.textGone) - .first() - .waitFor({ - state: "hidden", - timeout: Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)), - }); + await page.getByText(opts.textGone).first().waitFor({ + state: "hidden", + timeout, + }); + } + if (opts.selector) { + const selector = String(opts.selector).trim(); + if (selector) { + await page + .locator(selector) + .first() + .waitFor({ state: "visible", timeout }); + } + } + if (opts.url) { + const url = String(opts.url).trim(); + if (url) { + await page.waitForURL(url, { timeout }); + } + } + if (opts.loadState) { + await page.waitForLoadState(opts.loadState, { timeout }); + } + if (opts.fn) { + const fn = String(opts.fn).trim(); + if (fn) { + await page.waitForFunction(fn, { timeout }); + } } } diff --git a/src/browser/routes/agent.ts b/src/browser/routes/agent.ts index b99fad919..c9b737e6a 100644 --- a/src/browser/routes/agent.ts +++ b/src/browser/routes/agent.ts @@ -1,3 +1,5 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; import path from "node:path"; import type express from "express"; @@ -141,7 +143,7 @@ export function registerBrowserAgentRoutes( const body = readBody(req); const kind = toStringOrEmpty(body.kind) as ActKind; const targetId = toStringOrEmpty(body.targetId) || undefined; - if (Object.hasOwn(body, "selector")) { + if (Object.hasOwn(body, "selector") && kind !== "wait") { return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE); } @@ -172,6 +174,7 @@ export function registerBrowserAgentRoutes( const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); const doubleClick = toBoolean(body.doubleClick) ?? false; + const timeoutMs = toNumber(body.timeoutMs); const buttonRaw = toStringOrEmpty(body.button) || ""; const button = buttonRaw ? parseClickButton(buttonRaw) : undefined; if (buttonRaw && !button) @@ -205,6 +208,7 @@ export function registerBrowserAgentRoutes( }; if (button) clickRequest.button = button; if (modifiers) clickRequest.modifiers = modifiers; + if (timeoutMs) clickRequest.timeoutMs = timeoutMs; await pw.clickViaPlaywright(clickRequest); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } @@ -216,6 +220,7 @@ export function registerBrowserAgentRoutes( const text = body.text; const submit = toBoolean(body.submit) ?? false; const slowly = toBoolean(body.slowly) ?? false; + const timeoutMs = toNumber(body.timeoutMs); const typeRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, @@ -224,23 +229,32 @@ export function registerBrowserAgentRoutes( submit, slowly, }; + if (timeoutMs) typeRequest.timeoutMs = timeoutMs; await pw.typeViaPlaywright(typeRequest); return res.json({ ok: true, targetId: tab.targetId }); } case "press": { const key = toStringOrEmpty(body.key); if (!key) return jsonError(res, 400, "key is required"); + const delayMs = toNumber(body.delayMs); await pw.pressKeyViaPlaywright({ cdpUrl, targetId: tab.targetId, key, + delayMs: delayMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "hover": { const ref = toStringOrEmpty(body.ref); if (!ref) return jsonError(res, 400, "ref is required"); - await pw.hoverViaPlaywright({ cdpUrl, targetId: tab.targetId, ref }); + const timeoutMs = toNumber(body.timeoutMs); + await pw.hoverViaPlaywright({ + cdpUrl, + targetId: tab.targetId, + ref, + timeoutMs: timeoutMs ?? undefined, + }); return res.json({ ok: true, targetId: tab.targetId }); } case "drag": { @@ -248,11 +262,13 @@ export function registerBrowserAgentRoutes( const endRef = toStringOrEmpty(body.endRef); if (!startRef || !endRef) return jsonError(res, 400, "startRef and endRef are required"); + const timeoutMs = toNumber(body.timeoutMs); await pw.dragViaPlaywright({ cdpUrl, targetId: tab.targetId, startRef, endRef, + timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -261,11 +277,13 @@ export function registerBrowserAgentRoutes( const values = toStringArray(body.values); if (!ref || !values?.length) return jsonError(res, 400, "ref and values are required"); + const timeoutMs = toNumber(body.timeoutMs); await pw.selectOptionViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, values, + timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -290,10 +308,12 @@ export function registerBrowserAgentRoutes( }) .filter((field): field is BrowserFormField => field !== null); if (!fields.length) return jsonError(res, 400, "fields are required"); + const timeoutMs = toNumber(body.timeoutMs); await pw.fillFormViaPlaywright({ cdpUrl, targetId: tab.targetId, fields, + timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -314,12 +334,43 @@ export function registerBrowserAgentRoutes( const timeMs = toNumber(body.timeMs); const text = toStringOrEmpty(body.text) || undefined; const textGone = toStringOrEmpty(body.textGone) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; + const url = toStringOrEmpty(body.url) || undefined; + const loadStateRaw = toStringOrEmpty(body.loadState); + const loadState = + loadStateRaw === "load" || + loadStateRaw === "domcontentloaded" || + loadStateRaw === "networkidle" + ? (loadStateRaw as "load" | "domcontentloaded" | "networkidle") + : undefined; + const fn = toStringOrEmpty(body.fn) || undefined; + const timeoutMs = toNumber(body.timeoutMs) ?? undefined; + if ( + timeMs === undefined && + !text && + !textGone && + !selector && + !url && + !loadState && + !fn + ) { + return jsonError( + res, + 400, + "wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn", + ); + } await pw.waitForViaPlaywright({ cdpUrl, targetId: tab.targetId, timeMs, text, textGone, + selector, + url, + loadState, + fn, + timeoutMs, }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -452,6 +503,494 @@ export function registerBrowserAgentRoutes( } }); + app.get("/errors", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const targetId = + typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; + const clear = toBoolean(req.query.clear) ?? false; + + try { + const tab = await profileCtx.ensureTabAvailable(targetId || undefined); + const pw = await requirePwAi(res, "page errors"); + if (!pw) return; + const result = await pw.getPageErrorsViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + clear, + }); + res.json({ ok: true, targetId: tab.targetId, ...result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.get("/requests", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const targetId = + typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; + const filter = typeof req.query.filter === "string" ? req.query.filter : ""; + const clear = toBoolean(req.query.clear) ?? false; + + try { + const tab = await profileCtx.ensureTabAvailable(targetId || undefined); + const pw = await requirePwAi(res, "network requests"); + if (!pw) return; + const result = await pw.getNetworkRequestsViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + filter: filter.trim() || undefined, + clear, + }); + res.json({ ok: true, targetId: tab.targetId, ...result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/trace/start", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const screenshots = toBoolean(body.screenshots) ?? undefined; + const snapshots = toBoolean(body.snapshots) ?? undefined; + const sources = toBoolean(body.sources) ?? undefined; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "trace start"); + if (!pw) return; + await pw.traceStartViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + screenshots, + snapshots, + sources, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/trace/stop", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const out = toStringOrEmpty(body.path) || ""; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "trace stop"); + if (!pw) return; + const id = crypto.randomUUID(); + const dir = "/tmp/clawdbot"; + await fs.mkdir(dir, { recursive: true }); + const tracePath = out.trim() || path.join(dir, `browser-trace-${id}.zip`); + await pw.traceStopViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + path: tracePath, + }); + res.json({ + ok: true, + targetId: tab.targetId, + path: path.resolve(tracePath), + }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/highlight", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const ref = toStringOrEmpty(body.ref); + if (!ref) return jsonError(res, 400, "ref is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "highlight"); + if (!pw) return; + await pw.highlightViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + ref, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.get("/cookies", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const targetId = + typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; + try { + const tab = await profileCtx.ensureTabAvailable(targetId || undefined); + const pw = await requirePwAi(res, "cookies"); + if (!pw) return; + const result = await pw.cookiesGetViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + }); + res.json({ ok: true, targetId: tab.targetId, ...result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/cookies/set", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const cookie = + body.cookie && + typeof body.cookie === "object" && + !Array.isArray(body.cookie) + ? (body.cookie as Record) + : null; + if (!cookie) return jsonError(res, 400, "cookie is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "cookies set"); + if (!pw) return; + await pw.cookiesSetViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + cookie: { + name: toStringOrEmpty(cookie.name), + value: toStringOrEmpty(cookie.value), + url: toStringOrEmpty(cookie.url) || undefined, + domain: toStringOrEmpty(cookie.domain) || undefined, + path: toStringOrEmpty(cookie.path) || undefined, + expires: toNumber(cookie.expires) ?? undefined, + httpOnly: toBoolean(cookie.httpOnly) ?? undefined, + secure: toBoolean(cookie.secure) ?? undefined, + sameSite: + cookie.sameSite === "Lax" || + cookie.sameSite === "None" || + cookie.sameSite === "Strict" + ? (cookie.sameSite as "Lax" | "None" | "Strict") + : undefined, + }, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/cookies/clear", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "cookies clear"); + if (!pw) return; + await pw.cookiesClearViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.get("/storage/:kind", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const kind = toStringOrEmpty(req.params.kind); + if (kind !== "local" && kind !== "session") + return jsonError(res, 400, "kind must be local|session"); + const targetId = + typeof req.query.targetId === "string" ? req.query.targetId.trim() : ""; + const key = typeof req.query.key === "string" ? req.query.key : ""; + try { + const tab = await profileCtx.ensureTabAvailable(targetId || undefined); + const pw = await requirePwAi(res, "storage get"); + if (!pw) return; + const result = await pw.storageGetViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + kind, + key: key.trim() || undefined, + }); + res.json({ ok: true, targetId: tab.targetId, ...result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/storage/:kind/set", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const kind = toStringOrEmpty(req.params.kind); + if (kind !== "local" && kind !== "session") + return jsonError(res, 400, "kind must be local|session"); + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const key = toStringOrEmpty(body.key); + if (!key) return jsonError(res, 400, "key is required"); + const value = typeof body.value === "string" ? body.value : ""; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "storage set"); + if (!pw) return; + await pw.storageSetViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + kind, + key, + value, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/storage/:kind/clear", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const kind = toStringOrEmpty(req.params.kind); + if (kind !== "local" && kind !== "session") + return jsonError(res, 400, "kind must be local|session"); + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "storage clear"); + if (!pw) return; + await pw.storageClearViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + kind, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/offline", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const offline = toBoolean(body.offline); + if (offline === undefined) + return jsonError(res, 400, "offline is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "offline"); + if (!pw) return; + await pw.setOfflineViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + offline, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/headers", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const headers = + body.headers && + typeof body.headers === "object" && + !Array.isArray(body.headers) + ? (body.headers as Record) + : null; + if (!headers) return jsonError(res, 400, "headers is required"); + const parsed: Record = {}; + for (const [k, v] of Object.entries(headers)) { + if (typeof v === "string") parsed[k] = v; + } + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "headers"); + if (!pw) return; + await pw.setExtraHTTPHeadersViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + headers: parsed, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/credentials", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const clear = toBoolean(body.clear) ?? false; + const username = toStringOrEmpty(body.username) || undefined; + const password = + typeof body.password === "string" ? body.password : undefined; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "http credentials"); + if (!pw) return; + await pw.setHttpCredentialsViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + username, + password, + clear, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/geolocation", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const clear = toBoolean(body.clear) ?? false; + const latitude = toNumber(body.latitude); + const longitude = toNumber(body.longitude); + const accuracy = toNumber(body.accuracy) ?? undefined; + const origin = toStringOrEmpty(body.origin) || undefined; + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "geolocation"); + if (!pw) return; + await pw.setGeolocationViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + latitude, + longitude, + accuracy, + origin, + clear, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/media", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const schemeRaw = toStringOrEmpty(body.colorScheme); + const colorScheme = + schemeRaw === "dark" || + schemeRaw === "light" || + schemeRaw === "no-preference" + ? (schemeRaw as "dark" | "light" | "no-preference") + : schemeRaw === "none" + ? null + : undefined; + if (colorScheme === undefined) + return jsonError( + res, + 400, + "colorScheme must be dark|light|no-preference|none", + ); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "media emulation"); + if (!pw) return; + await pw.emulateMediaViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + colorScheme, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/timezone", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const timezoneId = toStringOrEmpty(body.timezoneId); + if (!timezoneId) return jsonError(res, 400, "timezoneId is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "timezone"); + if (!pw) return; + await pw.setTimezoneViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + timezoneId, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/locale", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const locale = toStringOrEmpty(body.locale); + if (!locale) return jsonError(res, 400, "locale is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "locale"); + if (!pw) return; + await pw.setLocaleViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + locale, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/set/device", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const name = toStringOrEmpty(body.name); + if (!name) return jsonError(res, 400, "name is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "device emulation"); + if (!pw) return; + await pw.setDeviceViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + name, + }); + res.json({ ok: true, targetId: tab.targetId }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + app.post("/pdf", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) return; @@ -577,6 +1116,7 @@ export function registerBrowserAgentRoutes( const compact = toBoolean(req.query.compact); const depth = toNumber(req.query.depth); const selector = toStringOrEmpty(req.query.selector); + const frameSelector = toStringOrEmpty(req.query.frame); try { const tab = await profileCtx.ensureTabAvailable(targetId || undefined); @@ -587,13 +1127,15 @@ export function registerBrowserAgentRoutes( interactive === true || compact === true || depth !== undefined || - Boolean(selector.trim()); + Boolean(selector.trim()) || + Boolean(frameSelector.trim()); const snap = wantsRoleSnapshot ? await pw.snapshotRoleViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, selector: selector.trim() || undefined, + frameSelector: frameSelector.trim() || undefined, options: { interactive: interactive ?? undefined, compact: compact ?? undefined, @@ -613,6 +1155,7 @@ export function registerBrowserAgentRoutes( cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, selector: selector.trim() || undefined, + frameSelector: frameSelector.trim() || undefined, options: { interactive: interactive ?? undefined, compact: compact ?? undefined,