From 6fc30962d6d0fa194eaad9f52656892240969b0e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 20 Dec 2025 02:53:22 +0000 Subject: [PATCH] refactor(browser): prune browser automation surface --- src/browser/client-actions-core.ts | 57 ++++-------- src/browser/client-actions-observe.ts | 53 ----------- src/browser/client.test.ts | 3 +- src/browser/client.ts | 34 ------- src/browser/pw-ai.ts | 9 +- src/browser/pw-session.ts | 13 +-- src/browser/pw-tools-core.ts | 97 +++++++++----------- src/browser/pw-tools-observe.ts | 41 --------- src/browser/routes/actions-core.test.ts | 28 ++---- src/browser/routes/actions-core.ts | 38 ++------ src/browser/routes/actions-extra.test.ts | 37 -------- src/browser/routes/actions-extra.ts | 83 +---------------- src/browser/routes/actions.ts | 30 +------ src/browser/routes/inspect.ts | 40 --------- src/browser/server.test.ts | 55 +----------- src/cli/browser-cli-actions-input.ts | 90 +------------------ src/cli/browser-cli-actions-observe.ts | 109 ----------------------- src/cli/browser-cli-examples.ts | 6 -- src/cli/browser-cli-inspect.ts | 64 ------------- 19 files changed, 85 insertions(+), 802 deletions(-) diff --git a/src/browser/client-actions-core.ts b/src/browser/client-actions-core.ts index ed1b1bff0..35701f3d9 100644 --- a/src/browser/client-actions-core.ts +++ b/src/browser/client-actions-core.ts @@ -1,5 +1,8 @@ import type { ScreenshotResult } from "./client.js"; -import type { BrowserActionTabResult } from "./client-actions-types.js"; +import type { + BrowserActionOk, + BrowserActionTabResult, +} from "./client-actions-types.js"; import { fetchBrowserJson } from "./client-fetch.js"; export async function browserNavigate( @@ -14,18 +17,6 @@ export async function browserNavigate( }); } -export async function browserBack( - baseUrl: string, - opts: { targetId?: string } = {}, -): Promise { - return await fetchBrowserJson(`${baseUrl}/back`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ targetId: opts.targetId }), - timeoutMs: 20000, - }); -} - export async function browserResize( baseUrl: string, opts: { width: number; height: number; targetId?: string }, @@ -185,20 +176,17 @@ export async function browserFillForm( export async function browserHandleDialog( baseUrl: string, opts: { accept: boolean; promptText?: string; targetId?: string }, -): Promise<{ ok: true; message: string; type: string }> { - return await fetchBrowserJson<{ ok: true; message: string; type: string }>( - `${baseUrl}/dialog`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - accept: opts.accept, - promptText: opts.promptText, - targetId: opts.targetId, - }), - timeoutMs: 20000, - }, - ); +): Promise { + return await fetchBrowserJson(`${baseUrl}/dialog`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + accept: opts.accept, + promptText: opts.promptText, + targetId: opts.targetId, + }), + timeoutMs: 20000, + }); } export async function browserWaitFor( @@ -242,21 +230,6 @@ export async function browserEvaluate( ); } -export async function browserRunCode( - baseUrl: string, - opts: { code: string; targetId?: string }, -): Promise<{ ok: true; result: unknown }> { - return await fetchBrowserJson<{ ok: true; result: unknown }>( - `${baseUrl}/run`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ code: opts.code, targetId: opts.targetId }), - timeoutMs: 20000, - }, - ); -} - export async function browserScreenshotAction( baseUrl: string, opts: { diff --git a/src/browser/client-actions-observe.ts b/src/browser/client-actions-observe.ts index 8ab03c927..4b648d3f6 100644 --- a/src/browser/client-actions-observe.ts +++ b/src/browser/client-actions-observe.ts @@ -92,56 +92,3 @@ export async function browserVerifyValue( timeoutMs: 20000, }); } - -export async function browserMouseMove( - baseUrl: string, - opts: { x: number; y: number; targetId?: string }, -): Promise { - return await fetchBrowserJson(`${baseUrl}/mouse/move`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ x: opts.x, y: opts.y, targetId: opts.targetId }), - timeoutMs: 20000, - }); -} - -export async function browserMouseClick( - baseUrl: string, - opts: { x: number; y: number; button?: string; targetId?: string }, -): Promise { - return await fetchBrowserJson(`${baseUrl}/mouse/click`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - x: opts.x, - y: opts.y, - button: opts.button, - targetId: opts.targetId, - }), - timeoutMs: 20000, - }); -} - -export async function browserMouseDrag( - baseUrl: string, - opts: { - startX: number; - startY: number; - endX: number; - endY: number; - targetId?: string; - }, -): Promise { - return await fetchBrowserJson(`${baseUrl}/mouse/drag`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - startX: opts.startX, - startY: opts.startY, - endX: opts.endX, - endY: opts.endY, - targetId: opts.targetId, - }), - timeoutMs: 20000, - }); -} diff --git a/src/browser/client.test.ts b/src/browser/client.test.ts index fbcf1fd84..544c80129 100644 --- a/src/browser/client.test.ts +++ b/src/browser/client.test.ts @@ -3,7 +3,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { browserClickRef, browserDom, - browserEval, browserOpenTab, browserQuery, browserScreenshot, @@ -50,7 +49,7 @@ describe("browser client", () => { ); await expect( - browserEval("http://127.0.0.1:18791", { js: "1+1" }), + browserDom("http://127.0.0.1:18791", { format: "text", maxChars: 1 }), ).rejects.toThrow(/409: conflict/i); }); diff --git a/src/browser/client.ts b/src/browser/client.ts index 4433d5a3c..59d921f80 100644 --- a/src/browser/client.ts +++ b/src/browser/client.ts @@ -29,20 +29,6 @@ export type ScreenshotResult = { url: string; }; -export type EvalResult = { - ok: true; - targetId: string; - url: string; - result: { - type: string; - subtype?: string; - value?: unknown; - description?: string; - unserializableValue?: string; - preview?: unknown; - }; -}; - export type QueryResult = { ok: true; targetId: string; @@ -201,26 +187,6 @@ export async function browserScreenshot( ); } -export async function browserEval( - baseUrl: string, - opts: { - js: string; - targetId?: string; - awaitPromise?: boolean; - }, -): Promise { - return await fetchBrowserJson(`${baseUrl}/eval`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - js: opts.js, - targetId: opts.targetId, - await: Boolean(opts.awaitPromise), - }), - timeoutMs: 15000, - }); -} - export async function browserQuery( baseUrl: string, opts: { diff --git a/src/browser/pw-ai.ts b/src/browser/pw-ai.ts index 78efe92c5..0a1a296b2 100644 --- a/src/browser/pw-ai.ts +++ b/src/browser/pw-ai.ts @@ -8,21 +8,19 @@ export { } from "./pw-session.js"; export { + armDialogViaPlaywright, + armFileUploadViaPlaywright, clickRefViaPlaywright, clickViaPlaywright, closePageViaPlaywright, dragViaPlaywright, evaluateViaPlaywright, - fileUploadViaPlaywright, fillFormViaPlaywright, - handleDialogViaPlaywright, hoverViaPlaywright, - navigateBackViaPlaywright, navigateViaPlaywright, pdfViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, - runCodeViaPlaywright, selectOptionViaPlaywright, snapshotAiViaPlaywright, takeScreenshotViaPlaywright, @@ -32,9 +30,6 @@ export { export { getConsoleMessagesViaPlaywright, - mouseClickViaPlaywright, - mouseDragViaPlaywright, - mouseMoveViaPlaywright, verifyElementVisibleViaPlaywright, verifyListVisibleViaPlaywright, verifyTextVisibleViaPlaywright, diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index 52a3fe204..610f200ad 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -13,15 +13,6 @@ export type BrowserConsoleMessage = { location?: { url?: string; lineNumber?: number; columnNumber?: number }; }; -export type BrowserNetworkRequest = { - requestId?: string; - url: string; - method: string; - status?: number; - resourceType?: string; - timestamp?: string; -}; - type SnapshotForAIResult = { full: string; incremental?: string }; type SnapshotForAIOptions = { timeout?: number; track?: string }; @@ -44,6 +35,8 @@ type ConnectedBrowser = { type PageState = { console: BrowserConsoleMessage[]; + armIdUpload: number; + armIdDialog: number; }; const pageStates = new WeakMap(); @@ -65,6 +58,8 @@ export function ensurePageState(page: Page): PageState { const state: PageState = { console: [], + armIdUpload: 0, + armIdDialog: 0, }; pageStates.set(page, state); diff --git a/src/browser/pw-tools-core.ts b/src/browser/pw-tools-core.ts index f24cc9293..15fb2085b 100644 --- a/src/browser/pw-tools-core.ts +++ b/src/browser/pw-tools-core.ts @@ -1,5 +1,3 @@ -import type { Page } from "playwright-core"; - import { ensurePageState, getPageForTargetId, @@ -7,6 +5,9 @@ import { type WithSnapshotForAI, } from "./pw-session.js"; +let nextUploadArmId = 0; +let nextDialogArmId = 0; + export async function snapshotAiViaPlaywright(opts: { cdpPort: number; targetId?: string; @@ -221,44 +222,63 @@ export async function evaluateViaPlaywright(opts: { }, fnText); } -export async function fileUploadViaPlaywright(opts: { +export async function armFileUploadViaPlaywright(opts: { cdpPort: number; targetId?: string; paths?: string[]; timeoutMs?: number; }): Promise { const page = await getPageForTargetId(opts); - ensurePageState(page); + const state = ensurePageState(page); const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000)); - const fileChooser = await page.waitForEvent("filechooser", { timeout }); - if (!opts.paths?.length) { - // Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead. - try { - await page.keyboard.press("Escape"); - } catch { - // Best-effort. - } - return; - } - await fileChooser.setFiles(opts.paths); + + state.armIdUpload = nextUploadArmId += 1; + const armId = state.armIdUpload; + + void page + .waitForEvent("filechooser", { timeout }) + .then(async (fileChooser) => { + if (state.armIdUpload !== armId) return; + if (!opts.paths?.length) { + // Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead. + try { + await page.keyboard.press("Escape"); + } catch { + // Best-effort. + } + return; + } + await fileChooser.setFiles(opts.paths); + }) + .catch(() => { + // Ignore timeouts; the chooser may never appear. + }); } -export async function handleDialogViaPlaywright(opts: { +export async function armDialogViaPlaywright(opts: { cdpPort: number; targetId?: string; accept: boolean; promptText?: string; timeoutMs?: number; -}): Promise<{ message: string; type: string }> { +}): Promise { const page = await getPageForTargetId(opts); - ensurePageState(page); + const state = ensurePageState(page); const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000)); - const dialog = await page.waitForEvent("dialog", { timeout }); - const message = dialog.message(); - const type = dialog.type(); - if (opts.accept) await dialog.accept(opts.promptText); - else await dialog.dismiss(); - return { message, type }; + + state.armIdDialog = nextDialogArmId += 1; + const armId = state.armIdDialog; + + void page + .waitForEvent("dialog", { timeout }) + .then(async (dialog) => { + if (state.armIdDialog !== armId) return; + if (opts.accept) await dialog.accept(opts.promptText); + else await dialog.dismiss(); + }) + .catch(() => { + // Ignore timeouts; the dialog may never appear. + }); } export async function navigateViaPlaywright(opts: { @@ -277,19 +297,6 @@ export async function navigateViaPlaywright(opts: { return { url: page.url() }; } -export async function navigateBackViaPlaywright(opts: { - cdpPort: number; - targetId?: string; - timeoutMs?: number; -}): Promise<{ url: string }> { - const page = await getPageForTargetId(opts); - ensurePageState(page); - await page.goBack({ - timeout: Math.max(1000, Math.min(120_000, opts.timeoutMs ?? 20_000)), - }); - return { url: page.url() }; -} - export async function waitForViaPlaywright(opts: { cdpPort: number; targetId?: string; @@ -323,22 +330,6 @@ export async function waitForViaPlaywright(opts: { } } -export async function runCodeViaPlaywright(opts: { - cdpPort: number; - targetId?: string; - code: string; -}): Promise { - const code = String(opts.code ?? "").trim(); - if (!code) throw new Error("code is required"); - const page = await getPageForTargetId(opts); - ensurePageState(page); - const fn = new Function(`return (${code});`)() as - | ((page: Page) => unknown) - | undefined; - if (typeof fn !== "function") throw new Error("code is not a function"); - return await fn(page); -} - export async function takeScreenshotViaPlaywright(opts: { cdpPort: number; targetId?: string; diff --git a/src/browser/pw-tools-observe.ts b/src/browser/pw-tools-observe.ts index 504992566..0301cfdb8 100644 --- a/src/browser/pw-tools-observe.ts +++ b/src/browser/pw-tools-observe.ts @@ -33,47 +33,6 @@ export async function getConsoleMessagesViaPlaywright(opts: { return state.console.filter((msg) => consolePriority(msg.type) >= min); } -export async function mouseMoveViaPlaywright(opts: { - cdpPort: number; - targetId?: string; - x: number; - y: number; -}): Promise { - const page = await getPageForTargetId(opts); - ensurePageState(page); - await page.mouse.move(opts.x, opts.y); -} - -export async function mouseClickViaPlaywright(opts: { - cdpPort: number; - targetId?: string; - x: number; - y: number; - button?: "left" | "right" | "middle"; -}): Promise { - const page = await getPageForTargetId(opts); - ensurePageState(page); - await page.mouse.click(opts.x, opts.y, { - button: opts.button, - }); -} - -export async function mouseDragViaPlaywright(opts: { - cdpPort: number; - targetId?: string; - startX: number; - startY: number; - endX: number; - endY: number; -}): Promise { - const page = await getPageForTargetId(opts); - ensurePageState(page); - await page.mouse.move(opts.startX, opts.startY); - await page.mouse.down(); - await page.mouse.move(opts.endX, opts.endY); - await page.mouse.up(); -} - export async function verifyElementVisibleViaPlaywright(opts: { cdpPort: number; targetId?: string; diff --git a/src/browser/routes/actions-core.test.ts b/src/browser/routes/actions-core.test.ts index 678f51186..dc77d7f70 100644 --- a/src/browser/routes/actions-core.test.ts +++ b/src/browser/routes/actions-core.test.ts @@ -3,23 +3,19 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { BrowserRouteContext } from "../server-context.js"; const pw = vi.hoisted(() => ({ + armDialogViaPlaywright: vi.fn().mockResolvedValue(undefined), + armFileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined), clickViaPlaywright: vi.fn().mockResolvedValue(undefined), closePageViaPlaywright: vi.fn().mockResolvedValue(undefined), dragViaPlaywright: vi.fn().mockResolvedValue(undefined), evaluateViaPlaywright: vi.fn().mockResolvedValue("result"), - fileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined), fillFormViaPlaywright: vi.fn().mockResolvedValue(undefined), - handleDialogViaPlaywright: vi - .fn() - .mockResolvedValue({ message: "ok", type: "alert" }), hoverViaPlaywright: vi.fn().mockResolvedValue(undefined), - navigateBackViaPlaywright: vi.fn().mockResolvedValue({ url: "about:blank" }), navigateViaPlaywright: vi .fn() .mockResolvedValue({ url: "https://example.com" }), pressKeyViaPlaywright: vi.fn().mockResolvedValue(undefined), resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined), - runCodeViaPlaywright: vi.fn().mockResolvedValue("ok"), selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined), typeViaPlaywright: vi.fn().mockResolvedValue(undefined), waitForViaPlaywright: vi.fn().mockResolvedValue(undefined), @@ -127,14 +123,14 @@ describe("handleBrowserActionCore", () => { { action: "dialog" as const, args: { accept: true, promptText: "ok" }, - fn: pw.handleDialogViaPlaywright, + fn: pw.armDialogViaPlaywright, expectArgs: { cdpPort: 18792, targetId: "tab1", accept: true, promptText: "ok", }, - expectBody: { ok: true, message: "ok", type: "alert" }, + expectBody: { ok: true }, }, { action: "evaluate" as const, @@ -151,7 +147,7 @@ describe("handleBrowserActionCore", () => { { action: "upload" as const, args: { paths: ["/tmp/file.txt"] }, - fn: pw.fileUploadViaPlaywright, + fn: pw.armFileUploadViaPlaywright, expectArgs: { cdpPort: 18792, targetId: "tab1", @@ -202,20 +198,6 @@ describe("handleBrowserActionCore", () => { }, expectBody: { ok: true, targetId: "tab1", url: baseTab.url }, }, - { - action: "back" as const, - args: {}, - fn: pw.navigateBackViaPlaywright, - expectArgs: { cdpPort: 18792, targetId: "tab1" }, - expectBody: { ok: true, targetId: "tab1", url: "about:blank" }, - }, - { - action: "run" as const, - args: { code: "return 1" }, - fn: pw.runCodeViaPlaywright, - expectArgs: { cdpPort: 18792, targetId: "tab1", code: "return 1" }, - expectBody: { ok: true, result: "ok" }, - }, { action: "click" as const, args: { diff --git a/src/browser/routes/actions-core.ts b/src/browser/routes/actions-core.ts index e975facde..dfbab56b4 100644 --- a/src/browser/routes/actions-core.ts +++ b/src/browser/routes/actions-core.ts @@ -1,19 +1,17 @@ import type express from "express"; import { + armDialogViaPlaywright, + armFileUploadViaPlaywright, clickViaPlaywright, closePageViaPlaywright, dragViaPlaywright, evaluateViaPlaywright, - fileUploadViaPlaywright, fillFormViaPlaywright, - handleDialogViaPlaywright, hoverViaPlaywright, - navigateBackViaPlaywright, navigateViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, - runCodeViaPlaywright, selectOptionViaPlaywright, typeViaPlaywright, waitForViaPlaywright, @@ -51,7 +49,6 @@ function normalizeModifiers(value: unknown): KeyboardModifier[] | undefined { } export type BrowserActionCore = - | "back" | "click" | "close" | "dialog" @@ -62,7 +59,6 @@ export type BrowserActionCore = | "navigate" | "press" | "resize" - | "run" | "select" | "type" | "upload" @@ -115,13 +111,13 @@ export async function handleBrowserActionCore( } const promptText = toStringOrEmpty(args.promptText) || undefined; const tab = await ctx.ensureTabAvailable(target); - const result = await handleDialogViaPlaywright({ + await armDialogViaPlaywright({ cdpPort, targetId: tab.targetId, accept, promptText, }); - res.json({ ok: true, ...result }); + res.json({ ok: true }); return true; } case "evaluate": { @@ -144,7 +140,7 @@ export async function handleBrowserActionCore( case "upload": { const paths = toStringArray(args.paths) ?? []; const tab = await ctx.ensureTabAvailable(target); - await fileUploadViaPlaywright({ + await armFileUploadViaPlaywright({ cdpPort, targetId: tab.targetId, paths: paths.length ? paths : undefined, @@ -220,30 +216,6 @@ export async function handleBrowserActionCore( res.json({ ok: true, targetId: tab.targetId, ...result }); return true; } - case "back": { - const tab = await ctx.ensureTabAvailable(target); - const result = await navigateBackViaPlaywright({ - cdpPort, - targetId: tab.targetId, - }); - res.json({ ok: true, targetId: tab.targetId, ...result }); - return true; - } - case "run": { - const code = toStringOrEmpty(args.code); - if (!code) { - jsonError(res, 400, "code is required"); - return true; - } - const tab = await ctx.ensureTabAvailable(target); - const result = await runCodeViaPlaywright({ - cdpPort, - targetId: tab.targetId, - code, - }); - res.json({ ok: true, result }); - return true; - } case "click": { const ref = toStringOrEmpty(args.ref); if (!ref) { diff --git a/src/browser/routes/actions-extra.test.ts b/src/browser/routes/actions-extra.test.ts index 849c7e85a..896f610cf 100644 --- a/src/browser/routes/actions-extra.test.ts +++ b/src/browser/routes/actions-extra.test.ts @@ -4,9 +4,6 @@ import type { BrowserRouteContext } from "../server-context.js"; const pw = vi.hoisted(() => ({ getConsoleMessagesViaPlaywright: vi.fn().mockResolvedValue([]), - mouseClickViaPlaywright: vi.fn().mockResolvedValue(undefined), - mouseDragViaPlaywright: vi.fn().mockResolvedValue(undefined), - mouseMoveViaPlaywright: vi.fn().mockResolvedValue(undefined), pdfViaPlaywright: vi.fn().mockResolvedValue({ buffer: Buffer.from("pdf") }), verifyElementVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined), verifyListVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined), @@ -149,40 +146,6 @@ describe("handleBrowserActionExtra", () => { }, expectBody: { ok: true }, }, - { - action: "mouseMove" as const, - args: { x: 10, y: 20 }, - fn: pw.mouseMoveViaPlaywright, - expectArgs: { cdpPort: 18792, targetId: "tab1", x: 10, y: 20 }, - expectBody: { ok: true }, - }, - { - action: "mouseClick" as const, - args: { x: 1, y: 2, button: "right" }, - fn: pw.mouseClickViaPlaywright, - expectArgs: { - cdpPort: 18792, - targetId: "tab1", - x: 1, - y: 2, - button: "right", - }, - expectBody: { ok: true }, - }, - { - action: "mouseDrag" as const, - args: { startX: 1, startY: 2, endX: 3, endY: 4 }, - fn: pw.mouseDragViaPlaywright, - expectArgs: { - cdpPort: 18792, - targetId: "tab1", - startX: 1, - startY: 2, - endX: 3, - endY: 4, - }, - expectBody: { ok: true }, - }, ]; for (const item of cases) { diff --git a/src/browser/routes/actions-extra.ts b/src/browser/routes/actions-extra.ts index ba1dedb92..db581f880 100644 --- a/src/browser/routes/actions-extra.ts +++ b/src/browser/routes/actions-extra.ts @@ -5,9 +5,6 @@ import type express from "express"; import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js"; import { getConsoleMessagesViaPlaywright, - mouseClickViaPlaywright, - mouseDragViaPlaywright, - mouseMoveViaPlaywright, pdfViaPlaywright, verifyElementVisibleViaPlaywright, verifyListVisibleViaPlaywright, @@ -15,26 +12,10 @@ import { verifyValueViaPlaywright, } from "../pw-ai.js"; import type { BrowserRouteContext } from "../server-context.js"; -import { - jsonError, - toNumber, - toStringArray, - toStringOrEmpty, -} from "./utils.js"; - -type MouseButton = "left" | "right" | "middle"; - -function normalizeMouseButton(value: unknown): MouseButton | undefined { - const raw = toStringOrEmpty(value); - if (raw === "left" || raw === "right" || raw === "middle") return raw; - return undefined; -} +import { jsonError, toStringArray, toStringOrEmpty } from "./utils.js"; export type BrowserActionExtra = | "console" - | "mouseClick" - | "mouseDrag" - | "mouseMove" | "pdf" | "verifyElement" | "verifyList" @@ -157,68 +138,6 @@ export async function handleBrowserActionExtra( res.json({ ok: true }); return true; } - case "mouseMove": { - const x = toNumber(args.x); - const y = toNumber(args.y); - if (x === undefined || y === undefined) { - jsonError(res, 400, "x and y are required"); - return true; - } - const tab = await ctx.ensureTabAvailable(target); - await mouseMoveViaPlaywright({ - cdpPort, - targetId: tab.targetId, - x, - y, - }); - res.json({ ok: true }); - return true; - } - case "mouseClick": { - const x = toNumber(args.x); - const y = toNumber(args.y); - if (x === undefined || y === undefined) { - jsonError(res, 400, "x and y are required"); - return true; - } - const button = normalizeMouseButton(args.button); - const tab = await ctx.ensureTabAvailable(target); - await mouseClickViaPlaywright({ - cdpPort, - targetId: tab.targetId, - x, - y, - button, - }); - res.json({ ok: true }); - return true; - } - case "mouseDrag": { - const startX = toNumber(args.startX); - const startY = toNumber(args.startY); - const endX = toNumber(args.endX); - const endY = toNumber(args.endY); - if ( - startX === undefined || - startY === undefined || - endX === undefined || - endY === undefined - ) { - jsonError(res, 400, "startX, startY, endX, endY are required"); - return true; - } - const tab = await ctx.ensureTabAvailable(target); - await mouseDragViaPlaywright({ - cdpPort, - targetId: tab.targetId, - startX, - startY, - endX, - endY, - }); - res.json({ ok: true }); - return true; - } default: return false; } diff --git a/src/browser/routes/actions.ts b/src/browser/routes/actions.ts index c32ffcc7f..76633dd8c 100644 --- a/src/browser/routes/actions.ts +++ b/src/browser/routes/actions.ts @@ -79,12 +79,6 @@ export function registerBrowserActionRoutes( await runCoreAction(ctx, res, "navigate", body, targetId); }); - app.post("/back", async (req, res) => { - const body = readBody(req); - const targetId = readTargetId(body.targetId); - await runCoreAction(ctx, res, "back", body, targetId); - }); - app.post("/resize", async (req, res) => { const body = readBody(req); const targetId = readTargetId(body.targetId); @@ -163,12 +157,6 @@ export function registerBrowserActionRoutes( await runCoreAction(ctx, res, "evaluate", body, targetId); }); - app.post("/run", async (req, res) => { - const body = readBody(req); - const targetId = readTargetId(body.targetId); - await runCoreAction(ctx, res, "run", body, targetId); - }); - app.get("/console", async (req, res) => { const targetId = readTargetId(req.query.targetId); const level = toStringOrEmpty(req.query.level); @@ -206,21 +194,5 @@ export function registerBrowserActionRoutes( await runExtraAction(ctx, res, "verifyValue", body, targetId); }); - app.post("/mouse/move", async (req, res) => { - const body = readBody(req); - const targetId = readTargetId(body.targetId); - await runExtraAction(ctx, res, "mouseMove", body, targetId); - }); - - app.post("/mouse/click", async (req, res) => { - const body = readBody(req); - const targetId = readTargetId(body.targetId); - await runExtraAction(ctx, res, "mouseClick", body, targetId); - }); - - app.post("/mouse/drag", async (req, res) => { - const body = readBody(req); - const targetId = readTargetId(body.targetId); - await runExtraAction(ctx, res, "mouseDrag", body, targetId); - }); + // Intentionally no coordinate-based mouse actions (move/click/drag). } diff --git a/src/browser/routes/inspect.ts b/src/browser/routes/inspect.ts index d5c750ecf..bedbc0309 100644 --- a/src/browser/routes/inspect.ts +++ b/src/browser/routes/inspect.ts @@ -5,7 +5,6 @@ import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js"; import { captureScreenshot, captureScreenshotPng, - evaluateJavaScript, getDomText, querySelector, snapshotAria, @@ -124,45 +123,6 @@ export function registerBrowserInspectRoutes( } }); - app.post("/eval", async (req, res) => { - const js = toStringOrEmpty((req.body as { js?: unknown })?.js); - const targetId = toStringOrEmpty( - (req.body as { targetId?: unknown })?.targetId, - ); - const awaitPromise = Boolean((req.body as { await?: unknown })?.await); - - if (!js) return jsonError(res, 400, "js is required"); - - try { - const tab = await ctx.ensureTabAvailable(targetId || undefined); - const evaluated = await evaluateJavaScript({ - wsUrl: tab.wsUrl ?? "", - expression: js, - awaitPromise, - returnByValue: true, - }); - - if (evaluated.exceptionDetails) { - const msg = - evaluated.exceptionDetails.exception?.description || - evaluated.exceptionDetails.text || - "JavaScript evaluation failed"; - return jsonError(res, 400, msg); - } - - res.json({ - ok: true, - targetId: tab.targetId, - url: tab.url, - result: evaluated.result, - }); - } catch (err) { - const mapped = ctx.mapTabError(err); - if (mapped) return jsonError(res, mapped.status, mapped.message); - jsonError(res, 500, String(err)); - } - }); - app.get("/query", async (req, res) => { const selector = typeof req.query.selector === "string" ? req.query.selector.trim() : ""; diff --git a/src/browser/server.test.ts b/src/browser/server.test.ts index 3cf7a0d81..037884eee 100644 --- a/src/browser/server.test.ts +++ b/src/browser/server.test.ts @@ -62,22 +62,11 @@ vi.mock("./chrome.js", () => ({ }), })); -const evalCalls = vi.hoisted(() => [] as Array); -let evalThrows = false; vi.mock("./cdp.js", () => ({ createTargetViaCdp: vi.fn(async () => { if (createTargetId) return { targetId: createTargetId }; throw new Error("cdp disabled"); }), - evaluateJavaScript: vi.fn(async ({ expression }: { expression: string }) => { - evalCalls.push(expression); - if (evalThrows) { - return { - exceptionDetails: { text: "boom" }, - }; - } - return { result: { type: "string", value: "ok" } }; - }), getDomText: vi.fn(async () => ({ text: "" })), querySelector: vi.fn(async () => ({ matches: [{ index: 0, tag: "a" }] })), snapshotAria: vi.fn(async () => ({ @@ -97,28 +86,20 @@ vi.mock("./cdp.js", () => ({ })); vi.mock("./pw-ai.js", () => ({ + armDialogViaPlaywright: vi.fn(async () => {}), + armFileUploadViaPlaywright: vi.fn(async () => {}), clickRefViaPlaywright: vi.fn(async () => {}), clickViaPlaywright: vi.fn(async () => {}), closePageViaPlaywright: vi.fn(async () => {}), closePlaywrightBrowserConnection: vi.fn(async () => {}), evaluateViaPlaywright: vi.fn(async () => "ok"), - fileUploadViaPlaywright: vi.fn(async () => {}), fillFormViaPlaywright: vi.fn(async () => {}), getConsoleMessagesViaPlaywright: vi.fn(async () => []), - handleDialogViaPlaywright: vi.fn(async () => ({ - message: "ok", - type: "alert", - })), hoverViaPlaywright: vi.fn(async () => {}), - mouseClickViaPlaywright: vi.fn(async () => {}), - mouseDragViaPlaywright: vi.fn(async () => {}), - mouseMoveViaPlaywright: vi.fn(async () => {}), - navigateBackViaPlaywright: vi.fn(async () => ({ url: "about:blank" })), navigateViaPlaywright: vi.fn(async () => ({ url: "https://example.com" })), pdfViaPlaywright: vi.fn(async () => ({ buffer: Buffer.from("pdf") })), pressKeyViaPlaywright: vi.fn(async () => {}), resizeViewportViaPlaywright: vi.fn(async () => {}), - runCodeViaPlaywright: vi.fn(async () => "ok"), selectOptionViaPlaywright: vi.fn(async () => {}), snapshotAiViaPlaywright: vi.fn(async () => ({ snapshot: "ok" })), takeScreenshotViaPlaywright: vi.fn(async () => ({ @@ -179,6 +160,7 @@ describe("browser control server", () => { cfgAttachOnly = false; createTargetId = null; screenshotThrowsOnce = false; + testPort = await getFreePort(); // Minimal CDP JSON endpoints used by the server. @@ -288,30 +270,6 @@ describe("browser control server", () => { expect(focus.status).toBe(409); }); - it("maps JS exceptions to a 400 and returns results otherwise", async () => { - const { startBrowserControlServerFromConfig } = await import("./server.js"); - await startBrowserControlServerFromConfig(); - const base = `http://127.0.0.1:${testPort}`; - await realFetch(`${base}/start`, { method: "POST" }).then((r) => r.json()); - - evalThrows = true; - const bad = await realFetch(`${base}/eval`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ js: "throw 1" }), - }); - expect(bad.status).toBe(400); - - evalThrows = false; - const ok = (await realFetch(`${base}/eval`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ js: "1+1", await: true }), - }).then((r) => r.json())) as { ok: boolean; result?: unknown }; - expect(ok.ok).toBe(true); - expect(evalCalls.length).toBeGreaterThan(0); - }); - it("supports query/dom/snapshot/click/screenshot and stop", async () => { const { startBrowserControlServerFromConfig } = await import("./server.js"); await startBrowserControlServerFromConfig(); @@ -443,13 +401,6 @@ describe("browser control server", () => { const shotAmbiguous = await realFetch(`${base}/screenshot?targetId=abc`); expect(shotAmbiguous.status).toBe(409); - const evalMissing = await realFetch(`${base}/eval`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({}), - }); - expect(evalMissing.status).toBe(400); - const queryMissing = await realFetch(`${base}/query`); expect(queryMissing.status).toBe(400); diff --git a/src/cli/browser-cli-actions-input.ts b/src/cli/browser-cli-actions-input.ts index 58e5e9117..aa0b7d1c0 100644 --- a/src/cli/browser-cli-actions-input.ts +++ b/src/cli/browser-cli-actions-input.ts @@ -1,7 +1,6 @@ import type { Command } from "commander"; import { resolveBrowserControlUrl } from "../browser/client.js"; import { - browserBack, browserClick, browserDrag, browserEvaluate, @@ -11,7 +10,6 @@ import { browserNavigate, browserPressKey, browserResize, - browserRunCode, browserSelectOption, browserType, browserUpload, @@ -21,31 +19,11 @@ import { danger } from "../globals.js"; import { defaultRuntime } from "../runtime.js"; import type { BrowserParentOpts } from "./browser-cli-shared.js"; -async function readStdin(): Promise { - const chunks: string[] = []; - return await new Promise((resolve, reject) => { - process.stdin.setEncoding("utf8"); - process.stdin.on("data", (chunk) => chunks.push(String(chunk))); - process.stdin.on("end", () => resolve(chunks.join(""))); - process.stdin.on("error", reject); - }); -} - async function readFile(path: string): Promise { const fs = await import("node:fs/promises"); return await fs.readFile(path, "utf8"); } -async function readCode(opts: { - code?: string; - codeFile?: string; - codeStdin?: boolean; -}): Promise { - if (opts.codeFile) return await readFile(opts.codeFile); - if (opts.codeStdin) return await readStdin(); - return opts.code ?? ""; -} - async function readFields(opts: { fields?: string; fieldsFile?: string; @@ -87,30 +65,6 @@ export function registerBrowserActionInputCommands( } }); - browser - .command("back") - .description("Navigate back in history") - .option("--target-id ", "CDP target id (or unique prefix)") - .action(async (opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - try { - const result = await browserBack(baseUrl, { - targetId: opts.targetId?.trim() || undefined, - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log( - `navigated back to ${result.url ?? "previous page"}`, - ); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); - browser .command("resize") .description("Resize the viewport") @@ -311,7 +265,7 @@ export function registerBrowserActionInputCommands( browser .command("upload") - .description("Upload file(s) when a file chooser is open") + .description("Arm file upload for the next file chooser") .argument("", "File paths to upload") .option("--target-id ", "CDP target id (or unique prefix)") .action(async (paths: string[], opts, cmd) => { @@ -326,7 +280,7 @@ export function registerBrowserActionInputCommands( defaultRuntime.log(JSON.stringify(result, null, 2)); return; } - defaultRuntime.log(`uploaded ${paths.length} file(s)`); + defaultRuntime.log(`upload armed for ${paths.length} file(s)`); } catch (err) { defaultRuntime.error(danger(String(err))); defaultRuntime.exit(1); @@ -364,7 +318,7 @@ export function registerBrowserActionInputCommands( browser .command("dialog") - .description("Handle a modal dialog (alert/confirm/prompt)") + .description("Arm the next modal dialog (alert/confirm/prompt)") .option("--accept", "Accept the dialog", false) .option("--dismiss", "Dismiss the dialog", false) .option("--prompt ", "Prompt response text") @@ -388,7 +342,7 @@ export function registerBrowserActionInputCommands( defaultRuntime.log(JSON.stringify(result, null, 2)); return; } - defaultRuntime.log(`dialog handled: ${result.type}`); + defaultRuntime.log("dialog armed"); } catch (err) { defaultRuntime.error(danger(String(err))); defaultRuntime.exit(1); @@ -453,40 +407,4 @@ export function registerBrowserActionInputCommands( defaultRuntime.exit(1); } }); - - browser - .command("run") - .description("Run a Playwright code function (page => ...) ") - .option("--code ", "Function source, e.g. (page) => page.title()") - .option("--code-file ", "Read function source from a file") - .option("--code-stdin", "Read function source from stdin", false) - .option("--target-id ", "CDP target id (or unique prefix)") - .action(async (opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - try { - const code = await readCode({ - code: opts.code, - codeFile: opts.codeFile, - codeStdin: Boolean(opts.codeStdin), - }); - if (!code.trim()) { - defaultRuntime.error(danger("Missing --code (or file/stdin)")); - defaultRuntime.exit(1); - return; - } - const result = await browserRunCode(baseUrl, { - code, - targetId: opts.targetId?.trim() || undefined, - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log(JSON.stringify(result.result, null, 2)); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); } diff --git a/src/cli/browser-cli-actions-observe.ts b/src/cli/browser-cli-actions-observe.ts index fa9966594..f08e8af03 100644 --- a/src/cli/browser-cli-actions-observe.ts +++ b/src/cli/browser-cli-actions-observe.ts @@ -2,9 +2,6 @@ import type { Command } from "commander"; import { resolveBrowserControlUrl } from "../browser/client.js"; import { browserConsoleMessages, - browserMouseClick, - browserMouseDrag, - browserMouseMove, browserPdfSave, browserVerifyElementVisible, browserVerifyListVisible, @@ -178,110 +175,4 @@ export function registerBrowserActionObserveCommands( defaultRuntime.exit(1); } }); - - browser - .command("mouse-move") - .description("Move mouse to viewport coordinates") - .option("--x ", "X coordinate", (v: string) => Number(v)) - .option("--y ", "Y coordinate", (v: string) => Number(v)) - .option("--target-id ", "CDP target id (or unique prefix)") - .action(async (opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - if (!Number.isFinite(opts.x) || !Number.isFinite(opts.y)) { - defaultRuntime.error(danger("--x and --y are required")); - defaultRuntime.exit(1); - return; - } - try { - const result = await browserMouseMove(baseUrl, { - x: opts.x, - y: opts.y, - targetId: opts.targetId?.trim() || undefined, - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log("mouse moved"); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); - - browser - .command("mouse-click") - .description("Click at viewport coordinates") - .option("--x ", "X coordinate", (v: string) => Number(v)) - .option("--y ", "Y coordinate", (v: string) => Number(v)) - .option("--button ", "Mouse button") - .option("--target-id ", "CDP target id (or unique prefix)") - .action(async (opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - if (!Number.isFinite(opts.x) || !Number.isFinite(opts.y)) { - defaultRuntime.error(danger("--x and --y are required")); - defaultRuntime.exit(1); - return; - } - try { - const result = await browserMouseClick(baseUrl, { - x: opts.x, - y: opts.y, - button: opts.button?.trim() || undefined, - targetId: opts.targetId?.trim() || undefined, - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log("mouse clicked"); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); - - browser - .command("mouse-drag") - .description("Drag by viewport coordinates") - .option("--start-x ", "Start X", (v: string) => Number(v)) - .option("--start-y ", "Start Y", (v: string) => Number(v)) - .option("--end-x ", "End X", (v: string) => Number(v)) - .option("--end-y ", "End Y", (v: string) => Number(v)) - .option("--target-id ", "CDP target id (or unique prefix)") - .action(async (opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - if ( - !Number.isFinite(opts.startX) || - !Number.isFinite(opts.startY) || - !Number.isFinite(opts.endX) || - !Number.isFinite(opts.endY) - ) { - defaultRuntime.error( - danger("--start-x, --start-y, --end-x, --end-y are required"), - ); - defaultRuntime.exit(1); - return; - } - try { - const result = await browserMouseDrag(baseUrl, { - startX: opts.startX, - startY: opts.startY, - endX: opts.endX, - endY: opts.endY, - targetId: opts.targetId?.trim() || undefined, - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log("mouse dragged"); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); } diff --git a/src/cli/browser-cli-examples.ts b/src/cli/browser-cli-examples.ts index f41de6e41..e5f2fb3d5 100644 --- a/src/cli/browser-cli-examples.ts +++ b/src/cli/browser-cli-examples.ts @@ -9,7 +9,6 @@ export const browserCoreExamples = [ "clawdis browser screenshot", "clawdis browser screenshot --full-page", "clawdis browser screenshot --ref 12", - 'clawdis browser eval "document.title"', 'clawdis browser query "a" --limit 5', "clawdis browser dom --format text --max-chars 5000", "clawdis browser snapshot --format aria --limit 200", @@ -18,7 +17,6 @@ export const browserCoreExamples = [ export const browserActionExamples = [ "clawdis browser navigate https://example.com", - "clawdis browser back", "clawdis browser resize 1280 720", "clawdis browser click 12 --double", 'clawdis browser type 23 "hello" --submit', @@ -31,14 +29,10 @@ export const browserActionExamples = [ "clawdis browser dialog --accept", 'clawdis browser wait --text "Done"', "clawdis browser evaluate --fn '(el) => el.textContent' --ref 7", - "clawdis browser run --code '(page) => page.title()'", "clawdis browser console --level error", "clawdis browser pdf", 'clawdis browser verify-element --role button --name "Submit"', 'clawdis browser verify-text "Welcome"', "clawdis browser verify-list 3 ItemA ItemB", "clawdis browser verify-value --ref 4 --type textbox --value hello", - "clawdis browser mouse-move --x 120 --y 240", - "clawdis browser mouse-click --x 120 --y 240", - "clawdis browser mouse-drag --start-x 10 --start-y 20 --end-x 200 --end-y 300", ]; diff --git a/src/cli/browser-cli-inspect.ts b/src/cli/browser-cli-inspect.ts index be0363ae9..f2141ed0b 100644 --- a/src/cli/browser-cli-inspect.ts +++ b/src/cli/browser-cli-inspect.ts @@ -2,7 +2,6 @@ import type { Command } from "commander"; import { browserDom, - browserEval, browserQuery, browserScreenshot, browserSnapshot, @@ -13,31 +12,6 @@ import { danger } from "../globals.js"; import { defaultRuntime } from "../runtime.js"; import type { BrowserParentOpts } from "./browser-cli-shared.js"; -async function readStdin(): Promise { - const chunks: string[] = []; - return await new Promise((resolve, reject) => { - process.stdin.setEncoding("utf8"); - process.stdin.on("data", (chunk) => chunks.push(String(chunk))); - process.stdin.on("end", () => resolve(chunks.join(""))); - process.stdin.on("error", reject); - }); -} - -async function readTextFromSource(opts: { - js?: string; - jsFile?: string; - jsStdin?: boolean; -}): Promise { - if (opts.jsFile) { - const fs = await import("node:fs/promises"); - return await fs.readFile(opts.jsFile, "utf8"); - } - if (opts.jsStdin) { - return await readStdin(); - } - return opts.js ?? ""; -} - export function registerBrowserInspectCommands( browser: Command, parentOpts: (cmd: Command) => BrowserParentOpts, @@ -80,44 +54,6 @@ export function registerBrowserInspectCommands( } }); - browser - .command("eval") - .description("Run JavaScript in the active tab") - .argument("[js]", "JavaScript expression") - .option("--js-file ", "Read JavaScript from a file") - .option("--js-stdin", "Read JavaScript from stdin", false) - .option("--target-id ", "CDP target id (or unique prefix)") - .option("--await", "Await promise result", false) - .action(async (js: string | undefined, opts, cmd) => { - const parent = parentOpts(cmd); - const baseUrl = resolveBrowserControlUrl(parent?.url); - try { - const source = await readTextFromSource({ - js, - jsFile: opts.jsFile, - jsStdin: Boolean(opts.jsStdin), - }); - if (!source.trim()) { - defaultRuntime.error(danger("Missing JavaScript input.")); - defaultRuntime.exit(1); - return; - } - const result = await browserEval(baseUrl, { - js: source, - targetId: opts.targetId?.trim() || undefined, - awaitPromise: Boolean(opts.await), - }); - if (parent?.json) { - defaultRuntime.log(JSON.stringify(result, null, 2)); - return; - } - defaultRuntime.log(JSON.stringify(result.result, null, 2)); - } catch (err) { - defaultRuntime.error(danger(String(err))); - defaultRuntime.exit(1); - } - }); - browser .command("query") .description("Query selector matches")