let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(browser): prune browser automation surface

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-20 02:53:22 +00:00
parent 849446ae17
commit 6fc30962d6
19 changed files with 85 additions and 802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/browser/client-actions-core.ts
View File

@@ -1,5 +1,8 @@
import type { ScreenshotResult } from "./client.js";
import type { BrowserActionTabResult } from "./client-actions-types.js";
import type {
BrowserActionOk,
BrowserActionTabResult,
} from "./client-actions-types.js";
import { fetchBrowserJson } from "./client-fetch.js";
export async function browserNavigate(
@@ -14,18 +17,6 @@ export async function browserNavigate(
});
}
export async function browserBack(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/back`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserResize(
baseUrl: string,
opts: { width: number; height: number; targetId?: string },
@@ -185,20 +176,17 @@ export async function browserFillForm(
export async function browserHandleDialog(
baseUrl: string,
opts: { accept: boolean; promptText?: string; targetId?: string },
): Promise<{ ok: true; message: string; type: string }> {
return await fetchBrowserJson<{ ok: true; message: string; type: string }>(
`${baseUrl}/dialog`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
accept: opts.accept,
promptText: opts.promptText,
targetId: opts.targetId,
}),
timeoutMs: 20000,
},
);
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/dialog`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
accept: opts.accept,
promptText: opts.promptText,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserWaitFor(
@@ -242,21 +230,6 @@ export async function browserEvaluate(
);
}
export async function browserRunCode(
baseUrl: string,
opts: { code: string; targetId?: string },
): Promise<{ ok: true; result: unknown }> {
return await fetchBrowserJson<{ ok: true; result: unknown }>(
`${baseUrl}/run`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ code: opts.code, targetId: opts.targetId }),
timeoutMs: 20000,
},
);
}
export async function browserScreenshotAction(
baseUrl: string,
opts: {

53
src/browser/client-actions-observe.ts
View File

@@ -92,56 +92,3 @@ export async function browserVerifyValue(
timeoutMs: 20000,
});
}
export async function browserMouseMove(
baseUrl: string,
opts: { x: number; y: number; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/move`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ x: opts.x, y: opts.y, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserMouseClick(
baseUrl: string,
opts: { x: number; y: number; button?: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/click`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
x: opts.x,
y: opts.y,
button: opts.button,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserMouseDrag(
baseUrl: string,
opts: {
startX: number;
startY: number;
endX: number;
endY: number;
targetId?: string;
},
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/drag`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
startX: opts.startX,
startY: opts.startY,
endX: opts.endX,
endY: opts.endY,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}

3
src/browser/client.test.ts
View File

@@ -3,7 +3,6 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import {
browserClickRef,
browserDom,
browserEval,
browserOpenTab,
browserQuery,
browserScreenshot,
@@ -50,7 +49,7 @@ describe("browser client", () => {
);
await expect(
browserEval("http://127.0.0.1:18791", { js: "1+1" }),
browserDom("http://127.0.0.1:18791", { format: "text", maxChars: 1 }),
).rejects.toThrow(/409: conflict/i);
});

34
src/browser/client.ts
View File

@@ -29,20 +29,6 @@ export type ScreenshotResult = {
url: string;
};
export type EvalResult = {
ok: true;
targetId: string;
url: string;
result: {
type: string;
subtype?: string;
value?: unknown;
description?: string;
unserializableValue?: string;
preview?: unknown;
};
};
export type QueryResult = {
ok: true;
targetId: string;
@@ -201,26 +187,6 @@ export async function browserScreenshot(
);
}
export async function browserEval(
baseUrl: string,
opts: {
js: string;
targetId?: string;
awaitPromise?: boolean;
},
): Promise<EvalResult> {
return await fetchBrowserJson<EvalResult>(`${baseUrl}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
js: opts.js,
targetId: opts.targetId,
await: Boolean(opts.awaitPromise),
}),
timeoutMs: 15000,
});
}
export async function browserQuery(
baseUrl: string,
opts: {

9
src/browser/pw-ai.ts
View File

@@ -8,21 +8,19 @@ export {
} from "./pw-session.js";
export {
armDialogViaPlaywright,
armFileUploadViaPlaywright,
clickRefViaPlaywright,
clickViaPlaywright,
closePageViaPlaywright,
dragViaPlaywright,
evaluateViaPlaywright,
fileUploadViaPlaywright,
fillFormViaPlaywright,
handleDialogViaPlaywright,
hoverViaPlaywright,
navigateBackViaPlaywright,
navigateViaPlaywright,
pdfViaPlaywright,
pressKeyViaPlaywright,
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
snapshotAiViaPlaywright,
takeScreenshotViaPlaywright,
@@ -32,9 +30,6 @@ export {
export {
getConsoleMessagesViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
verifyTextVisibleViaPlaywright,

13
src/browser/pw-session.ts
View File

@@ -13,15 +13,6 @@ export type BrowserConsoleMessage = {
location?: { url?: string; lineNumber?: number; columnNumber?: number };
};
export type BrowserNetworkRequest = {
requestId?: string;
url: string;
method: string;
status?: number;
resourceType?: string;
timestamp?: string;
};
type SnapshotForAIResult = { full: string; incremental?: string };
type SnapshotForAIOptions = { timeout?: number; track?: string };
@@ -44,6 +35,8 @@ type ConnectedBrowser = {
type PageState = {
console: BrowserConsoleMessage[];
armIdUpload: number;
armIdDialog: number;
};
const pageStates = new WeakMap<Page, PageState>();
@@ -65,6 +58,8 @@ export function ensurePageState(page: Page): PageState {
const state: PageState = {
console: [],
armIdUpload: 0,
armIdDialog: 0,
};
pageStates.set(page, state);

97
src/browser/pw-tools-core.ts
View File

@@ -1,5 +1,3 @@
import type { Page } from "playwright-core";
import {
ensurePageState,
getPageForTargetId,
@@ -7,6 +5,9 @@ import {
type WithSnapshotForAI,
} from "./pw-session.js";
let nextUploadArmId = 0;
let nextDialogArmId = 0;
export async function snapshotAiViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -221,44 +222,63 @@ export async function evaluateViaPlaywright(opts: {
}, fnText);
}
export async function fileUploadViaPlaywright(opts: {
export async function armFileUploadViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
paths?: string[];
timeoutMs?: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const state = ensurePageState(page);
const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000));
const fileChooser = await page.waitForEvent("filechooser", { timeout });
if (!opts.paths?.length) {
// Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead.
try {
await page.keyboard.press("Escape");
} catch {
// Best-effort.
}
return;
}
await fileChooser.setFiles(opts.paths);
state.armIdUpload = nextUploadArmId += 1;
const armId = state.armIdUpload;
void page
.waitForEvent("filechooser", { timeout })
.then(async (fileChooser) => {
if (state.armIdUpload !== armId) return;
if (!opts.paths?.length) {
// Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead.
try {
await page.keyboard.press("Escape");
} catch {
// Best-effort.
}
return;
}
await fileChooser.setFiles(opts.paths);
})
.catch(() => {
// Ignore timeouts; the chooser may never appear.
});
}
export async function handleDialogViaPlaywright(opts: {
export async function armDialogViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
accept: boolean;
promptText?: string;
timeoutMs?: number;
}): Promise<{ message: string; type: string }> {
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const state = ensurePageState(page);
const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000));
const dialog = await page.waitForEvent("dialog", { timeout });
const message = dialog.message();
const type = dialog.type();
if (opts.accept) await dialog.accept(opts.promptText);
else await dialog.dismiss();
return { message, type };
state.armIdDialog = nextDialogArmId += 1;
const armId = state.armIdDialog;
void page
.waitForEvent("dialog", { timeout })
.then(async (dialog) => {
if (state.armIdDialog !== armId) return;
if (opts.accept) await dialog.accept(opts.promptText);
else await dialog.dismiss();
})
.catch(() => {
// Ignore timeouts; the dialog may never appear.
});
}
export async function navigateViaPlaywright(opts: {
@@ -277,19 +297,6 @@ export async function navigateViaPlaywright(opts: {
return { url: page.url() };
}
export async function navigateBackViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
timeoutMs?: number;
}): Promise<{ url: string }> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.goBack({
timeout: Math.max(1000, Math.min(120_000, opts.timeoutMs ?? 20_000)),
});
return { url: page.url() };
}
export async function waitForViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -323,22 +330,6 @@ export async function waitForViaPlaywright(opts: {
}
}
export async function runCodeViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
code: string;
}): Promise<unknown> {
const code = String(opts.code ?? "").trim();
if (!code) throw new Error("code is required");
const page = await getPageForTargetId(opts);
ensurePageState(page);
const fn = new Function(`return (${code});`)() as
| ((page: Page) => unknown)
| undefined;
if (typeof fn !== "function") throw new Error("code is not a function");
return await fn(page);
}
export async function takeScreenshotViaPlaywright(opts: {
cdpPort: number;
targetId?: string;

41
src/browser/pw-tools-observe.ts
View File

@@ -33,47 +33,6 @@ export async function getConsoleMessagesViaPlaywright(opts: {
return state.console.filter((msg) => consolePriority(msg.type) >= min);
}
export async function mouseMoveViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
x: number;
y: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.move(opts.x, opts.y);
}
export async function mouseClickViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
x: number;
y: number;
button?: "left" | "right" | "middle";
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.click(opts.x, opts.y, {
button: opts.button,
});
}
export async function mouseDragViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
startX: number;
startY: number;
endX: number;
endY: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.move(opts.startX, opts.startY);
await page.mouse.down();
await page.mouse.move(opts.endX, opts.endY);
await page.mouse.up();
}
export async function verifyElementVisibleViaPlaywright(opts: {
cdpPort: number;
targetId?: string;

28
src/browser/routes/actions-core.test.ts
View File

@@ -3,23 +3,19 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { BrowserRouteContext } from "../server-context.js";
const pw = vi.hoisted(() => ({
armDialogViaPlaywright: vi.fn().mockResolvedValue(undefined),
armFileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined),
clickViaPlaywright: vi.fn().mockResolvedValue(undefined),
closePageViaPlaywright: vi.fn().mockResolvedValue(undefined),
dragViaPlaywright: vi.fn().mockResolvedValue(undefined),
evaluateViaPlaywright: vi.fn().mockResolvedValue("result"),
fileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined),
fillFormViaPlaywright: vi.fn().mockResolvedValue(undefined),
handleDialogViaPlaywright: vi
.fn()
.mockResolvedValue({ message: "ok", type: "alert" }),
hoverViaPlaywright: vi.fn().mockResolvedValue(undefined),
navigateBackViaPlaywright: vi.fn().mockResolvedValue({ url: "about:blank" }),
navigateViaPlaywright: vi
.fn()
.mockResolvedValue({ url: "https://example.com" }),
pressKeyViaPlaywright: vi.fn().mockResolvedValue(undefined),
resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined),
runCodeViaPlaywright: vi.fn().mockResolvedValue("ok"),
selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined),
typeViaPlaywright: vi.fn().mockResolvedValue(undefined),
waitForViaPlaywright: vi.fn().mockResolvedValue(undefined),
@@ -127,14 +123,14 @@ describe("handleBrowserActionCore", () => {
{
action: "dialog" as const,
args: { accept: true, promptText: "ok" },
fn: pw.handleDialogViaPlaywright,
fn: pw.armDialogViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
accept: true,
promptText: "ok",
},
expectBody: { ok: true, message: "ok", type: "alert" },
expectBody: { ok: true },
},
{
action: "evaluate" as const,
@@ -151,7 +147,7 @@ describe("handleBrowserActionCore", () => {
{
action: "upload" as const,
args: { paths: ["/tmp/file.txt"] },
fn: pw.fileUploadViaPlaywright,
fn: pw.armFileUploadViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
@@ -202,20 +198,6 @@ describe("handleBrowserActionCore", () => {
},
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
action: "back" as const,
args: {},
fn: pw.navigateBackViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: "about:blank" },
},
{
action: "run" as const,
args: { code: "return 1" },
fn: pw.runCodeViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", code: "return 1" },
expectBody: { ok: true, result: "ok" },
},
{
action: "click" as const,
args: {

38
src/browser/routes/actions-core.ts
View File

@@ -1,19 +1,17 @@
import type express from "express";
import {
armDialogViaPlaywright,
armFileUploadViaPlaywright,
clickViaPlaywright,
closePageViaPlaywright,
dragViaPlaywright,
evaluateViaPlaywright,
fileUploadViaPlaywright,
fillFormViaPlaywright,
handleDialogViaPlaywright,
hoverViaPlaywright,
navigateBackViaPlaywright,
navigateViaPlaywright,
pressKeyViaPlaywright,
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
typeViaPlaywright,
waitForViaPlaywright,
@@ -51,7 +49,6 @@ function normalizeModifiers(value: unknown): KeyboardModifier[] | undefined {
}
export type BrowserActionCore =
| "back"
| "click"
| "close"
| "dialog"
@@ -62,7 +59,6 @@ export type BrowserActionCore =
| "navigate"
| "press"
| "resize"
| "run"
| "select"
| "type"
| "upload"
@@ -115,13 +111,13 @@ export async function handleBrowserActionCore(
}
const promptText = toStringOrEmpty(args.promptText) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const result = await handleDialogViaPlaywright({
await armDialogViaPlaywright({
cdpPort,
targetId: tab.targetId,
accept,
promptText,
});
res.json({ ok: true, ...result });
res.json({ ok: true });
return true;
}
case "evaluate": {
@@ -144,7 +140,7 @@ export async function handleBrowserActionCore(
case "upload": {
const paths = toStringArray(args.paths) ?? [];
const tab = await ctx.ensureTabAvailable(target);
await fileUploadViaPlaywright({
await armFileUploadViaPlaywright({
cdpPort,
targetId: tab.targetId,
paths: paths.length ? paths : undefined,
@@ -220,30 +216,6 @@ export async function handleBrowserActionCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "back": {
const tab = await ctx.ensureTabAvailable(target);
const result = await navigateBackViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "run": {
const code = toStringOrEmpty(args.code);
if (!code) {
jsonError(res, 400, "code is required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
const result = await runCodeViaPlaywright({
cdpPort,
targetId: tab.targetId,
code,
});
res.json({ ok: true, result });
return true;
}
case "click": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {

37
src/browser/routes/actions-extra.test.ts
View File

@@ -4,9 +4,6 @@ import type { BrowserRouteContext } from "../server-context.js";
const pw = vi.hoisted(() => ({
getConsoleMessagesViaPlaywright: vi.fn().mockResolvedValue([]),
mouseClickViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseDragViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseMoveViaPlaywright: vi.fn().mockResolvedValue(undefined),
pdfViaPlaywright: vi.fn().mockResolvedValue({ buffer: Buffer.from("pdf") }),
verifyElementVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
verifyListVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
@@ -149,40 +146,6 @@ describe("handleBrowserActionExtra", () => {
},
expectBody: { ok: true },
},
{
action: "mouseMove" as const,
args: { x: 10, y: 20 },
fn: pw.mouseMoveViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", x: 10, y: 20 },
expectBody: { ok: true },
},
{
action: "mouseClick" as const,
args: { x: 1, y: 2, button: "right" },
fn: pw.mouseClickViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
x: 1,
y: 2,
button: "right",
},
expectBody: { ok: true },
},
{
action: "mouseDrag" as const,
args: { startX: 1, startY: 2, endX: 3, endY: 4 },
fn: pw.mouseDragViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
startX: 1,
startY: 2,
endX: 3,
endY: 4,
},
expectBody: { ok: true },
},
];
for (const item of cases) {

83
src/browser/routes/actions-extra.ts
View File

@@ -5,9 +5,6 @@ import type express from "express";
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
getConsoleMessagesViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
pdfViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
@@ -15,26 +12,10 @@ import {
verifyValueViaPlaywright,
} from "../pw-ai.js";
import type { BrowserRouteContext } from "../server-context.js";
import {
jsonError,
toNumber,
toStringArray,
toStringOrEmpty,
} from "./utils.js";
type MouseButton = "left" | "right" | "middle";
function normalizeMouseButton(value: unknown): MouseButton | undefined {
const raw = toStringOrEmpty(value);
if (raw === "left" || raw === "right" || raw === "middle") return raw;
return undefined;
}
import { jsonError, toStringArray, toStringOrEmpty } from "./utils.js";
export type BrowserActionExtra =
| "console"
| "mouseClick"
| "mouseDrag"
| "mouseMove"
| "pdf"
| "verifyElement"
| "verifyList"
@@ -157,68 +138,6 @@ export async function handleBrowserActionExtra(
res.json({ ok: true });
return true;
}
case "mouseMove": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
jsonError(res, 400, "x and y are required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
await mouseMoveViaPlaywright({
cdpPort,
targetId: tab.targetId,
x,
y,
});
res.json({ ok: true });
return true;
}
case "mouseClick": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
jsonError(res, 400, "x and y are required");
return true;
}
const button = normalizeMouseButton(args.button);
const tab = await ctx.ensureTabAvailable(target);
await mouseClickViaPlaywright({
cdpPort,
targetId: tab.targetId,
x,
y,
button,
});
res.json({ ok: true });
return true;
}
case "mouseDrag": {
const startX = toNumber(args.startX);
const startY = toNumber(args.startY);
const endX = toNumber(args.endX);
const endY = toNumber(args.endY);
if (
startX === undefined ||
startY === undefined ||
endX === undefined ||
endY === undefined
) {
jsonError(res, 400, "startX, startY, endX, endY are required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
await mouseDragViaPlaywright({
cdpPort,
targetId: tab.targetId,
startX,
startY,
endX,
endY,
});
res.json({ ok: true });
return true;
}
default:
return false;
}

30
src/browser/routes/actions.ts
View File

@@ -79,12 +79,6 @@ export function registerBrowserActionRoutes(
await runCoreAction(ctx, res, "navigate", body, targetId);
});
app.post("/back", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "back", body, targetId);
});
app.post("/resize", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
@@ -163,12 +157,6 @@ export function registerBrowserActionRoutes(
await runCoreAction(ctx, res, "evaluate", body, targetId);
});
app.post("/run", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "run", body, targetId);
});
app.get("/console", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const level = toStringOrEmpty(req.query.level);
@@ -206,21 +194,5 @@ export function registerBrowserActionRoutes(
await runExtraAction(ctx, res, "verifyValue", body, targetId);
});
app.post("/mouse/move", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseMove", body, targetId);
});
app.post("/mouse/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseClick", body, targetId);
});
app.post("/mouse/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseDrag", body, targetId);
});
// Intentionally no coordinate-based mouse actions (move/click/drag).
}

40
src/browser/routes/inspect.ts
View File

@@ -5,7 +5,6 @@ import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
captureScreenshot,
captureScreenshotPng,
evaluateJavaScript,
getDomText,
querySelector,
snapshotAria,
@@ -124,45 +123,6 @@ export function registerBrowserInspectRoutes(
}
});
app.post("/eval", async (req, res) => {
const js = toStringOrEmpty((req.body as { js?: unknown })?.js);
const targetId = toStringOrEmpty(
(req.body as { targetId?: unknown })?.targetId,
);
const awaitPromise = Boolean((req.body as { await?: unknown })?.await);
if (!js) return jsonError(res, 400, "js is required");
try {
const tab = await ctx.ensureTabAvailable(targetId || undefined);
const evaluated = await evaluateJavaScript({
wsUrl: tab.wsUrl ?? "",
expression: js,
awaitPromise,
returnByValue: true,
});
if (evaluated.exceptionDetails) {
const msg =
evaluated.exceptionDetails.exception?.description ||
evaluated.exceptionDetails.text ||
"JavaScript evaluation failed";
return jsonError(res, 400, msg);
}
res.json({
ok: true,
targetId: tab.targetId,
url: tab.url,
result: evaluated.result,
});
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
app.get("/query", async (req, res) => {
const selector =
typeof req.query.selector === "string" ? req.query.selector.trim() : "";

55
src/browser/server.test.ts
View File

@@ -62,22 +62,11 @@ vi.mock("./chrome.js", () => ({
}),
}));
const evalCalls = vi.hoisted(() => [] as Array<string>);
let evalThrows = false;
vi.mock("./cdp.js", () => ({
createTargetViaCdp: vi.fn(async () => {
if (createTargetId) return { targetId: createTargetId };
throw new Error("cdp disabled");
}),
evaluateJavaScript: vi.fn(async ({ expression }: { expression: string }) => {
evalCalls.push(expression);
if (evalThrows) {
return {
exceptionDetails: { text: "boom" },
};
}
return { result: { type: "string", value: "ok" } };
}),
getDomText: vi.fn(async () => ({ text: "<html/>" })),
querySelector: vi.fn(async () => ({ matches: [{ index: 0, tag: "a" }] })),
snapshotAria: vi.fn(async () => ({
@@ -97,28 +86,20 @@ vi.mock("./cdp.js", () => ({
}));
vi.mock("./pw-ai.js", () => ({
armDialogViaPlaywright: vi.fn(async () => {}),
armFileUploadViaPlaywright: vi.fn(async () => {}),
clickRefViaPlaywright: vi.fn(async () => {}),
clickViaPlaywright: vi.fn(async () => {}),
closePageViaPlaywright: vi.fn(async () => {}),
closePlaywrightBrowserConnection: vi.fn(async () => {}),
evaluateViaPlaywright: vi.fn(async () => "ok"),
fileUploadViaPlaywright: vi.fn(async () => {}),
fillFormViaPlaywright: vi.fn(async () => {}),
getConsoleMessagesViaPlaywright: vi.fn(async () => []),
handleDialogViaPlaywright: vi.fn(async () => ({
message: "ok",
type: "alert",
})),
hoverViaPlaywright: vi.fn(async () => {}),
mouseClickViaPlaywright: vi.fn(async () => {}),
mouseDragViaPlaywright: vi.fn(async () => {}),
mouseMoveViaPlaywright: vi.fn(async () => {}),
navigateBackViaPlaywright: vi.fn(async () => ({ url: "about:blank" })),
navigateViaPlaywright: vi.fn(async () => ({ url: "https://example.com" })),
pdfViaPlaywright: vi.fn(async () => ({ buffer: Buffer.from("pdf") })),
pressKeyViaPlaywright: vi.fn(async () => {}),
resizeViewportViaPlaywright: vi.fn(async () => {}),
runCodeViaPlaywright: vi.fn(async () => "ok"),
selectOptionViaPlaywright: vi.fn(async () => {}),
snapshotAiViaPlaywright: vi.fn(async () => ({ snapshot: "ok" })),
takeScreenshotViaPlaywright: vi.fn(async () => ({
@@ -179,6 +160,7 @@ describe("browser control server", () => {
cfgAttachOnly = false;
createTargetId = null;
screenshotThrowsOnce = false;
testPort = await getFreePort();
// Minimal CDP JSON endpoints used by the server.
@@ -288,30 +270,6 @@ describe("browser control server", () => {
expect(focus.status).toBe(409);
});
it("maps JS exceptions to a 400 and returns results otherwise", async () => {
const { startBrowserControlServerFromConfig } = await import("./server.js");
await startBrowserControlServerFromConfig();
const base = `http://127.0.0.1:${testPort}`;
await realFetch(`${base}/start`, { method: "POST" }).then((r) => r.json());
evalThrows = true;
const bad = await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ js: "throw 1" }),
});
expect(bad.status).toBe(400);
evalThrows = false;
const ok = (await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ js: "1+1", await: true }),
}).then((r) => r.json())) as { ok: boolean; result?: unknown };
expect(ok.ok).toBe(true);
expect(evalCalls.length).toBeGreaterThan(0);
});
it("supports query/dom/snapshot/click/screenshot and stop", async () => {
const { startBrowserControlServerFromConfig } = await import("./server.js");
await startBrowserControlServerFromConfig();
@@ -443,13 +401,6 @@ describe("browser control server", () => {
const shotAmbiguous = await realFetch(`${base}/screenshot?targetId=abc`);
expect(shotAmbiguous.status).toBe(409);
const evalMissing = await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({}),
});
expect(evalMissing.status).toBe(400);
const queryMissing = await realFetch(`${base}/query`);
expect(queryMissing.status).toBe(400);