refactor(browser): prune browser automation surface

This commit is contained in:
Peter Steinberger
2025-12-20 02:53:22 +00:00
parent 849446ae17
commit 6fc30962d6
19 changed files with 85 additions and 802 deletions

View File

@@ -1,5 +1,8 @@
import type { ScreenshotResult } from "./client.js";
import type { BrowserActionTabResult } from "./client-actions-types.js";
import type {
BrowserActionOk,
BrowserActionTabResult,
} from "./client-actions-types.js";
import { fetchBrowserJson } from "./client-fetch.js";
export async function browserNavigate(
@@ -14,18 +17,6 @@ export async function browserNavigate(
});
}
export async function browserBack(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/back`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserResize(
baseUrl: string,
opts: { width: number; height: number; targetId?: string },
@@ -185,20 +176,17 @@ export async function browserFillForm(
export async function browserHandleDialog(
baseUrl: string,
opts: { accept: boolean; promptText?: string; targetId?: string },
): Promise<{ ok: true; message: string; type: string }> {
return await fetchBrowserJson<{ ok: true; message: string; type: string }>(
`${baseUrl}/dialog`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
accept: opts.accept,
promptText: opts.promptText,
targetId: opts.targetId,
}),
timeoutMs: 20000,
},
);
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/dialog`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
accept: opts.accept,
promptText: opts.promptText,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserWaitFor(
@@ -242,21 +230,6 @@ export async function browserEvaluate(
);
}
export async function browserRunCode(
baseUrl: string,
opts: { code: string; targetId?: string },
): Promise<{ ok: true; result: unknown }> {
return await fetchBrowserJson<{ ok: true; result: unknown }>(
`${baseUrl}/run`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ code: opts.code, targetId: opts.targetId }),
timeoutMs: 20000,
},
);
}
export async function browserScreenshotAction(
baseUrl: string,
opts: {

View File

@@ -92,56 +92,3 @@ export async function browserVerifyValue(
timeoutMs: 20000,
});
}
export async function browserMouseMove(
baseUrl: string,
opts: { x: number; y: number; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/move`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ x: opts.x, y: opts.y, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserMouseClick(
baseUrl: string,
opts: { x: number; y: number; button?: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/click`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
x: opts.x,
y: opts.y,
button: opts.button,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserMouseDrag(
baseUrl: string,
opts: {
startX: number;
startY: number;
endX: number;
endY: number;
targetId?: string;
},
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/drag`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
startX: opts.startX,
startY: opts.startY,
endX: opts.endX,
endY: opts.endY,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}

View File

@@ -3,7 +3,6 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import {
browserClickRef,
browserDom,
browserEval,
browserOpenTab,
browserQuery,
browserScreenshot,
@@ -50,7 +49,7 @@ describe("browser client", () => {
);
await expect(
browserEval("http://127.0.0.1:18791", { js: "1+1" }),
browserDom("http://127.0.0.1:18791", { format: "text", maxChars: 1 }),
).rejects.toThrow(/409: conflict/i);
});

View File

@@ -29,20 +29,6 @@ export type ScreenshotResult = {
url: string;
};
export type EvalResult = {
ok: true;
targetId: string;
url: string;
result: {
type: string;
subtype?: string;
value?: unknown;
description?: string;
unserializableValue?: string;
preview?: unknown;
};
};
export type QueryResult = {
ok: true;
targetId: string;
@@ -201,26 +187,6 @@ export async function browserScreenshot(
);
}
export async function browserEval(
baseUrl: string,
opts: {
js: string;
targetId?: string;
awaitPromise?: boolean;
},
): Promise<EvalResult> {
return await fetchBrowserJson<EvalResult>(`${baseUrl}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
js: opts.js,
targetId: opts.targetId,
await: Boolean(opts.awaitPromise),
}),
timeoutMs: 15000,
});
}
export async function browserQuery(
baseUrl: string,
opts: {

View File

@@ -8,21 +8,19 @@ export {
} from "./pw-session.js";
export {
armDialogViaPlaywright,
armFileUploadViaPlaywright,
clickRefViaPlaywright,
clickViaPlaywright,
closePageViaPlaywright,
dragViaPlaywright,
evaluateViaPlaywright,
fileUploadViaPlaywright,
fillFormViaPlaywright,
handleDialogViaPlaywright,
hoverViaPlaywright,
navigateBackViaPlaywright,
navigateViaPlaywright,
pdfViaPlaywright,
pressKeyViaPlaywright,
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
snapshotAiViaPlaywright,
takeScreenshotViaPlaywright,
@@ -32,9 +30,6 @@ export {
export {
getConsoleMessagesViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
verifyTextVisibleViaPlaywright,

View File

@@ -13,15 +13,6 @@ export type BrowserConsoleMessage = {
location?: { url?: string; lineNumber?: number; columnNumber?: number };
};
export type BrowserNetworkRequest = {
requestId?: string;
url: string;
method: string;
status?: number;
resourceType?: string;
timestamp?: string;
};
type SnapshotForAIResult = { full: string; incremental?: string };
type SnapshotForAIOptions = { timeout?: number; track?: string };
@@ -44,6 +35,8 @@ type ConnectedBrowser = {
type PageState = {
console: BrowserConsoleMessage[];
armIdUpload: number;
armIdDialog: number;
};
const pageStates = new WeakMap<Page, PageState>();
@@ -65,6 +58,8 @@ export function ensurePageState(page: Page): PageState {
const state: PageState = {
console: [],
armIdUpload: 0,
armIdDialog: 0,
};
pageStates.set(page, state);

View File

@@ -1,5 +1,3 @@
import type { Page } from "playwright-core";
import {
ensurePageState,
getPageForTargetId,
@@ -7,6 +5,9 @@ import {
type WithSnapshotForAI,
} from "./pw-session.js";
let nextUploadArmId = 0;
let nextDialogArmId = 0;
export async function snapshotAiViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -221,44 +222,63 @@ export async function evaluateViaPlaywright(opts: {
}, fnText);
}
export async function fileUploadViaPlaywright(opts: {
export async function armFileUploadViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
paths?: string[];
timeoutMs?: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const state = ensurePageState(page);
const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000));
const fileChooser = await page.waitForEvent("filechooser", { timeout });
if (!opts.paths?.length) {
// Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead.
try {
await page.keyboard.press("Escape");
} catch {
// Best-effort.
}
return;
}
await fileChooser.setFiles(opts.paths);
state.armIdUpload = nextUploadArmId += 1;
const armId = state.armIdUpload;
void page
.waitForEvent("filechooser", { timeout })
.then(async (fileChooser) => {
if (state.armIdUpload !== armId) return;
if (!opts.paths?.length) {
// Playwright removed `FileChooser.cancel()`; best-effort close the chooser instead.
try {
await page.keyboard.press("Escape");
} catch {
// Best-effort.
}
return;
}
await fileChooser.setFiles(opts.paths);
})
.catch(() => {
// Ignore timeouts; the chooser may never appear.
});
}
export async function handleDialogViaPlaywright(opts: {
export async function armDialogViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
accept: boolean;
promptText?: string;
timeoutMs?: number;
}): Promise<{ message: string; type: string }> {
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const state = ensurePageState(page);
const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 10_000));
const dialog = await page.waitForEvent("dialog", { timeout });
const message = dialog.message();
const type = dialog.type();
if (opts.accept) await dialog.accept(opts.promptText);
else await dialog.dismiss();
return { message, type };
state.armIdDialog = nextDialogArmId += 1;
const armId = state.armIdDialog;
void page
.waitForEvent("dialog", { timeout })
.then(async (dialog) => {
if (state.armIdDialog !== armId) return;
if (opts.accept) await dialog.accept(opts.promptText);
else await dialog.dismiss();
})
.catch(() => {
// Ignore timeouts; the dialog may never appear.
});
}
export async function navigateViaPlaywright(opts: {
@@ -277,19 +297,6 @@ export async function navigateViaPlaywright(opts: {
return { url: page.url() };
}
export async function navigateBackViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
timeoutMs?: number;
}): Promise<{ url: string }> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.goBack({
timeout: Math.max(1000, Math.min(120_000, opts.timeoutMs ?? 20_000)),
});
return { url: page.url() };
}
export async function waitForViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -323,22 +330,6 @@ export async function waitForViaPlaywright(opts: {
}
}
export async function runCodeViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
code: string;
}): Promise<unknown> {
const code = String(opts.code ?? "").trim();
if (!code) throw new Error("code is required");
const page = await getPageForTargetId(opts);
ensurePageState(page);
const fn = new Function(`return (${code});`)() as
| ((page: Page) => unknown)
| undefined;
if (typeof fn !== "function") throw new Error("code is not a function");
return await fn(page);
}
export async function takeScreenshotViaPlaywright(opts: {
cdpPort: number;
targetId?: string;

View File

@@ -33,47 +33,6 @@ export async function getConsoleMessagesViaPlaywright(opts: {
return state.console.filter((msg) => consolePriority(msg.type) >= min);
}
export async function mouseMoveViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
x: number;
y: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.move(opts.x, opts.y);
}
export async function mouseClickViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
x: number;
y: number;
button?: "left" | "right" | "middle";
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.click(opts.x, opts.y, {
button: opts.button,
});
}
export async function mouseDragViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
startX: number;
startY: number;
endX: number;
endY: number;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
await page.mouse.move(opts.startX, opts.startY);
await page.mouse.down();
await page.mouse.move(opts.endX, opts.endY);
await page.mouse.up();
}
export async function verifyElementVisibleViaPlaywright(opts: {
cdpPort: number;
targetId?: string;

View File

@@ -3,23 +3,19 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { BrowserRouteContext } from "../server-context.js";
const pw = vi.hoisted(() => ({
armDialogViaPlaywright: vi.fn().mockResolvedValue(undefined),
armFileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined),
clickViaPlaywright: vi.fn().mockResolvedValue(undefined),
closePageViaPlaywright: vi.fn().mockResolvedValue(undefined),
dragViaPlaywright: vi.fn().mockResolvedValue(undefined),
evaluateViaPlaywright: vi.fn().mockResolvedValue("result"),
fileUploadViaPlaywright: vi.fn().mockResolvedValue(undefined),
fillFormViaPlaywright: vi.fn().mockResolvedValue(undefined),
handleDialogViaPlaywright: vi
.fn()
.mockResolvedValue({ message: "ok", type: "alert" }),
hoverViaPlaywright: vi.fn().mockResolvedValue(undefined),
navigateBackViaPlaywright: vi.fn().mockResolvedValue({ url: "about:blank" }),
navigateViaPlaywright: vi
.fn()
.mockResolvedValue({ url: "https://example.com" }),
pressKeyViaPlaywright: vi.fn().mockResolvedValue(undefined),
resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined),
runCodeViaPlaywright: vi.fn().mockResolvedValue("ok"),
selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined),
typeViaPlaywright: vi.fn().mockResolvedValue(undefined),
waitForViaPlaywright: vi.fn().mockResolvedValue(undefined),
@@ -127,14 +123,14 @@ describe("handleBrowserActionCore", () => {
{
action: "dialog" as const,
args: { accept: true, promptText: "ok" },
fn: pw.handleDialogViaPlaywright,
fn: pw.armDialogViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
accept: true,
promptText: "ok",
},
expectBody: { ok: true, message: "ok", type: "alert" },
expectBody: { ok: true },
},
{
action: "evaluate" as const,
@@ -151,7 +147,7 @@ describe("handleBrowserActionCore", () => {
{
action: "upload" as const,
args: { paths: ["/tmp/file.txt"] },
fn: pw.fileUploadViaPlaywright,
fn: pw.armFileUploadViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
@@ -202,20 +198,6 @@ describe("handleBrowserActionCore", () => {
},
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
action: "back" as const,
args: {},
fn: pw.navigateBackViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: "about:blank" },
},
{
action: "run" as const,
args: { code: "return 1" },
fn: pw.runCodeViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", code: "return 1" },
expectBody: { ok: true, result: "ok" },
},
{
action: "click" as const,
args: {

View File

@@ -1,19 +1,17 @@
import type express from "express";
import {
armDialogViaPlaywright,
armFileUploadViaPlaywright,
clickViaPlaywright,
closePageViaPlaywright,
dragViaPlaywright,
evaluateViaPlaywright,
fileUploadViaPlaywright,
fillFormViaPlaywright,
handleDialogViaPlaywright,
hoverViaPlaywright,
navigateBackViaPlaywright,
navigateViaPlaywright,
pressKeyViaPlaywright,
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
typeViaPlaywright,
waitForViaPlaywright,
@@ -51,7 +49,6 @@ function normalizeModifiers(value: unknown): KeyboardModifier[] | undefined {
}
export type BrowserActionCore =
| "back"
| "click"
| "close"
| "dialog"
@@ -62,7 +59,6 @@ export type BrowserActionCore =
| "navigate"
| "press"
| "resize"
| "run"
| "select"
| "type"
| "upload"
@@ -115,13 +111,13 @@ export async function handleBrowserActionCore(
}
const promptText = toStringOrEmpty(args.promptText) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const result = await handleDialogViaPlaywright({
await armDialogViaPlaywright({
cdpPort,
targetId: tab.targetId,
accept,
promptText,
});
res.json({ ok: true, ...result });
res.json({ ok: true });
return true;
}
case "evaluate": {
@@ -144,7 +140,7 @@ export async function handleBrowserActionCore(
case "upload": {
const paths = toStringArray(args.paths) ?? [];
const tab = await ctx.ensureTabAvailable(target);
await fileUploadViaPlaywright({
await armFileUploadViaPlaywright({
cdpPort,
targetId: tab.targetId,
paths: paths.length ? paths : undefined,
@@ -220,30 +216,6 @@ export async function handleBrowserActionCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "back": {
const tab = await ctx.ensureTabAvailable(target);
const result = await navigateBackViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "run": {
const code = toStringOrEmpty(args.code);
if (!code) {
jsonError(res, 400, "code is required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
const result = await runCodeViaPlaywright({
cdpPort,
targetId: tab.targetId,
code,
});
res.json({ ok: true, result });
return true;
}
case "click": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {

View File

@@ -4,9 +4,6 @@ import type { BrowserRouteContext } from "../server-context.js";
const pw = vi.hoisted(() => ({
getConsoleMessagesViaPlaywright: vi.fn().mockResolvedValue([]),
mouseClickViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseDragViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseMoveViaPlaywright: vi.fn().mockResolvedValue(undefined),
pdfViaPlaywright: vi.fn().mockResolvedValue({ buffer: Buffer.from("pdf") }),
verifyElementVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
verifyListVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
@@ -149,40 +146,6 @@ describe("handleBrowserActionExtra", () => {
},
expectBody: { ok: true },
},
{
action: "mouseMove" as const,
args: { x: 10, y: 20 },
fn: pw.mouseMoveViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", x: 10, y: 20 },
expectBody: { ok: true },
},
{
action: "mouseClick" as const,
args: { x: 1, y: 2, button: "right" },
fn: pw.mouseClickViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
x: 1,
y: 2,
button: "right",
},
expectBody: { ok: true },
},
{
action: "mouseDrag" as const,
args: { startX: 1, startY: 2, endX: 3, endY: 4 },
fn: pw.mouseDragViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
startX: 1,
startY: 2,
endX: 3,
endY: 4,
},
expectBody: { ok: true },
},
];
for (const item of cases) {

View File

@@ -5,9 +5,6 @@ import type express from "express";
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
getConsoleMessagesViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
pdfViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
@@ -15,26 +12,10 @@ import {
verifyValueViaPlaywright,
} from "../pw-ai.js";
import type { BrowserRouteContext } from "../server-context.js";
import {
jsonError,
toNumber,
toStringArray,
toStringOrEmpty,
} from "./utils.js";
type MouseButton = "left" | "right" | "middle";
function normalizeMouseButton(value: unknown): MouseButton | undefined {
const raw = toStringOrEmpty(value);
if (raw === "left" || raw === "right" || raw === "middle") return raw;
return undefined;
}
import { jsonError, toStringArray, toStringOrEmpty } from "./utils.js";
export type BrowserActionExtra =
| "console"
| "mouseClick"
| "mouseDrag"
| "mouseMove"
| "pdf"
| "verifyElement"
| "verifyList"
@@ -157,68 +138,6 @@ export async function handleBrowserActionExtra(
res.json({ ok: true });
return true;
}
case "mouseMove": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
jsonError(res, 400, "x and y are required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
await mouseMoveViaPlaywright({
cdpPort,
targetId: tab.targetId,
x,
y,
});
res.json({ ok: true });
return true;
}
case "mouseClick": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
jsonError(res, 400, "x and y are required");
return true;
}
const button = normalizeMouseButton(args.button);
const tab = await ctx.ensureTabAvailable(target);
await mouseClickViaPlaywright({
cdpPort,
targetId: tab.targetId,
x,
y,
button,
});
res.json({ ok: true });
return true;
}
case "mouseDrag": {
const startX = toNumber(args.startX);
const startY = toNumber(args.startY);
const endX = toNumber(args.endX);
const endY = toNumber(args.endY);
if (
startX === undefined ||
startY === undefined ||
endX === undefined ||
endY === undefined
) {
jsonError(res, 400, "startX, startY, endX, endY are required");
return true;
}
const tab = await ctx.ensureTabAvailable(target);
await mouseDragViaPlaywright({
cdpPort,
targetId: tab.targetId,
startX,
startY,
endX,
endY,
});
res.json({ ok: true });
return true;
}
default:
return false;
}

View File

@@ -79,12 +79,6 @@ export function registerBrowserActionRoutes(
await runCoreAction(ctx, res, "navigate", body, targetId);
});
app.post("/back", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "back", body, targetId);
});
app.post("/resize", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
@@ -163,12 +157,6 @@ export function registerBrowserActionRoutes(
await runCoreAction(ctx, res, "evaluate", body, targetId);
});
app.post("/run", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "run", body, targetId);
});
app.get("/console", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const level = toStringOrEmpty(req.query.level);
@@ -206,21 +194,5 @@ export function registerBrowserActionRoutes(
await runExtraAction(ctx, res, "verifyValue", body, targetId);
});
app.post("/mouse/move", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseMove", body, targetId);
});
app.post("/mouse/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseClick", body, targetId);
});
app.post("/mouse/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseDrag", body, targetId);
});
// Intentionally no coordinate-based mouse actions (move/click/drag).
}

View File

@@ -5,7 +5,6 @@ import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
captureScreenshot,
captureScreenshotPng,
evaluateJavaScript,
getDomText,
querySelector,
snapshotAria,
@@ -124,45 +123,6 @@ export function registerBrowserInspectRoutes(
}
});
app.post("/eval", async (req, res) => {
const js = toStringOrEmpty((req.body as { js?: unknown })?.js);
const targetId = toStringOrEmpty(
(req.body as { targetId?: unknown })?.targetId,
);
const awaitPromise = Boolean((req.body as { await?: unknown })?.await);
if (!js) return jsonError(res, 400, "js is required");
try {
const tab = await ctx.ensureTabAvailable(targetId || undefined);
const evaluated = await evaluateJavaScript({
wsUrl: tab.wsUrl ?? "",
expression: js,
awaitPromise,
returnByValue: true,
});
if (evaluated.exceptionDetails) {
const msg =
evaluated.exceptionDetails.exception?.description ||
evaluated.exceptionDetails.text ||
"JavaScript evaluation failed";
return jsonError(res, 400, msg);
}
res.json({
ok: true,
targetId: tab.targetId,
url: tab.url,
result: evaluated.result,
});
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
app.get("/query", async (req, res) => {
const selector =
typeof req.query.selector === "string" ? req.query.selector.trim() : "";

View File

@@ -62,22 +62,11 @@ vi.mock("./chrome.js", () => ({
}),
}));
const evalCalls = vi.hoisted(() => [] as Array<string>);
let evalThrows = false;
vi.mock("./cdp.js", () => ({
createTargetViaCdp: vi.fn(async () => {
if (createTargetId) return { targetId: createTargetId };
throw new Error("cdp disabled");
}),
evaluateJavaScript: vi.fn(async ({ expression }: { expression: string }) => {
evalCalls.push(expression);
if (evalThrows) {
return {
exceptionDetails: { text: "boom" },
};
}
return { result: { type: "string", value: "ok" } };
}),
getDomText: vi.fn(async () => ({ text: "<html/>" })),
querySelector: vi.fn(async () => ({ matches: [{ index: 0, tag: "a" }] })),
snapshotAria: vi.fn(async () => ({
@@ -97,28 +86,20 @@ vi.mock("./cdp.js", () => ({
}));
vi.mock("./pw-ai.js", () => ({
armDialogViaPlaywright: vi.fn(async () => {}),
armFileUploadViaPlaywright: vi.fn(async () => {}),
clickRefViaPlaywright: vi.fn(async () => {}),
clickViaPlaywright: vi.fn(async () => {}),
closePageViaPlaywright: vi.fn(async () => {}),
closePlaywrightBrowserConnection: vi.fn(async () => {}),
evaluateViaPlaywright: vi.fn(async () => "ok"),
fileUploadViaPlaywright: vi.fn(async () => {}),
fillFormViaPlaywright: vi.fn(async () => {}),
getConsoleMessagesViaPlaywright: vi.fn(async () => []),
handleDialogViaPlaywright: vi.fn(async () => ({
message: "ok",
type: "alert",
})),
hoverViaPlaywright: vi.fn(async () => {}),
mouseClickViaPlaywright: vi.fn(async () => {}),
mouseDragViaPlaywright: vi.fn(async () => {}),
mouseMoveViaPlaywright: vi.fn(async () => {}),
navigateBackViaPlaywright: vi.fn(async () => ({ url: "about:blank" })),
navigateViaPlaywright: vi.fn(async () => ({ url: "https://example.com" })),
pdfViaPlaywright: vi.fn(async () => ({ buffer: Buffer.from("pdf") })),
pressKeyViaPlaywright: vi.fn(async () => {}),
resizeViewportViaPlaywright: vi.fn(async () => {}),
runCodeViaPlaywright: vi.fn(async () => "ok"),
selectOptionViaPlaywright: vi.fn(async () => {}),
snapshotAiViaPlaywright: vi.fn(async () => ({ snapshot: "ok" })),
takeScreenshotViaPlaywright: vi.fn(async () => ({
@@ -179,6 +160,7 @@ describe("browser control server", () => {
cfgAttachOnly = false;
createTargetId = null;
screenshotThrowsOnce = false;
testPort = await getFreePort();
// Minimal CDP JSON endpoints used by the server.
@@ -288,30 +270,6 @@ describe("browser control server", () => {
expect(focus.status).toBe(409);
});
it("maps JS exceptions to a 400 and returns results otherwise", async () => {
const { startBrowserControlServerFromConfig } = await import("./server.js");
await startBrowserControlServerFromConfig();
const base = `http://127.0.0.1:${testPort}`;
await realFetch(`${base}/start`, { method: "POST" }).then((r) => r.json());
evalThrows = true;
const bad = await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ js: "throw 1" }),
});
expect(bad.status).toBe(400);
evalThrows = false;
const ok = (await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ js: "1+1", await: true }),
}).then((r) => r.json())) as { ok: boolean; result?: unknown };
expect(ok.ok).toBe(true);
expect(evalCalls.length).toBeGreaterThan(0);
});
it("supports query/dom/snapshot/click/screenshot and stop", async () => {
const { startBrowserControlServerFromConfig } = await import("./server.js");
await startBrowserControlServerFromConfig();
@@ -443,13 +401,6 @@ describe("browser control server", () => {
const shotAmbiguous = await realFetch(`${base}/screenshot?targetId=abc`);
expect(shotAmbiguous.status).toBe(409);
const evalMissing = await realFetch(`${base}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({}),
});
expect(evalMissing.status).toBe(400);
const queryMissing = await realFetch(`${base}/query`);
expect(queryMissing.status).toBe(400);

View File

@@ -1,7 +1,6 @@
import type { Command } from "commander";
import { resolveBrowserControlUrl } from "../browser/client.js";
import {
browserBack,
browserClick,
browserDrag,
browserEvaluate,
@@ -11,7 +10,6 @@ import {
browserNavigate,
browserPressKey,
browserResize,
browserRunCode,
browserSelectOption,
browserType,
browserUpload,
@@ -21,31 +19,11 @@ import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js";
import type { BrowserParentOpts } from "./browser-cli-shared.js";
async function readStdin(): Promise<string> {
const chunks: string[] = [];
return await new Promise((resolve, reject) => {
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => chunks.push(String(chunk)));
process.stdin.on("end", () => resolve(chunks.join("")));
process.stdin.on("error", reject);
});
}
async function readFile(path: string): Promise<string> {
const fs = await import("node:fs/promises");
return await fs.readFile(path, "utf8");
}
async function readCode(opts: {
code?: string;
codeFile?: string;
codeStdin?: boolean;
}): Promise<string> {
if (opts.codeFile) return await readFile(opts.codeFile);
if (opts.codeStdin) return await readStdin();
return opts.code ?? "";
}
async function readFields(opts: {
fields?: string;
fieldsFile?: string;
@@ -87,30 +65,6 @@ export function registerBrowserActionInputCommands(
}
});
browser
.command("back")
.description("Navigate back in history")
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const result = await browserBack(baseUrl, {
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(
`navigated back to ${result.url ?? "previous page"}`,
);
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("resize")
.description("Resize the viewport")
@@ -311,7 +265,7 @@ export function registerBrowserActionInputCommands(
browser
.command("upload")
.description("Upload file(s) when a file chooser is open")
.description("Arm file upload for the next file chooser")
.argument("<paths...>", "File paths to upload")
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (paths: string[], opts, cmd) => {
@@ -326,7 +280,7 @@ export function registerBrowserActionInputCommands(
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(`uploaded ${paths.length} file(s)`);
defaultRuntime.log(`upload armed for ${paths.length} file(s)`);
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
@@ -364,7 +318,7 @@ export function registerBrowserActionInputCommands(
browser
.command("dialog")
.description("Handle a modal dialog (alert/confirm/prompt)")
.description("Arm the next modal dialog (alert/confirm/prompt)")
.option("--accept", "Accept the dialog", false)
.option("--dismiss", "Dismiss the dialog", false)
.option("--prompt <text>", "Prompt response text")
@@ -388,7 +342,7 @@ export function registerBrowserActionInputCommands(
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(`dialog handled: ${result.type}`);
defaultRuntime.log("dialog armed");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
@@ -453,40 +407,4 @@ export function registerBrowserActionInputCommands(
defaultRuntime.exit(1);
}
});
browser
.command("run")
.description("Run a Playwright code function (page => ...) ")
.option("--code <code>", "Function source, e.g. (page) => page.title()")
.option("--code-file <path>", "Read function source from a file")
.option("--code-stdin", "Read function source from stdin", false)
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const code = await readCode({
code: opts.code,
codeFile: opts.codeFile,
codeStdin: Boolean(opts.codeStdin),
});
if (!code.trim()) {
defaultRuntime.error(danger("Missing --code (or file/stdin)"));
defaultRuntime.exit(1);
return;
}
const result = await browserRunCode(baseUrl, {
code,
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(JSON.stringify(result.result, null, 2));
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

View File

@@ -2,9 +2,6 @@ import type { Command } from "commander";
import { resolveBrowserControlUrl } from "../browser/client.js";
import {
browserConsoleMessages,
browserMouseClick,
browserMouseDrag,
browserMouseMove,
browserPdfSave,
browserVerifyElementVisible,
browserVerifyListVisible,
@@ -178,110 +175,4 @@ export function registerBrowserActionObserveCommands(
defaultRuntime.exit(1);
}
});
browser
.command("mouse-move")
.description("Move mouse to viewport coordinates")
.option("--x <n>", "X coordinate", (v: string) => Number(v))
.option("--y <n>", "Y coordinate", (v: string) => Number(v))
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
if (!Number.isFinite(opts.x) || !Number.isFinite(opts.y)) {
defaultRuntime.error(danger("--x and --y are required"));
defaultRuntime.exit(1);
return;
}
try {
const result = await browserMouseMove(baseUrl, {
x: opts.x,
y: opts.y,
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log("mouse moved");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("mouse-click")
.description("Click at viewport coordinates")
.option("--x <n>", "X coordinate", (v: string) => Number(v))
.option("--y <n>", "Y coordinate", (v: string) => Number(v))
.option("--button <left|right|middle>", "Mouse button")
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
if (!Number.isFinite(opts.x) || !Number.isFinite(opts.y)) {
defaultRuntime.error(danger("--x and --y are required"));
defaultRuntime.exit(1);
return;
}
try {
const result = await browserMouseClick(baseUrl, {
x: opts.x,
y: opts.y,
button: opts.button?.trim() || undefined,
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log("mouse clicked");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("mouse-drag")
.description("Drag by viewport coordinates")
.option("--start-x <n>", "Start X", (v: string) => Number(v))
.option("--start-y <n>", "Start Y", (v: string) => Number(v))
.option("--end-x <n>", "End X", (v: string) => Number(v))
.option("--end-y <n>", "End Y", (v: string) => Number(v))
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
if (
!Number.isFinite(opts.startX) ||
!Number.isFinite(opts.startY) ||
!Number.isFinite(opts.endX) ||
!Number.isFinite(opts.endY)
) {
defaultRuntime.error(
danger("--start-x, --start-y, --end-x, --end-y are required"),
);
defaultRuntime.exit(1);
return;
}
try {
const result = await browserMouseDrag(baseUrl, {
startX: opts.startX,
startY: opts.startY,
endX: opts.endX,
endY: opts.endY,
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log("mouse dragged");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

View File

@@ -9,7 +9,6 @@ export const browserCoreExamples = [
"clawdis browser screenshot",
"clawdis browser screenshot --full-page",
"clawdis browser screenshot --ref 12",
'clawdis browser eval "document.title"',
'clawdis browser query "a" --limit 5',
"clawdis browser dom --format text --max-chars 5000",
"clawdis browser snapshot --format aria --limit 200",
@@ -18,7 +17,6 @@ export const browserCoreExamples = [
export const browserActionExamples = [
"clawdis browser navigate https://example.com",
"clawdis browser back",
"clawdis browser resize 1280 720",
"clawdis browser click 12 --double",
'clawdis browser type 23 "hello" --submit',
@@ -31,14 +29,10 @@ export const browserActionExamples = [
"clawdis browser dialog --accept",
'clawdis browser wait --text "Done"',
"clawdis browser evaluate --fn '(el) => el.textContent' --ref 7",
"clawdis browser run --code '(page) => page.title()'",
"clawdis browser console --level error",
"clawdis browser pdf",
'clawdis browser verify-element --role button --name "Submit"',
'clawdis browser verify-text "Welcome"',
"clawdis browser verify-list 3 ItemA ItemB",
"clawdis browser verify-value --ref 4 --type textbox --value hello",
"clawdis browser mouse-move --x 120 --y 240",
"clawdis browser mouse-click --x 120 --y 240",
"clawdis browser mouse-drag --start-x 10 --start-y 20 --end-x 200 --end-y 300",
];

View File

@@ -2,7 +2,6 @@ import type { Command } from "commander";
import {
browserDom,
browserEval,
browserQuery,
browserScreenshot,
browserSnapshot,
@@ -13,31 +12,6 @@ import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js";
import type { BrowserParentOpts } from "./browser-cli-shared.js";
async function readStdin(): Promise<string> {
const chunks: string[] = [];
return await new Promise((resolve, reject) => {
process.stdin.setEncoding("utf8");
process.stdin.on("data", (chunk) => chunks.push(String(chunk)));
process.stdin.on("end", () => resolve(chunks.join("")));
process.stdin.on("error", reject);
});
}
async function readTextFromSource(opts: {
js?: string;
jsFile?: string;
jsStdin?: boolean;
}): Promise<string> {
if (opts.jsFile) {
const fs = await import("node:fs/promises");
return await fs.readFile(opts.jsFile, "utf8");
}
if (opts.jsStdin) {
return await readStdin();
}
return opts.js ?? "";
}
export function registerBrowserInspectCommands(
browser: Command,
parentOpts: (cmd: Command) => BrowserParentOpts,
@@ -80,44 +54,6 @@ export function registerBrowserInspectCommands(
}
});
browser
.command("eval")
.description("Run JavaScript in the active tab")
.argument("[js]", "JavaScript expression")
.option("--js-file <path>", "Read JavaScript from a file")
.option("--js-stdin", "Read JavaScript from stdin", false)
.option("--target-id <id>", "CDP target id (or unique prefix)")
.option("--await", "Await promise result", false)
.action(async (js: string | undefined, opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const source = await readTextFromSource({
js,
jsFile: opts.jsFile,
jsStdin: Boolean(opts.jsStdin),
});
if (!source.trim()) {
defaultRuntime.error(danger("Missing JavaScript input."));
defaultRuntime.exit(1);
return;
}
const result = await browserEval(baseUrl, {
js: source,
targetId: opts.targetId?.trim() || undefined,
awaitPromise: Boolean(opts.await),
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(JSON.stringify(result.result, null, 2));
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("query")
.description("Query selector matches")