let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser): add native action commands

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-20 00:53:45 +00:00
parent d67bec0740
commit a526d3c1f2
26 changed files with 2589 additions and 1234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
src/browser/routes/tool-core.test.ts → src/browser/routes/actions-core.test.ts
View File

@@ -21,37 +21,13 @@ const pw = vi.hoisted(() => ({
resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined),
runCodeViaPlaywright: vi.fn().mockResolvedValue("ok"),
selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined),
snapshotAiViaPlaywright: vi
.fn()
.mockResolvedValue({ snapshot: "SNAP" }),
takeScreenshotViaPlaywright: vi
.fn()
.mockResolvedValue({ buffer: Buffer.from("png") }),
typeViaPlaywright: vi.fn().mockResolvedValue(undefined),
waitForViaPlaywright: vi.fn().mockResolvedValue(undefined),
}));
const screenshot = vi.hoisted(() => ({
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES: 128,
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE: 64,
normalizeBrowserScreenshot: vi
.fn()
.mockImplementation(async (buf: Buffer) => ({
buffer: buf,
contentType: "image/png",
})),
}));
const media = vi.hoisted(() => ({
ensureMediaDir: vi.fn().mockResolvedValue(undefined),
saveMediaBuffer: vi.fn().mockResolvedValue({ path: "/tmp/fake.png" }),
}));
vi.mock("../pw-ai.js", () => pw);
vi.mock("../screenshot.js", () => screenshot);
vi.mock("../../media/store.js", () => media);
import { handleBrowserToolCore } from "./tool-core.js";
import { handleBrowserActionCore } from "./actions-core.js";
const baseTab = {
targetId: "tab1",
@@ -84,10 +60,12 @@ function createCtx(
ensureBrowserAvailable: vi.fn().mockResolvedValue(undefined),
ensureTabAvailable: vi.fn().mockResolvedValue(baseTab),
isReachable: vi.fn().mockResolvedValue(true),
listTabs: vi.fn().mockResolvedValue([
baseTab,
{ targetId: "tab2", title: "Two", url: "https://example.com/2" },
]),
listTabs: vi
.fn()
.mockResolvedValue([
baseTab,
{ targetId: "tab2", title: "Two", url: "https://example.com/2" },
]),
openTab: vi.fn().mockResolvedValue({
targetId: "newtab",
title: "",
@@ -102,15 +80,15 @@ function createCtx(
};
}
async function callTool(
name: string,
async function callAction(
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
args: Record<string, unknown> = {},
ctxOverride?: Partial<BrowserRouteContext>,
) {
const res = createRes();
const ctx = createCtx(ctxOverride);
const handled = await handleBrowserToolCore({
name,
const handled = await handleBrowserActionCore({
action,
args,
targetId: "",
cdpPort: 18792,
@@ -124,25 +102,30 @@ beforeEach(() => {
vi.clearAllMocks();
});
describe("handleBrowserToolCore", () => {
it("dispatches core Playwright tools", async () => {
describe("handleBrowserActionCore", () => {
it("dispatches core browser actions", async () => {
const cases = [
{
name: "browser_close",
action: "close" as const,
args: {},
fn: pw.closePageViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_resize",
action: "resize" as const,
args: { width: 800, height: 600 },
fn: pw.resizeViewportViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", width: 800, height: 600 },
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
width: 800,
height: 600,
},
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_handle_dialog",
action: "dialog" as const,
args: { accept: true, promptText: "ok" },
fn: pw.handleDialogViaPlaywright,
expectArgs: {
@@ -154,7 +137,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, message: "ok", type: "alert" },
},
{
name: "browser_evaluate",
action: "evaluate" as const,
args: { function: "() => 1", ref: "1" },
fn: pw.evaluateViaPlaywright,
expectArgs: {
@@ -166,7 +149,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, result: "result" },
},
{
name: "browser_file_upload",
action: "upload" as const,
args: { paths: ["/tmp/file.txt"] },
fn: pw.fileUploadViaPlaywright,
expectArgs: {
@@ -177,7 +160,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_fill_form",
action: "fill" as const,
args: { fields: [{ ref: "1", value: "x" }] },
fn: pw.fillFormViaPlaywright,
expectArgs: {
@@ -188,14 +171,14 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_press_key",
action: "press" as const,
args: { key: "Enter" },
fn: pw.pressKeyViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", key: "Enter" },
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_type",
action: "type" as const,
args: { ref: "2", text: "hi", submit: true, slowly: true },
fn: pw.typeViaPlaywright,
expectArgs: {
@@ -209,7 +192,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_navigate",
action: "navigate" as const,
args: { url: "https://example.com" },
fn: pw.navigateViaPlaywright,
expectArgs: {
@@ -220,21 +203,21 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_navigate_back",
action: "back" as const,
args: {},
fn: pw.navigateBackViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: "about:blank" },
},
{
name: "browser_run_code",
action: "run" as const,
args: { code: "return 1" },
fn: pw.runCodeViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", code: "return 1" },
expectBody: { ok: true, result: "ok" },
},
{
name: "browser_click",
action: "click" as const,
args: {
ref: "1",
doubleClick: true,
@@ -253,7 +236,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_drag",
action: "drag" as const,
args: { startRef: "1", endRef: "2" },
fn: pw.dragViaPlaywright,
expectArgs: {
@@ -265,14 +248,14 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_hover",
action: "hover" as const,
args: { ref: "3" },
fn: pw.hoverViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", ref: "3" },
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_select_option",
action: "select" as const,
args: { ref: "4", values: ["A"] },
fn: pw.selectOptionViaPlaywright,
expectArgs: {
@@ -284,7 +267,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_wait_for",
action: "wait" as const,
args: { time: 500, text: "ok", textGone: "bye" },
fn: pw.waitForViaPlaywright,
expectArgs: {
@@ -299,120 +282,10 @@ describe("handleBrowserToolCore", () => {
];
for (const item of cases) {
const { res, handled } = await callTool(item.name, item.args);
const { res, handled } = await callAction(item.action, item.args);
expect(handled).toBe(true);
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
expect(res.body).toEqual(item.expectBody);
}
});
it("handles screenshots via media storage", async () => {
const { res } = await callTool("browser_take_screenshot", {
type: "jpeg",
ref: "1",
fullPage: true,
element: "main",
filename: "shot.jpg",
});
expect(pw.takeScreenshotViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
ref: "1",
element: "main",
fullPage: true,
type: "jpeg",
});
expect(media.ensureMediaDir).toHaveBeenCalled();
expect(media.saveMediaBuffer).toHaveBeenCalled();
expect(res.body).toMatchObject({
ok: true,
path: "/tmp/fake.png",
filename: "shot.jpg",
targetId: "tab1",
url: baseTab.url,
});
});
it("handles snapshots with optional file output", async () => {
const { res } = await callTool("browser_snapshot", {
filename: "snapshot.txt",
});
expect(pw.snapshotAiViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
});
expect(media.ensureMediaDir).toHaveBeenCalled();
expect(media.saveMediaBuffer).toHaveBeenCalledWith(
expect.any(Buffer),
"text/plain",
"browser",
);
expect(res.body).toMatchObject({
ok: true,
path: "/tmp/fake.png",
filename: "snapshot.txt",
targetId: "tab1",
url: baseTab.url,
});
});
it("returns a message for browser_install", async () => {
const { res } = await callTool("browser_install");
expect(res.body).toMatchObject({ ok: true });
});
it("supports browser_tabs actions", async () => {
const ctx = createCtx();
const listRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "list" },
targetId: "",
cdpPort: 18792,
ctx,
res: listRes,
});
expect(listRes.body).toMatchObject({ ok: true });
expect(ctx.listTabs).toHaveBeenCalled();
const newRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "new" },
targetId: "",
cdpPort: 18792,
ctx,
res: newRes,
});
expect(ctx.ensureBrowserAvailable).toHaveBeenCalled();
expect(ctx.openTab).toHaveBeenCalled();
expect(newRes.body).toMatchObject({ ok: true, tab: { targetId: "newtab" } });
const closeRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "close", index: 1 },
targetId: "",
cdpPort: 18792,
ctx,
res: closeRes,
});
expect(ctx.closeTab).toHaveBeenCalledWith("tab2");
expect(closeRes.body).toMatchObject({ ok: true, targetId: "tab2" });
const selectRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "select", index: 0 },
targetId: "",
cdpPort: 18792,
ctx,
res: selectRes,
});
expect(ctx.focusTab).toHaveBeenCalledWith("tab1");
expect(selectRes.body).toMatchObject({ ok: true, targetId: "tab1" });
});
});

198
src/browser/routes/tool-core.ts → src/browser/routes/actions-core.ts
View File

@@ -1,8 +1,5 @@
import path from "node:path";
import type express from "express";
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
clickViaPlaywright,
closePageViaPlaywright,
@@ -18,16 +15,9 @@ import {
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
snapshotAiViaPlaywright,
takeScreenshotViaPlaywright,
typeViaPlaywright,
waitForViaPlaywright,
} from "../pw-ai.js";
import {
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
normalizeBrowserScreenshot,
} from "../screenshot.js";
import type { BrowserRouteContext } from "../server-context.js";
import {
jsonError,
@@ -37,8 +27,26 @@ import {
toStringOrEmpty,
} from "./utils.js";
type ToolCoreParams = {
name: string;
export type BrowserActionCore =
| "back"
| "click"
| "close"
| "dialog"
| "drag"
| "evaluate"
| "fill"
| "hover"
| "navigate"
| "press"
| "resize"
| "run"
| "select"
| "type"
| "upload"
| "wait";
type ActionCoreParams = {
action: BrowserActionCore;
args: Record<string, unknown>;
targetId: string;
cdpPort: number;
@@ -46,20 +54,20 @@ type ToolCoreParams = {
res: express.Response;
};
export async function handleBrowserToolCore(
params: ToolCoreParams,
export async function handleBrowserActionCore(
params: ActionCoreParams,
): Promise<boolean> {
const { name, args, targetId, cdpPort, ctx, res } = params;
const { action, args, targetId, cdpPort, ctx, res } = params;
const target = targetId || undefined;
switch (name) {
case "browser_close": {
switch (action) {
case "close": {
const tab = await ctx.ensureTabAvailable(target);
await closePageViaPlaywright({ cdpPort, targetId: tab.targetId });
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_resize": {
case "resize": {
const width = toNumber(args.width);
const height = toNumber(args.height);
if (!width || !height) {
@@ -76,7 +84,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_handle_dialog": {
case "dialog": {
const accept = toBoolean(args.accept);
if (accept === undefined) {
jsonError(res, 400, "accept is required");
@@ -93,7 +101,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, ...result });
return true;
}
case "browser_evaluate": {
case "evaluate": {
const fn = toStringOrEmpty(args.function);
if (!fn) {
jsonError(res, 400, "function is required");
@@ -110,7 +118,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, result });
return true;
}
case "browser_file_upload": {
case "upload": {
const paths = toStringArray(args.paths) ?? [];
const tab = await ctx.ensureTabAvailable(target);
await fileUploadViaPlaywright({
@@ -121,7 +129,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_fill_form": {
case "fill": {
const fields = Array.isArray(args.fields)
? (args.fields as Array<Record<string, unknown>>)
: null;
@@ -138,15 +146,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_install": {
res.json({
ok: true,
message:
"clawd browser uses system Chrome/Chromium; no Playwright install needed.",
});
return true;
}
case "browser_press_key": {
case "press": {
const key = toStringOrEmpty(args.key);
if (!key) {
jsonError(res, 400, "key is required");
@@ -161,7 +161,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_type": {
case "type": {
const ref = toStringOrEmpty(args.ref);
const text = toStringOrEmpty(args.text);
if (!ref || !text) {
@@ -182,7 +182,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_navigate": {
case "navigate": {
const url = toStringOrEmpty(args.url);
if (!url) {
jsonError(res, 400, "url is required");
@@ -197,7 +197,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "browser_navigate_back": {
case "back": {
const tab = await ctx.ensureTabAvailable(target);
const result = await navigateBackViaPlaywright({
cdpPort,
@@ -206,7 +206,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "browser_run_code": {
case "run": {
const code = toStringOrEmpty(args.code);
if (!code) {
jsonError(res, 400, "code is required");
@@ -221,73 +221,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, result });
return true;
}
case "browser_take_screenshot": {
const type = args.type === "jpeg" ? "jpeg" : "png";
const ref = toStringOrEmpty(args.ref) || undefined;
const fullPage = toBoolean(args.fullPage) ?? false;
const element = toStringOrEmpty(args.element) || undefined;
const filename = toStringOrEmpty(args.filename) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const snap = await takeScreenshotViaPlaywright({
cdpPort,
targetId: tab.targetId,
ref,
element,
fullPage,
type,
});
const normalized = await normalizeBrowserScreenshot(snap.buffer, {
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
});
await ensureMediaDir();
const saved = await saveMediaBuffer(
normalized.buffer,
normalized.contentType ?? `image/${type}`,
"browser",
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
);
res.json({
ok: true,
path: path.resolve(saved.path),
filename,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
case "browser_snapshot": {
const filename = toStringOrEmpty(args.filename) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const snap = await snapshotAiViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
if (filename) {
await ensureMediaDir();
const saved = await saveMediaBuffer(
Buffer.from(snap.snapshot, "utf8"),
"text/plain",
"browser",
);
res.json({
ok: true,
path: path.resolve(saved.path),
filename,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
res.json({
ok: true,
snapshot: snap.snapshot,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
case "browser_click": {
case "click": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");
@@ -310,7 +244,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_drag": {
case "drag": {
const startRef = toStringOrEmpty(args.startRef);
const endRef = toStringOrEmpty(args.endRef);
if (!startRef || !endRef) {
@@ -327,7 +261,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_hover": {
case "hover": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");
@@ -342,7 +276,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_select_option": {
case "select": {
const ref = toStringOrEmpty(args.ref);
const values = toStringArray(args.values);
if (!ref || !values?.length) {
@@ -359,59 +293,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_tabs": {
const action = toStringOrEmpty(args.action);
const index = toNumber(args.index);
if (!action) {
jsonError(res, 400, "action is required");
return true;
}
if (action === "list") {
const reachable = await ctx.isReachable(300);
if (!reachable) {
res.json({ ok: true, tabs: [] });
return true;
}
const tabs = await ctx.listTabs();
res.json({ ok: true, tabs });
return true;
}
if (action === "new") {
await ctx.ensureBrowserAvailable();
const tab = await ctx.openTab("about:blank");
res.json({ ok: true, tab });
return true;
}
if (action === "close") {
const tabs = await ctx.listTabs();
const targetTab = typeof index === "number" ? tabs[index] : tabs.at(0);
if (!targetTab) {
jsonError(res, 404, "tab not found");
return true;
}
await ctx.closeTab(targetTab.targetId);
res.json({ ok: true, targetId: targetTab.targetId });
return true;
}
if (action === "select") {
if (typeof index !== "number") {
jsonError(res, 400, "index is required");
return true;
}
const tabs = await ctx.listTabs();
const targetTab = tabs[index];
if (!targetTab) {
jsonError(res, 404, "tab not found");
return true;
}
await ctx.focusTab(targetTab.targetId);
res.json({ ok: true, targetId: targetTab.targetId });
return true;
}
jsonError(res, 400, "unknown tab action");
return true;
}
case "browser_wait_for": {
case "wait": {
const time = toNumber(args.time);
const text = toStringOrEmpty(args.text) || undefined;
const textGone = toStringOrEmpty(args.textGone) || undefined;

43
src/browser/routes/tool-extra.test.ts → src/browser/routes/actions-extra.test.ts
View File

@@ -30,7 +30,7 @@ const media = vi.hoisted(() => ({
vi.mock("../pw-ai.js", () => pw);
vi.mock("../../media/store.js", () => media);
import { handleBrowserToolExtra } from "./tool-extra.js";
import { handleBrowserActionExtra } from "./actions-extra.js";
const baseTab = {
targetId: "tab1",
@@ -78,11 +78,14 @@ function createCtx(
};
}
async function callTool(name: string, args: Record<string, unknown> = {}) {
async function callAction(
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
args: Record<string, unknown> = {},
) {
const res = createRes();
const ctx = createCtx();
const handled = await handleBrowserToolExtra({
name,
const handled = await handleBrowserActionExtra({
action,
args,
targetId: "",
cdpPort: 18792,
@@ -96,11 +99,11 @@ beforeEach(() => {
vi.clearAllMocks();
});
describe("handleBrowserToolExtra", () => {
it("dispatches extra Playwright tools", async () => {
describe("handleBrowserActionExtra", () => {
it("dispatches extra browser actions", async () => {
const cases = [
{
name: "browser_console_messages",
action: "console" as const,
args: { level: "error" },
fn: pw.getConsoleMessagesViaPlaywright,
expectArgs: {
@@ -111,7 +114,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true, messages: [], targetId: "tab1" },
},
{
name: "browser_network_requests",
action: "network" as const,
args: { includeStatic: true },
fn: pw.getNetworkRequestsViaPlaywright,
expectArgs: {
@@ -122,14 +125,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true, requests: [], targetId: "tab1" },
},
{
name: "browser_start_tracing",
action: "traceStart" as const,
args: {},
fn: pw.startTracingViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true },
},
{
name: "browser_verify_element_visible",
action: "verifyElement" as const,
args: { role: "button", accessibleName: "Submit" },
fn: pw.verifyElementVisibleViaPlaywright,
expectArgs: {
@@ -141,14 +144,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_verify_text_visible",
action: "verifyText" as const,
args: { text: "Hello" },
fn: pw.verifyTextVisibleViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", text: "Hello" },
expectBody: { ok: true },
},
{
name: "browser_verify_list_visible",
action: "verifyList" as const,
args: { ref: "1", items: ["a", "b"] },
fn: pw.verifyListVisibleViaPlaywright,
expectArgs: {
@@ -160,7 +163,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_verify_value",
action: "verifyValue" as const,
args: { ref: "2", type: "textbox", value: "x" },
fn: pw.verifyValueViaPlaywright,
expectArgs: {
@@ -173,14 +176,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_mouse_move_xy",
action: "mouseMove" as const,
args: { x: 10, y: 20 },
fn: pw.mouseMoveViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", x: 10, y: 20 },
expectBody: { ok: true },
},
{
name: "browser_mouse_click_xy",
action: "mouseClick" as const,
args: { x: 1, y: 2, button: "right" },
fn: pw.mouseClickViaPlaywright,
expectArgs: {
@@ -193,7 +196,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_mouse_drag_xy",
action: "mouseDrag" as const,
args: { startX: 1, startY: 2, endX: 3, endY: 4 },
fn: pw.mouseDragViaPlaywright,
expectArgs: {
@@ -207,7 +210,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_generate_locator",
action: "locator" as const,
args: { ref: "99" },
fn: pw.generateLocatorForRef,
expectArgs: "99",
@@ -216,7 +219,7 @@ describe("handleBrowserToolExtra", () => {
];
for (const item of cases) {
const { res, handled } = await callTool(item.name, item.args);
const { res, handled } = await callAction(item.action, item.args);
expect(handled).toBe(true);
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
expect(res.body).toEqual(item.expectBody);
@@ -224,7 +227,7 @@ describe("handleBrowserToolExtra", () => {
});
it("stores PDF and trace outputs", async () => {
const { res: pdfRes } = await callTool("browser_pdf_save");
const { res: pdfRes } = await callAction("pdf");
expect(pw.pdfViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
@@ -239,7 +242,7 @@ describe("handleBrowserToolExtra", () => {
});
media.saveMediaBuffer.mockResolvedValueOnce({ path: "/tmp/fake.zip" });
const { res: traceRes } = await callTool("browser_stop_tracing");
const { res: traceRes } = await callAction("traceStop");
expect(pw.stopTracingViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",

53
src/browser/routes/tool-extra.ts → src/browser/routes/actions-extra.ts
View File

@@ -27,8 +27,23 @@ import {
toStringOrEmpty,
} from "./utils.js";
type ToolExtraParams = {
name: string;
export type BrowserActionExtra =
| "console"
| "locator"
| "mouseClick"
| "mouseDrag"
| "mouseMove"
| "network"
| "pdf"
| "traceStart"
| "traceStop"
| "verifyElement"
| "verifyList"
| "verifyText"
| "verifyValue";
type ActionExtraParams = {
action: BrowserActionExtra;
args: Record<string, unknown>;
targetId: string;
cdpPort: number;
@@ -36,14 +51,14 @@ type ToolExtraParams = {
res: express.Response;
};
export async function handleBrowserToolExtra(
params: ToolExtraParams,
export async function handleBrowserActionExtra(
params: ActionExtraParams,
): Promise<boolean> {
const { name, args, targetId, cdpPort, ctx, res } = params;
const { action, args, targetId, cdpPort, ctx, res } = params;
const target = targetId || undefined;
switch (name) {
case "browser_console_messages": {
switch (action) {
case "console": {
const level = toStringOrEmpty(args.level) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const messages = await getConsoleMessagesViaPlaywright({
@@ -54,7 +69,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true, messages, targetId: tab.targetId });
return true;
}
case "browser_network_requests": {
case "network": {
const includeStatic = toBoolean(args.includeStatic) ?? false;
const tab = await ctx.ensureTabAvailable(target);
const requests = await getNetworkRequestsViaPlaywright({
@@ -65,7 +80,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true, requests, targetId: tab.targetId });
return true;
}
case "browser_pdf_save": {
case "pdf": {
const tab = await ctx.ensureTabAvailable(target);
const pdf = await pdfViaPlaywright({
cdpPort,
@@ -86,7 +101,7 @@ export async function handleBrowserToolExtra(
});
return true;
}
case "browser_start_tracing": {
case "traceStart": {
const tab = await ctx.ensureTabAvailable(target);
await startTracingViaPlaywright({
cdpPort,
@@ -95,7 +110,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_stop_tracing": {
case "traceStop": {
const tab = await ctx.ensureTabAvailable(target);
const trace = await stopTracingViaPlaywright({
cdpPort,
@@ -116,7 +131,7 @@ export async function handleBrowserToolExtra(
});
return true;
}
case "browser_verify_element_visible": {
case "verifyElement": {
const role = toStringOrEmpty(args.role);
const accessibleName = toStringOrEmpty(args.accessibleName);
if (!role || !accessibleName) {
@@ -133,7 +148,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_text_visible": {
case "verifyText": {
const text = toStringOrEmpty(args.text);
if (!text) {
jsonError(res, 400, "text is required");
@@ -148,7 +163,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_list_visible": {
case "verifyList": {
const ref = toStringOrEmpty(args.ref);
const items = toStringArray(args.items);
if (!ref || !items?.length) {
@@ -165,7 +180,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_value": {
case "verifyValue": {
const ref = toStringOrEmpty(args.ref);
const type = toStringOrEmpty(args.type);
const value = toStringOrEmpty(args.value);
@@ -184,7 +199,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_move_xy": {
case "mouseMove": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
@@ -201,7 +216,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_click_xy": {
case "mouseClick": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
@@ -220,7 +235,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_drag_xy": {
case "mouseDrag": {
const startX = toNumber(args.startX);
const startY = toNumber(args.startY);
const endX = toNumber(args.endX);
@@ -246,7 +261,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_generate_locator": {
case "locator": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");

249
src/browser/routes/actions.ts Normal file
View File

@@ -0,0 +1,249 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { handleBrowserActionCore } from "./actions-core.js";
import { handleBrowserActionExtra } from "./actions-extra.js";
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
function readBody(req: express.Request): Record<string, unknown> {
const body = req.body as Record<string, unknown> | undefined;
if (!body || typeof body !== "object" || Array.isArray(body)) return {};
return body;
}
function readTargetId(value: unknown): string {
return toStringOrEmpty(value);
}
function handleActionError(
ctx: BrowserRouteContext,
res: express.Response,
err: unknown,
) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
async function runCoreAction(
ctx: BrowserRouteContext,
res: express.Response,
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
args: Record<string, unknown>,
targetId: string,
) {
try {
const cdpPort = ctx.state().cdpPort;
await handleBrowserActionCore({
action,
args,
targetId,
cdpPort,
ctx,
res,
});
} catch (err) {
handleActionError(ctx, res, err);
}
}
async function runExtraAction(
ctx: BrowserRouteContext,
res: express.Response,
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
args: Record<string, unknown>,
targetId: string,
) {
try {
const cdpPort = ctx.state().cdpPort;
await handleBrowserActionExtra({
action,
args,
targetId,
cdpPort,
ctx,
res,
});
} catch (err) {
handleActionError(ctx, res, err);
}
}
export function registerBrowserActionRoutes(
app: express.Express,
ctx: BrowserRouteContext,
) {
app.post("/navigate", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "navigate", body, targetId);
});
app.post("/back", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "back", body, targetId);
});
app.post("/resize", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "resize", body, targetId);
});
app.post("/close", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "close", body, targetId);
});
app.post("/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "click", body, targetId);
});
app.post("/type", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "type", body, targetId);
});
app.post("/press", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "press", body, targetId);
});
app.post("/hover", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "hover", body, targetId);
});
app.post("/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "drag", body, targetId);
});
app.post("/select", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "select", body, targetId);
});
app.post("/upload", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "upload", body, targetId);
});
app.post("/fill", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "fill", body, targetId);
});
app.post("/dialog", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "dialog", body, targetId);
});
app.post("/wait", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "wait", body, targetId);
});
app.post("/evaluate", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "evaluate", body, targetId);
});
app.post("/run", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "run", body, targetId);
});
app.get("/console", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const level = toStringOrEmpty(req.query.level);
const args = level ? { level } : {};
await runExtraAction(ctx, res, "console", args, targetId);
});
app.get("/network", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const includeStatic = toBoolean(req.query.includeStatic) ?? false;
await runExtraAction(ctx, res, "network", { includeStatic }, targetId);
});
app.post("/trace/start", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStart", body, targetId);
});
app.post("/trace/stop", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStop", body, targetId);
});
app.post("/pdf", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "pdf", body, targetId);
});
app.post("/verify/element", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyElement", body, targetId);
});
app.post("/verify/text", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyText", body, targetId);
});
app.post("/verify/list", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyList", body, targetId);
});
app.post("/verify/value", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyValue", body, targetId);
});
app.post("/mouse/move", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseMove", body, targetId);
});
app.post("/mouse/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseClick", body, targetId);
});
app.post("/mouse/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseDrag", body, targetId);
});
app.post("/locator", async (req, res) => {
const body = readBody(req);
await runExtraAction(ctx, res, "locator", body, "");
});
}

4
src/browser/routes/index.ts
View File

@@ -1,10 +1,10 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { registerBrowserActionRoutes } from "./actions.js";
import { registerBrowserBasicRoutes } from "./basic.js";
import { registerBrowserInspectRoutes } from "./inspect.js";
import { registerBrowserTabRoutes } from "./tabs.js";
import { registerBrowserToolRoutes } from "./tool.js";
export function registerBrowserRoutes(
app: express.Express,
@@ -13,5 +13,5 @@ export function registerBrowserRoutes(
registerBrowserBasicRoutes(app, ctx);
registerBrowserTabRoutes(app, ctx);
registerBrowserInspectRoutes(app, ctx);
registerBrowserToolRoutes(app, ctx);
registerBrowserActionRoutes(app, ctx);
}

23
src/browser/routes/inspect.ts
View File

@@ -281,27 +281,4 @@ export function registerBrowserInspectRoutes(
jsonError(res, 500, String(err));
}
});
app.post("/click", async (req, res) => {
const ref = toStringOrEmpty((req.body as { ref?: unknown })?.ref);
const targetId = toStringOrEmpty(
(req.body as { targetId?: unknown })?.targetId,
);
if (!ref) return jsonError(res, 400, "ref is required");
try {
const tab = await ctx.ensureTabAvailable(targetId || undefined);
await clickViaPlaywright({
cdpPort: ctx.state().cdpPort,
targetId: tab.targetId,
ref,
});
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
}

65
src/browser/routes/tool.ts
View File

@@ -1,65 +0,0 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { handleBrowserToolCore } from "./tool-core.js";
import { handleBrowserToolExtra } from "./tool-extra.js";
import { jsonError, toStringOrEmpty } from "./utils.js";
type ToolRequestBody = {
name?: unknown;
args?: unknown;
targetId?: unknown;
};
function toolArgs(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
return value as Record<string, unknown>;
}
export function registerBrowserToolRoutes(
app: express.Express,
ctx: BrowserRouteContext,
) {
app.post("/tool", async (req, res) => {
const body = req.body as ToolRequestBody;
const name = toStringOrEmpty(body?.name);
if (!name) return jsonError(res, 400, "name is required");
const args = toolArgs(body?.args);
const targetId = toStringOrEmpty(body?.targetId || args?.targetId);
try {
let cdpPort: number;
try {
cdpPort = ctx.state().cdpPort;
} catch {
return jsonError(res, 503, "browser server not started");
}
const handledCore = await handleBrowserToolCore({
name,
args,
targetId,
cdpPort,
ctx,
res,
});
if (handledCore) return;
const handledExtra = await handleBrowserToolExtra({
name,
args,
targetId,
cdpPort,
ctx,
res,
});
if (handledExtra) return;
return jsonError(res, 400, "unknown tool name");
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
}