let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(browser): add native action commands

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-20 00:53:45 +00:00
parent d67bec0740
commit a526d3c1f2
26 changed files with 2589 additions and 1234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
src/browser/client-actions-core.ts Normal file
View File

@@ -0,0 +1,287 @@
import type { ScreenshotResult } from "./client.js";
import type { BrowserActionTabResult } from "./client-actions-types.js";
import { fetchBrowserJson } from "./client-fetch.js";
export async function browserNavigate(
baseUrl: string,
opts: { url: string; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/navigate`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url: opts.url, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserBack(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/back`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserResize(
baseUrl: string,
opts: { width: number; height: number; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/resize`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
width: opts.width,
height: opts.height,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserClosePage(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/close`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserClick(
baseUrl: string,
opts: {
ref: string;
targetId?: string;
doubleClick?: boolean;
button?: string;
modifiers?: string[];
},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/click`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ref: opts.ref,
targetId: opts.targetId,
doubleClick: opts.doubleClick,
button: opts.button,
modifiers: opts.modifiers,
}),
timeoutMs: 20000,
});
}
export async function browserType(
baseUrl: string,
opts: {
ref: string;
text: string;
targetId?: string;
submit?: boolean;
slowly?: boolean;
},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/type`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ref: opts.ref,
text: opts.text,
targetId: opts.targetId,
submit: opts.submit,
slowly: opts.slowly,
}),
timeoutMs: 20000,
});
}
export async function browserPressKey(
baseUrl: string,
opts: { key: string; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/press`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ key: opts.key, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserHover(
baseUrl: string,
opts: { ref: string; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/hover`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ref: opts.ref, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserDrag(
baseUrl: string,
opts: { startRef: string; endRef: string; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/drag`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
startRef: opts.startRef,
endRef: opts.endRef,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserSelectOption(
baseUrl: string,
opts: { ref: string; values: string[]; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/select`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ref: opts.ref,
values: opts.values,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserUpload(
baseUrl: string,
opts: { paths?: string[]; targetId?: string } = {},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/upload`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ paths: opts.paths, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserFillForm(
baseUrl: string,
opts: { fields: Array<Record<string, unknown>>; targetId?: string },
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/fill`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ fields: opts.fields, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserHandleDialog(
baseUrl: string,
opts: { accept: boolean; promptText?: string; targetId?: string },
): Promise<{ ok: true; message: string; type: string }> {
return await fetchBrowserJson<{ ok: true; message: string; type: string }>(
`${baseUrl}/dialog`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
accept: opts.accept,
promptText: opts.promptText,
targetId: opts.targetId,
}),
timeoutMs: 20000,
},
);
}
export async function browserWaitFor(
baseUrl: string,
opts: {
time?: number;
text?: string;
textGone?: string;
targetId?: string;
},
): Promise<BrowserActionTabResult> {
return await fetchBrowserJson<BrowserActionTabResult>(`${baseUrl}/wait`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
time: opts.time,
text: opts.text,
textGone: opts.textGone,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserEvaluate(
baseUrl: string,
opts: { fn: string; ref?: string; targetId?: string },
): Promise<{ ok: true; result: unknown }> {
return await fetchBrowserJson<{ ok: true; result: unknown }>(
`${baseUrl}/evaluate`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
function: opts.fn,
ref: opts.ref,
targetId: opts.targetId,
}),
timeoutMs: 20000,
},
);
}
export async function browserRunCode(
baseUrl: string,
opts: { code: string; targetId?: string },
): Promise<{ ok: true; result: unknown }> {
return await fetchBrowserJson<{ ok: true; result: unknown }>(
`${baseUrl}/run`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ code: opts.code, targetId: opts.targetId }),
timeoutMs: 20000,
},
);
}
export async function browserScreenshotAction(
baseUrl: string,
opts: {
targetId?: string;
fullPage?: boolean;
ref?: string;
element?: string;
type?: "png" | "jpeg";
filename?: string;
},
): Promise<ScreenshotResult & { filename?: string }> {
return await fetchBrowserJson<ScreenshotResult & { filename?: string }>(
`${baseUrl}/screenshot`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
targetId: opts.targetId,
fullPage: opts.fullPage,
ref: opts.ref,
element: opts.element,
type: opts.type,
filename: opts.filename,
}),
timeoutMs: 20000,
},
);
}

207
src/browser/client-actions-observe.ts Normal file
View File

@@ -0,0 +1,207 @@
import type {
BrowserActionOk,
BrowserActionPathResult,
} from "./client-actions-types.js";
import { fetchBrowserJson } from "./client-fetch.js";
import type {
BrowserConsoleMessage,
BrowserNetworkRequest,
} from "./pw-session.js";
export async function browserConsoleMessages(
baseUrl: string,
opts: { level?: string; targetId?: string } = {},
): Promise<{ ok: true; messages: BrowserConsoleMessage[]; targetId: string }> {
const q = new URLSearchParams();
if (opts.level) q.set("level", opts.level);
if (opts.targetId) q.set("targetId", opts.targetId);
const suffix = q.toString() ? `?${q.toString()}` : "";
return await fetchBrowserJson<{
ok: true;
messages: BrowserConsoleMessage[];
targetId: string;
}>(`${baseUrl}/console${suffix}`, { timeoutMs: 20000 });
}
export async function browserNetworkRequests(
baseUrl: string,
opts: { includeStatic?: boolean; targetId?: string } = {},
): Promise<{ ok: true; requests: BrowserNetworkRequest[]; targetId: string }> {
const q = new URLSearchParams();
if (opts.includeStatic) q.set("includeStatic", "true");
if (opts.targetId) q.set("targetId", opts.targetId);
const suffix = q.toString() ? `?${q.toString()}` : "";
return await fetchBrowserJson<{
ok: true;
requests: BrowserNetworkRequest[];
targetId: string;
}>(`${baseUrl}/network${suffix}`, { timeoutMs: 20000 });
}
export async function browserStartTracing(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/trace/start`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserStopTracing(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionPathResult> {
return await fetchBrowserJson<BrowserActionPathResult>(
`${baseUrl}/trace/stop`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
},
);
}
export async function browserPdfSave(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionPathResult> {
return await fetchBrowserJson<BrowserActionPathResult>(`${baseUrl}/pdf`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserVerifyElementVisible(
baseUrl: string,
opts: { role: string; accessibleName: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/verify/element`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
role: opts.role,
accessibleName: opts.accessibleName,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserVerifyTextVisible(
baseUrl: string,
opts: { text: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/verify/text`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ text: opts.text, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserVerifyListVisible(
baseUrl: string,
opts: { ref: string; items: string[]; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/verify/list`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ref: opts.ref,
items: opts.items,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserVerifyValue(
baseUrl: string,
opts: { ref: string; type: string; value?: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/verify/value`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
ref: opts.ref,
type: opts.type,
value: opts.value,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserMouseMove(
baseUrl: string,
opts: { x: number; y: number; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/move`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ x: opts.x, y: opts.y, targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserMouseClick(
baseUrl: string,
opts: { x: number; y: number; button?: string; targetId?: string },
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/click`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
x: opts.x,
y: opts.y,
button: opts.button,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserMouseDrag(
baseUrl: string,
opts: {
startX: number;
startY: number;
endX: number;
endY: number;
targetId?: string;
},
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/mouse/drag`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
startX: opts.startX,
startY: opts.startY,
endX: opts.endX,
endY: opts.endY,
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
export async function browserGenerateLocator(
baseUrl: string,
opts: { ref: string },
): Promise<{ ok: true; locator: string }> {
return await fetchBrowserJson<{ ok: true; locator: string }>(
`${baseUrl}/locator`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ref: opts.ref }),
timeoutMs: 20000,
},
);
}

15
src/browser/client-actions-types.ts Normal file
View File

@@ -0,0 +1,15 @@
export type BrowserActionOk = { ok: true };
export type BrowserActionTabResult = {
ok: true;
targetId: string;
url?: string;
};
export type BrowserActionPathResult = {
ok: true;
path: string;
targetId: string;
url?: string;
filename?: string;
};

3
src/browser/client-actions.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from "./client-actions-core.js";
export * from "./client-actions-observe.js";
export * from "./client-actions-types.js";

67
src/browser/client-fetch.ts Normal file
View File

@@ -0,0 +1,67 @@
function unwrapCause(err: unknown): unknown {
if (!err || typeof err !== "object") return null;
const cause = (err as { cause?: unknown }).cause;
return cause ?? null;
}
function enhanceBrowserFetchError(
url: string,
err: unknown,
timeoutMs: number,
): Error {
const cause = unwrapCause(err);
const code =
(cause && typeof cause === "object" && "code" in cause
? String((cause as { code?: unknown }).code ?? "")
: "") ||
(err && typeof err === "object" && "code" in err
? String((err as { code?: unknown }).code ?? "")
: "");
const hint =
"Start (or restart) the Clawdis gateway (Clawdis.app menubar, or `clawdis gateway`) and try again.";
if (code === "ECONNREFUSED") {
return new Error(
`Can't reach the clawd browser control server at ${url} (connection refused). ${hint}`,
);
}
if (code === "ETIMEDOUT" || code === "UND_ERR_CONNECT_TIMEOUT") {
return new Error(
`Can't reach the clawd browser control server at ${url} (timed out after ${timeoutMs}ms). ${hint}`,
);
}
const msg = String(err);
if (msg.toLowerCase().includes("abort")) {
return new Error(
`Can't reach the clawd browser control server at ${url} (timed out after ${timeoutMs}ms). ${hint}`,
);
}
return new Error(
`Can't reach the clawd browser control server at ${url}. ${hint} (${msg})`,
);
}
export async function fetchBrowserJson<T>(
url: string,
init?: RequestInit & { timeoutMs?: number },
): Promise<T> {
const timeoutMs = init?.timeoutMs ?? 5000;
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), timeoutMs);
let res: Response;
try {
res = await fetch(url, { ...init, signal: ctrl.signal } as RequestInit);
} catch (err) {
throw enhanceBrowserFetchError(url, err, timeoutMs);
} finally {
clearTimeout(t);
}
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(text ? `${res.status}: ${text}` : `HTTP ${res.status}`);
}
return (await res.json()) as T;
}

142
src/browser/client.ts
View File

@@ -1,4 +1,5 @@
import { loadConfig } from "../config/config.js";
import { fetchBrowserJson } from "./client-fetch.js";
import { resolveBrowserConfig } from "./config.js";
export type BrowserStatus = {
@@ -21,11 +22,6 @@ export type BrowserTab = {
type?: string;
};
export type BrowserToolResponse = {
ok: true;
[key: string]: unknown;
};
export type ScreenshotResult = {
ok: true;
path: string;
@@ -117,74 +113,6 @@ export type SnapshotResult =
snapshot: string;
};
function unwrapCause(err: unknown): unknown {
if (!err || typeof err !== "object") return null;
const cause = (err as { cause?: unknown }).cause;
return cause ?? null;
}
function enhanceBrowserFetchError(
url: string,
err: unknown,
timeoutMs: number,
): Error {
const cause = unwrapCause(err);
const code =
(cause && typeof cause === "object" && "code" in cause
? String((cause as { code?: unknown }).code ?? "")
: "") ||
(err && typeof err === "object" && "code" in err
? String((err as { code?: unknown }).code ?? "")
: "");
const hint =
"Start (or restart) the Clawdis gateway (Clawdis.app menubar, or `clawdis gateway`) and try again.";
if (code === "ECONNREFUSED") {
return new Error(
`Can't reach the clawd browser control server at ${url} (connection refused). ${hint}`,
);
}
if (code === "ETIMEDOUT" || code === "UND_ERR_CONNECT_TIMEOUT") {
return new Error(
`Can't reach the clawd browser control server at ${url} (timed out after ${timeoutMs}ms). ${hint}`,
);
}
const msg = String(err);
if (msg.toLowerCase().includes("abort")) {
return new Error(
`Can't reach the clawd browser control server at ${url} (timed out after ${timeoutMs}ms). ${hint}`,
);
}
return new Error(
`Can't reach the clawd browser control server at ${url}. ${hint} (${msg})`,
);
}
async function fetchJson<T>(
url: string,
init?: RequestInit & { timeoutMs?: number },
): Promise<T> {
const timeoutMs = init?.timeoutMs ?? 5000;
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), timeoutMs);
let res: Response;
try {
res = await fetch(url, { ...init, signal: ctrl.signal } as RequestInit);
} catch (err) {
throw enhanceBrowserFetchError(url, err, timeoutMs);
} finally {
clearTimeout(t);
}
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new Error(text ? `${res.status}: ${text}` : `HTTP ${res.status}`);
}
return (await res.json()) as T;
}
export function resolveBrowserControlUrl(overrideUrl?: string) {
const cfg = loadConfig();
const resolved = resolveBrowserConfig(cfg.browser);
@@ -193,19 +121,27 @@ export function resolveBrowserControlUrl(overrideUrl?: string) {
}
export async function browserStatus(baseUrl: string): Promise<BrowserStatus> {
return await fetchJson<BrowserStatus>(`${baseUrl}/`, { timeoutMs: 1500 });
return await fetchBrowserJson<BrowserStatus>(`${baseUrl}/`, {
timeoutMs: 1500,
});
}
export async function browserStart(baseUrl: string): Promise<void> {
await fetchJson(`${baseUrl}/start`, { method: "POST", timeoutMs: 15000 });
await fetchBrowserJson(`${baseUrl}/start`, {
method: "POST",
timeoutMs: 15000,
});
}
export async function browserStop(baseUrl: string): Promise<void> {
await fetchJson(`${baseUrl}/stop`, { method: "POST", timeoutMs: 15000 });
await fetchBrowserJson(`${baseUrl}/stop`, {
method: "POST",
timeoutMs: 15000,
});
}
export async function browserTabs(baseUrl: string): Promise<BrowserTab[]> {
const res = await fetchJson<{ running: boolean; tabs: BrowserTab[] }>(
const res = await fetchBrowserJson<{ running: boolean; tabs: BrowserTab[] }>(
`${baseUrl}/tabs`,
{ timeoutMs: 3000 },
);
@@ -216,7 +152,7 @@ export async function browserOpenTab(
baseUrl: string,
url: string,
): Promise<BrowserTab> {
return await fetchJson<BrowserTab>(`${baseUrl}/tabs/open`, {
return await fetchBrowserJson<BrowserTab>(`${baseUrl}/tabs/open`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url }),
@@ -228,7 +164,7 @@ export async function browserFocusTab(
baseUrl: string,
targetId: string,
): Promise<void> {
await fetchJson(`${baseUrl}/tabs/focus`, {
await fetchBrowserJson(`${baseUrl}/tabs/focus`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId }),
@@ -240,7 +176,7 @@ export async function browserCloseTab(
baseUrl: string,
targetId: string,
): Promise<void> {
await fetchJson(`${baseUrl}/tabs/${encodeURIComponent(targetId)}`, {
await fetchBrowserJson(`${baseUrl}/tabs/${encodeURIComponent(targetId)}`, {
method: "DELETE",
timeoutMs: 5000,
});
@@ -257,9 +193,12 @@ export async function browserScreenshot(
if (opts.targetId) q.set("targetId", opts.targetId);
if (opts.fullPage) q.set("fullPage", "true");
const suffix = q.toString() ? `?${q.toString()}` : "";
return await fetchJson<ScreenshotResult>(`${baseUrl}/screenshot${suffix}`, {
timeoutMs: 20000,
});
return await fetchBrowserJson<ScreenshotResult>(
`${baseUrl}/screenshot${suffix}`,
{
timeoutMs: 20000,
},
);
}
export async function browserEval(
@@ -270,7 +209,7 @@ export async function browserEval(
awaitPromise?: boolean;
},
): Promise<EvalResult> {
return await fetchJson<EvalResult>(`${baseUrl}/eval`, {
return await fetchBrowserJson<EvalResult>(`${baseUrl}/eval`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
@@ -294,9 +233,12 @@ export async function browserQuery(
q.set("selector", opts.selector);
if (opts.targetId) q.set("targetId", opts.targetId);
if (typeof opts.limit === "number") q.set("limit", String(opts.limit));
return await fetchJson<QueryResult>(`${baseUrl}/query?${q.toString()}`, {
timeoutMs: 15000,
});
return await fetchBrowserJson<QueryResult>(
`${baseUrl}/query?${q.toString()}`,
{
timeoutMs: 15000,
},
);
}
export async function browserDom(
@@ -314,7 +256,7 @@ export async function browserDom(
if (typeof opts.maxChars === "number")
q.set("maxChars", String(opts.maxChars));
if (opts.selector) q.set("selector", opts.selector);
return await fetchJson<DomResult>(`${baseUrl}/dom?${q.toString()}`, {
return await fetchBrowserJson<DomResult>(`${baseUrl}/dom?${q.toString()}`, {
timeoutMs: 20000,
});
}
@@ -331,7 +273,7 @@ export async function browserSnapshot(
q.set("format", opts.format);
if (opts.targetId) q.set("targetId", opts.targetId);
if (typeof opts.limit === "number") q.set("limit", String(opts.limit));
return await fetchJson<SnapshotResult>(
return await fetchBrowserJson<SnapshotResult>(
`${baseUrl}/snapshot?${q.toString()}`,
{
timeoutMs: 20000,
@@ -346,7 +288,7 @@ export async function browserClickRef(
targetId?: string;
},
): Promise<{ ok: true; targetId: string; url: string }> {
return await fetchJson<{ ok: true; targetId: string; url: string }>(
return await fetchBrowserJson<{ ok: true; targetId: string; url: string }>(
`${baseUrl}/click`,
{
method: "POST",
@@ -360,22 +302,4 @@ export async function browserClickRef(
);
}
export async function browserTool(
baseUrl: string,
opts: {
name: string;
args?: Record<string, unknown>;
targetId?: string;
},
): Promise<BrowserToolResponse> {
return await fetchJson<BrowserToolResponse>(`${baseUrl}/tool`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name: opts.name,
args: opts.args ?? {},
targetId: opts.targetId,
}),
timeoutMs: 20000,
});
}
// Actions beyond the basic read-only commands live in client-actions.ts.

199
src/browser/routes/tool-core.test.ts → src/browser/routes/actions-core.test.ts
View File

@@ -21,37 +21,13 @@ const pw = vi.hoisted(() => ({
resizeViewportViaPlaywright: vi.fn().mockResolvedValue(undefined),
runCodeViaPlaywright: vi.fn().mockResolvedValue("ok"),
selectOptionViaPlaywright: vi.fn().mockResolvedValue(undefined),
snapshotAiViaPlaywright: vi
.fn()
.mockResolvedValue({ snapshot: "SNAP" }),
takeScreenshotViaPlaywright: vi
.fn()
.mockResolvedValue({ buffer: Buffer.from("png") }),
typeViaPlaywright: vi.fn().mockResolvedValue(undefined),
waitForViaPlaywright: vi.fn().mockResolvedValue(undefined),
}));
const screenshot = vi.hoisted(() => ({
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES: 128,
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE: 64,
normalizeBrowserScreenshot: vi
.fn()
.mockImplementation(async (buf: Buffer) => ({
buffer: buf,
contentType: "image/png",
})),
}));
const media = vi.hoisted(() => ({
ensureMediaDir: vi.fn().mockResolvedValue(undefined),
saveMediaBuffer: vi.fn().mockResolvedValue({ path: "/tmp/fake.png" }),
}));
vi.mock("../pw-ai.js", () => pw);
vi.mock("../screenshot.js", () => screenshot);
vi.mock("../../media/store.js", () => media);
import { handleBrowserToolCore } from "./tool-core.js";
import { handleBrowserActionCore } from "./actions-core.js";
const baseTab = {
targetId: "tab1",
@@ -84,10 +60,12 @@ function createCtx(
ensureBrowserAvailable: vi.fn().mockResolvedValue(undefined),
ensureTabAvailable: vi.fn().mockResolvedValue(baseTab),
isReachable: vi.fn().mockResolvedValue(true),
listTabs: vi.fn().mockResolvedValue([
baseTab,
{ targetId: "tab2", title: "Two", url: "https://example.com/2" },
]),
listTabs: vi
.fn()
.mockResolvedValue([
baseTab,
{ targetId: "tab2", title: "Two", url: "https://example.com/2" },
]),
openTab: vi.fn().mockResolvedValue({
targetId: "newtab",
title: "",
@@ -102,15 +80,15 @@ function createCtx(
};
}
async function callTool(
name: string,
async function callAction(
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
args: Record<string, unknown> = {},
ctxOverride?: Partial<BrowserRouteContext>,
) {
const res = createRes();
const ctx = createCtx(ctxOverride);
const handled = await handleBrowserToolCore({
name,
const handled = await handleBrowserActionCore({
action,
args,
targetId: "",
cdpPort: 18792,
@@ -124,25 +102,30 @@ beforeEach(() => {
vi.clearAllMocks();
});
describe("handleBrowserToolCore", () => {
it("dispatches core Playwright tools", async () => {
describe("handleBrowserActionCore", () => {
it("dispatches core browser actions", async () => {
const cases = [
{
name: "browser_close",
action: "close" as const,
args: {},
fn: pw.closePageViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_resize",
action: "resize" as const,
args: { width: 800, height: 600 },
fn: pw.resizeViewportViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", width: 800, height: 600 },
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
width: 800,
height: 600,
},
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_handle_dialog",
action: "dialog" as const,
args: { accept: true, promptText: "ok" },
fn: pw.handleDialogViaPlaywright,
expectArgs: {
@@ -154,7 +137,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, message: "ok", type: "alert" },
},
{
name: "browser_evaluate",
action: "evaluate" as const,
args: { function: "() => 1", ref: "1" },
fn: pw.evaluateViaPlaywright,
expectArgs: {
@@ -166,7 +149,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, result: "result" },
},
{
name: "browser_file_upload",
action: "upload" as const,
args: { paths: ["/tmp/file.txt"] },
fn: pw.fileUploadViaPlaywright,
expectArgs: {
@@ -177,7 +160,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_fill_form",
action: "fill" as const,
args: { fields: [{ ref: "1", value: "x" }] },
fn: pw.fillFormViaPlaywright,
expectArgs: {
@@ -188,14 +171,14 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_press_key",
action: "press" as const,
args: { key: "Enter" },
fn: pw.pressKeyViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", key: "Enter" },
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_type",
action: "type" as const,
args: { ref: "2", text: "hi", submit: true, slowly: true },
fn: pw.typeViaPlaywright,
expectArgs: {
@@ -209,7 +192,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_navigate",
action: "navigate" as const,
args: { url: "https://example.com" },
fn: pw.navigateViaPlaywright,
expectArgs: {
@@ -220,21 +203,21 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_navigate_back",
action: "back" as const,
args: {},
fn: pw.navigateBackViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true, targetId: "tab1", url: "about:blank" },
},
{
name: "browser_run_code",
action: "run" as const,
args: { code: "return 1" },
fn: pw.runCodeViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", code: "return 1" },
expectBody: { ok: true, result: "ok" },
},
{
name: "browser_click",
action: "click" as const,
args: {
ref: "1",
doubleClick: true,
@@ -253,7 +236,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1", url: baseTab.url },
},
{
name: "browser_drag",
action: "drag" as const,
args: { startRef: "1", endRef: "2" },
fn: pw.dragViaPlaywright,
expectArgs: {
@@ -265,14 +248,14 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_hover",
action: "hover" as const,
args: { ref: "3" },
fn: pw.hoverViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", ref: "3" },
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_select_option",
action: "select" as const,
args: { ref: "4", values: ["A"] },
fn: pw.selectOptionViaPlaywright,
expectArgs: {
@@ -284,7 +267,7 @@ describe("handleBrowserToolCore", () => {
expectBody: { ok: true, targetId: "tab1" },
},
{
name: "browser_wait_for",
action: "wait" as const,
args: { time: 500, text: "ok", textGone: "bye" },
fn: pw.waitForViaPlaywright,
expectArgs: {
@@ -299,120 +282,10 @@ describe("handleBrowserToolCore", () => {
];
for (const item of cases) {
const { res, handled } = await callTool(item.name, item.args);
const { res, handled } = await callAction(item.action, item.args);
expect(handled).toBe(true);
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
expect(res.body).toEqual(item.expectBody);
}
});
it("handles screenshots via media storage", async () => {
const { res } = await callTool("browser_take_screenshot", {
type: "jpeg",
ref: "1",
fullPage: true,
element: "main",
filename: "shot.jpg",
});
expect(pw.takeScreenshotViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
ref: "1",
element: "main",
fullPage: true,
type: "jpeg",
});
expect(media.ensureMediaDir).toHaveBeenCalled();
expect(media.saveMediaBuffer).toHaveBeenCalled();
expect(res.body).toMatchObject({
ok: true,
path: "/tmp/fake.png",
filename: "shot.jpg",
targetId: "tab1",
url: baseTab.url,
});
});
it("handles snapshots with optional file output", async () => {
const { res } = await callTool("browser_snapshot", {
filename: "snapshot.txt",
});
expect(pw.snapshotAiViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
});
expect(media.ensureMediaDir).toHaveBeenCalled();
expect(media.saveMediaBuffer).toHaveBeenCalledWith(
expect.any(Buffer),
"text/plain",
"browser",
);
expect(res.body).toMatchObject({
ok: true,
path: "/tmp/fake.png",
filename: "snapshot.txt",
targetId: "tab1",
url: baseTab.url,
});
});
it("returns a message for browser_install", async () => {
const { res } = await callTool("browser_install");
expect(res.body).toMatchObject({ ok: true });
});
it("supports browser_tabs actions", async () => {
const ctx = createCtx();
const listRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "list" },
targetId: "",
cdpPort: 18792,
ctx,
res: listRes,
});
expect(listRes.body).toMatchObject({ ok: true });
expect(ctx.listTabs).toHaveBeenCalled();
const newRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "new" },
targetId: "",
cdpPort: 18792,
ctx,
res: newRes,
});
expect(ctx.ensureBrowserAvailable).toHaveBeenCalled();
expect(ctx.openTab).toHaveBeenCalled();
expect(newRes.body).toMatchObject({ ok: true, tab: { targetId: "newtab" } });
const closeRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "close", index: 1 },
targetId: "",
cdpPort: 18792,
ctx,
res: closeRes,
});
expect(ctx.closeTab).toHaveBeenCalledWith("tab2");
expect(closeRes.body).toMatchObject({ ok: true, targetId: "tab2" });
const selectRes = createRes();
await handleBrowserToolCore({
name: "browser_tabs",
args: { action: "select", index: 0 },
targetId: "",
cdpPort: 18792,
ctx,
res: selectRes,
});
expect(ctx.focusTab).toHaveBeenCalledWith("tab1");
expect(selectRes.body).toMatchObject({ ok: true, targetId: "tab1" });
});
});

198
src/browser/routes/tool-core.ts → src/browser/routes/actions-core.ts
View File

@@ -1,8 +1,5 @@
import path from "node:path";
import type express from "express";
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
clickViaPlaywright,
closePageViaPlaywright,
@@ -18,16 +15,9 @@ import {
resizeViewportViaPlaywright,
runCodeViaPlaywright,
selectOptionViaPlaywright,
snapshotAiViaPlaywright,
takeScreenshotViaPlaywright,
typeViaPlaywright,
waitForViaPlaywright,
} from "../pw-ai.js";
import {
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
normalizeBrowserScreenshot,
} from "../screenshot.js";
import type { BrowserRouteContext } from "../server-context.js";
import {
jsonError,
@@ -37,8 +27,26 @@ import {
toStringOrEmpty,
} from "./utils.js";
type ToolCoreParams = {
name: string;
export type BrowserActionCore =
| "back"
| "click"
| "close"
| "dialog"
| "drag"
| "evaluate"
| "fill"
| "hover"
| "navigate"
| "press"
| "resize"
| "run"
| "select"
| "type"
| "upload"
| "wait";
type ActionCoreParams = {
action: BrowserActionCore;
args: Record<string, unknown>;
targetId: string;
cdpPort: number;
@@ -46,20 +54,20 @@ type ToolCoreParams = {
res: express.Response;
};
export async function handleBrowserToolCore(
params: ToolCoreParams,
export async function handleBrowserActionCore(
params: ActionCoreParams,
): Promise<boolean> {
const { name, args, targetId, cdpPort, ctx, res } = params;
const { action, args, targetId, cdpPort, ctx, res } = params;
const target = targetId || undefined;
switch (name) {
case "browser_close": {
switch (action) {
case "close": {
const tab = await ctx.ensureTabAvailable(target);
await closePageViaPlaywright({ cdpPort, targetId: tab.targetId });
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_resize": {
case "resize": {
const width = toNumber(args.width);
const height = toNumber(args.height);
if (!width || !height) {
@@ -76,7 +84,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_handle_dialog": {
case "dialog": {
const accept = toBoolean(args.accept);
if (accept === undefined) {
jsonError(res, 400, "accept is required");
@@ -93,7 +101,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, ...result });
return true;
}
case "browser_evaluate": {
case "evaluate": {
const fn = toStringOrEmpty(args.function);
if (!fn) {
jsonError(res, 400, "function is required");
@@ -110,7 +118,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, result });
return true;
}
case "browser_file_upload": {
case "upload": {
const paths = toStringArray(args.paths) ?? [];
const tab = await ctx.ensureTabAvailable(target);
await fileUploadViaPlaywright({
@@ -121,7 +129,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_fill_form": {
case "fill": {
const fields = Array.isArray(args.fields)
? (args.fields as Array<Record<string, unknown>>)
: null;
@@ -138,15 +146,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_install": {
res.json({
ok: true,
message:
"clawd browser uses system Chrome/Chromium; no Playwright install needed.",
});
return true;
}
case "browser_press_key": {
case "press": {
const key = toStringOrEmpty(args.key);
if (!key) {
jsonError(res, 400, "key is required");
@@ -161,7 +161,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_type": {
case "type": {
const ref = toStringOrEmpty(args.ref);
const text = toStringOrEmpty(args.text);
if (!ref || !text) {
@@ -182,7 +182,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_navigate": {
case "navigate": {
const url = toStringOrEmpty(args.url);
if (!url) {
jsonError(res, 400, "url is required");
@@ -197,7 +197,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "browser_navigate_back": {
case "back": {
const tab = await ctx.ensureTabAvailable(target);
const result = await navigateBackViaPlaywright({
cdpPort,
@@ -206,7 +206,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, ...result });
return true;
}
case "browser_run_code": {
case "run": {
const code = toStringOrEmpty(args.code);
if (!code) {
jsonError(res, 400, "code is required");
@@ -221,73 +221,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, result });
return true;
}
case "browser_take_screenshot": {
const type = args.type === "jpeg" ? "jpeg" : "png";
const ref = toStringOrEmpty(args.ref) || undefined;
const fullPage = toBoolean(args.fullPage) ?? false;
const element = toStringOrEmpty(args.element) || undefined;
const filename = toStringOrEmpty(args.filename) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const snap = await takeScreenshotViaPlaywright({
cdpPort,
targetId: tab.targetId,
ref,
element,
fullPage,
type,
});
const normalized = await normalizeBrowserScreenshot(snap.buffer, {
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
});
await ensureMediaDir();
const saved = await saveMediaBuffer(
normalized.buffer,
normalized.contentType ?? `image/${type}`,
"browser",
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
);
res.json({
ok: true,
path: path.resolve(saved.path),
filename,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
case "browser_snapshot": {
const filename = toStringOrEmpty(args.filename) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const snap = await snapshotAiViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
if (filename) {
await ensureMediaDir();
const saved = await saveMediaBuffer(
Buffer.from(snap.snapshot, "utf8"),
"text/plain",
"browser",
);
res.json({
ok: true,
path: path.resolve(saved.path),
filename,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
res.json({
ok: true,
snapshot: snap.snapshot,
targetId: tab.targetId,
url: tab.url,
});
return true;
}
case "browser_click": {
case "click": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");
@@ -310,7 +244,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
return true;
}
case "browser_drag": {
case "drag": {
const startRef = toStringOrEmpty(args.startRef);
const endRef = toStringOrEmpty(args.endRef);
if (!startRef || !endRef) {
@@ -327,7 +261,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_hover": {
case "hover": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");
@@ -342,7 +276,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_select_option": {
case "select": {
const ref = toStringOrEmpty(args.ref);
const values = toStringArray(args.values);
if (!ref || !values?.length) {
@@ -359,59 +293,7 @@ export async function handleBrowserToolCore(
res.json({ ok: true, targetId: tab.targetId });
return true;
}
case "browser_tabs": {
const action = toStringOrEmpty(args.action);
const index = toNumber(args.index);
if (!action) {
jsonError(res, 400, "action is required");
return true;
}
if (action === "list") {
const reachable = await ctx.isReachable(300);
if (!reachable) {
res.json({ ok: true, tabs: [] });
return true;
}
const tabs = await ctx.listTabs();
res.json({ ok: true, tabs });
return true;
}
if (action === "new") {
await ctx.ensureBrowserAvailable();
const tab = await ctx.openTab("about:blank");
res.json({ ok: true, tab });
return true;
}
if (action === "close") {
const tabs = await ctx.listTabs();
const targetTab = typeof index === "number" ? tabs[index] : tabs.at(0);
if (!targetTab) {
jsonError(res, 404, "tab not found");
return true;
}
await ctx.closeTab(targetTab.targetId);
res.json({ ok: true, targetId: targetTab.targetId });
return true;
}
if (action === "select") {
if (typeof index !== "number") {
jsonError(res, 400, "index is required");
return true;
}
const tabs = await ctx.listTabs();
const targetTab = tabs[index];
if (!targetTab) {
jsonError(res, 404, "tab not found");
return true;
}
await ctx.focusTab(targetTab.targetId);
res.json({ ok: true, targetId: targetTab.targetId });
return true;
}
jsonError(res, 400, "unknown tab action");
return true;
}
case "browser_wait_for": {
case "wait": {
const time = toNumber(args.time);
const text = toStringOrEmpty(args.text) || undefined;
const textGone = toStringOrEmpty(args.textGone) || undefined;

43
src/browser/routes/tool-extra.test.ts → src/browser/routes/actions-extra.test.ts
View File

@@ -30,7 +30,7 @@ const media = vi.hoisted(() => ({
vi.mock("../pw-ai.js", () => pw);
vi.mock("../../media/store.js", () => media);
import { handleBrowserToolExtra } from "./tool-extra.js";
import { handleBrowserActionExtra } from "./actions-extra.js";
const baseTab = {
targetId: "tab1",
@@ -78,11 +78,14 @@ function createCtx(
};
}
async function callTool(name: string, args: Record<string, unknown> = {}) {
async function callAction(
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
args: Record<string, unknown> = {},
) {
const res = createRes();
const ctx = createCtx();
const handled = await handleBrowserToolExtra({
name,
const handled = await handleBrowserActionExtra({
action,
args,
targetId: "",
cdpPort: 18792,
@@ -96,11 +99,11 @@ beforeEach(() => {
vi.clearAllMocks();
});
describe("handleBrowserToolExtra", () => {
it("dispatches extra Playwright tools", async () => {
describe("handleBrowserActionExtra", () => {
it("dispatches extra browser actions", async () => {
const cases = [
{
name: "browser_console_messages",
action: "console" as const,
args: { level: "error" },
fn: pw.getConsoleMessagesViaPlaywright,
expectArgs: {
@@ -111,7 +114,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true, messages: [], targetId: "tab1" },
},
{
name: "browser_network_requests",
action: "network" as const,
args: { includeStatic: true },
fn: pw.getNetworkRequestsViaPlaywright,
expectArgs: {
@@ -122,14 +125,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true, requests: [], targetId: "tab1" },
},
{
name: "browser_start_tracing",
action: "traceStart" as const,
args: {},
fn: pw.startTracingViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true },
},
{
name: "browser_verify_element_visible",
action: "verifyElement" as const,
args: { role: "button", accessibleName: "Submit" },
fn: pw.verifyElementVisibleViaPlaywright,
expectArgs: {
@@ -141,14 +144,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_verify_text_visible",
action: "verifyText" as const,
args: { text: "Hello" },
fn: pw.verifyTextVisibleViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", text: "Hello" },
expectBody: { ok: true },
},
{
name: "browser_verify_list_visible",
action: "verifyList" as const,
args: { ref: "1", items: ["a", "b"] },
fn: pw.verifyListVisibleViaPlaywright,
expectArgs: {
@@ -160,7 +163,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_verify_value",
action: "verifyValue" as const,
args: { ref: "2", type: "textbox", value: "x" },
fn: pw.verifyValueViaPlaywright,
expectArgs: {
@@ -173,14 +176,14 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_mouse_move_xy",
action: "mouseMove" as const,
args: { x: 10, y: 20 },
fn: pw.mouseMoveViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1", x: 10, y: 20 },
expectBody: { ok: true },
},
{
name: "browser_mouse_click_xy",
action: "mouseClick" as const,
args: { x: 1, y: 2, button: "right" },
fn: pw.mouseClickViaPlaywright,
expectArgs: {
@@ -193,7 +196,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_mouse_drag_xy",
action: "mouseDrag" as const,
args: { startX: 1, startY: 2, endX: 3, endY: 4 },
fn: pw.mouseDragViaPlaywright,
expectArgs: {
@@ -207,7 +210,7 @@ describe("handleBrowserToolExtra", () => {
expectBody: { ok: true },
},
{
name: "browser_generate_locator",
action: "locator" as const,
args: { ref: "99" },
fn: pw.generateLocatorForRef,
expectArgs: "99",
@@ -216,7 +219,7 @@ describe("handleBrowserToolExtra", () => {
];
for (const item of cases) {
const { res, handled } = await callTool(item.name, item.args);
const { res, handled } = await callAction(item.action, item.args);
expect(handled).toBe(true);
expect(item.fn).toHaveBeenCalledWith(item.expectArgs);
expect(res.body).toEqual(item.expectBody);
@@ -224,7 +227,7 @@ describe("handleBrowserToolExtra", () => {
});
it("stores PDF and trace outputs", async () => {
const { res: pdfRes } = await callTool("browser_pdf_save");
const { res: pdfRes } = await callAction("pdf");
expect(pw.pdfViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
@@ -239,7 +242,7 @@ describe("handleBrowserToolExtra", () => {
});
media.saveMediaBuffer.mockResolvedValueOnce({ path: "/tmp/fake.zip" });
const { res: traceRes } = await callTool("browser_stop_tracing");
const { res: traceRes } = await callAction("traceStop");
expect(pw.stopTracingViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",

53
src/browser/routes/tool-extra.ts → src/browser/routes/actions-extra.ts
View File

@@ -27,8 +27,23 @@ import {
toStringOrEmpty,
} from "./utils.js";
type ToolExtraParams = {
name: string;
export type BrowserActionExtra =
| "console"
| "locator"
| "mouseClick"
| "mouseDrag"
| "mouseMove"
| "network"
| "pdf"
| "traceStart"
| "traceStop"
| "verifyElement"
| "verifyList"
| "verifyText"
| "verifyValue";
type ActionExtraParams = {
action: BrowserActionExtra;
args: Record<string, unknown>;
targetId: string;
cdpPort: number;
@@ -36,14 +51,14 @@ type ToolExtraParams = {
res: express.Response;
};
export async function handleBrowserToolExtra(
params: ToolExtraParams,
export async function handleBrowserActionExtra(
params: ActionExtraParams,
): Promise<boolean> {
const { name, args, targetId, cdpPort, ctx, res } = params;
const { action, args, targetId, cdpPort, ctx, res } = params;
const target = targetId || undefined;
switch (name) {
case "browser_console_messages": {
switch (action) {
case "console": {
const level = toStringOrEmpty(args.level) || undefined;
const tab = await ctx.ensureTabAvailable(target);
const messages = await getConsoleMessagesViaPlaywright({
@@ -54,7 +69,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true, messages, targetId: tab.targetId });
return true;
}
case "browser_network_requests": {
case "network": {
const includeStatic = toBoolean(args.includeStatic) ?? false;
const tab = await ctx.ensureTabAvailable(target);
const requests = await getNetworkRequestsViaPlaywright({
@@ -65,7 +80,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true, requests, targetId: tab.targetId });
return true;
}
case "browser_pdf_save": {
case "pdf": {
const tab = await ctx.ensureTabAvailable(target);
const pdf = await pdfViaPlaywright({
cdpPort,
@@ -86,7 +101,7 @@ export async function handleBrowserToolExtra(
});
return true;
}
case "browser_start_tracing": {
case "traceStart": {
const tab = await ctx.ensureTabAvailable(target);
await startTracingViaPlaywright({
cdpPort,
@@ -95,7 +110,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_stop_tracing": {
case "traceStop": {
const tab = await ctx.ensureTabAvailable(target);
const trace = await stopTracingViaPlaywright({
cdpPort,
@@ -116,7 +131,7 @@ export async function handleBrowserToolExtra(
});
return true;
}
case "browser_verify_element_visible": {
case "verifyElement": {
const role = toStringOrEmpty(args.role);
const accessibleName = toStringOrEmpty(args.accessibleName);
if (!role || !accessibleName) {
@@ -133,7 +148,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_text_visible": {
case "verifyText": {
const text = toStringOrEmpty(args.text);
if (!text) {
jsonError(res, 400, "text is required");
@@ -148,7 +163,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_list_visible": {
case "verifyList": {
const ref = toStringOrEmpty(args.ref);
const items = toStringArray(args.items);
if (!ref || !items?.length) {
@@ -165,7 +180,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_verify_value": {
case "verifyValue": {
const ref = toStringOrEmpty(args.ref);
const type = toStringOrEmpty(args.type);
const value = toStringOrEmpty(args.value);
@@ -184,7 +199,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_move_xy": {
case "mouseMove": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
@@ -201,7 +216,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_click_xy": {
case "mouseClick": {
const x = toNumber(args.x);
const y = toNumber(args.y);
if (x === undefined || y === undefined) {
@@ -220,7 +235,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_mouse_drag_xy": {
case "mouseDrag": {
const startX = toNumber(args.startX);
const startY = toNumber(args.startY);
const endX = toNumber(args.endX);
@@ -246,7 +261,7 @@ export async function handleBrowserToolExtra(
res.json({ ok: true });
return true;
}
case "browser_generate_locator": {
case "locator": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");

249
src/browser/routes/actions.ts Normal file
View File

@@ -0,0 +1,249 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { handleBrowserActionCore } from "./actions-core.js";
import { handleBrowserActionExtra } from "./actions-extra.js";
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
function readBody(req: express.Request): Record<string, unknown> {
const body = req.body as Record<string, unknown> | undefined;
if (!body || typeof body !== "object" || Array.isArray(body)) return {};
return body;
}
function readTargetId(value: unknown): string {
return toStringOrEmpty(value);
}
function handleActionError(
ctx: BrowserRouteContext,
res: express.Response,
err: unknown,
) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
async function runCoreAction(
ctx: BrowserRouteContext,
res: express.Response,
action: Parameters<typeof handleBrowserActionCore>[0]["action"],
args: Record<string, unknown>,
targetId: string,
) {
try {
const cdpPort = ctx.state().cdpPort;
await handleBrowserActionCore({
action,
args,
targetId,
cdpPort,
ctx,
res,
});
} catch (err) {
handleActionError(ctx, res, err);
}
}
async function runExtraAction(
ctx: BrowserRouteContext,
res: express.Response,
action: Parameters<typeof handleBrowserActionExtra>[0]["action"],
args: Record<string, unknown>,
targetId: string,
) {
try {
const cdpPort = ctx.state().cdpPort;
await handleBrowserActionExtra({
action,
args,
targetId,
cdpPort,
ctx,
res,
});
} catch (err) {
handleActionError(ctx, res, err);
}
}
export function registerBrowserActionRoutes(
app: express.Express,
ctx: BrowserRouteContext,
) {
app.post("/navigate", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "navigate", body, targetId);
});
app.post("/back", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "back", body, targetId);
});
app.post("/resize", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "resize", body, targetId);
});
app.post("/close", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "close", body, targetId);
});
app.post("/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "click", body, targetId);
});
app.post("/type", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "type", body, targetId);
});
app.post("/press", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "press", body, targetId);
});
app.post("/hover", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "hover", body, targetId);
});
app.post("/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "drag", body, targetId);
});
app.post("/select", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "select", body, targetId);
});
app.post("/upload", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "upload", body, targetId);
});
app.post("/fill", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "fill", body, targetId);
});
app.post("/dialog", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "dialog", body, targetId);
});
app.post("/wait", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "wait", body, targetId);
});
app.post("/evaluate", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "evaluate", body, targetId);
});
app.post("/run", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runCoreAction(ctx, res, "run", body, targetId);
});
app.get("/console", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const level = toStringOrEmpty(req.query.level);
const args = level ? { level } : {};
await runExtraAction(ctx, res, "console", args, targetId);
});
app.get("/network", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const includeStatic = toBoolean(req.query.includeStatic) ?? false;
await runExtraAction(ctx, res, "network", { includeStatic }, targetId);
});
app.post("/trace/start", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStart", body, targetId);
});
app.post("/trace/stop", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStop", body, targetId);
});
app.post("/pdf", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "pdf", body, targetId);
});
app.post("/verify/element", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyElement", body, targetId);
});
app.post("/verify/text", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyText", body, targetId);
});
app.post("/verify/list", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyList", body, targetId);
});
app.post("/verify/value", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "verifyValue", body, targetId);
});
app.post("/mouse/move", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseMove", body, targetId);
});
app.post("/mouse/click", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseClick", body, targetId);
});
app.post("/mouse/drag", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseDrag", body, targetId);
});
app.post("/locator", async (req, res) => {
const body = readBody(req);
await runExtraAction(ctx, res, "locator", body, "");
});
}

4
src/browser/routes/index.ts
View File

@@ -1,10 +1,10 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { registerBrowserActionRoutes } from "./actions.js";
import { registerBrowserBasicRoutes } from "./basic.js";
import { registerBrowserInspectRoutes } from "./inspect.js";
import { registerBrowserTabRoutes } from "./tabs.js";
import { registerBrowserToolRoutes } from "./tool.js";
export function registerBrowserRoutes(
app: express.Express,
@@ -13,5 +13,5 @@ export function registerBrowserRoutes(
registerBrowserBasicRoutes(app, ctx);
registerBrowserTabRoutes(app, ctx);
registerBrowserInspectRoutes(app, ctx);
registerBrowserToolRoutes(app, ctx);
registerBrowserActionRoutes(app, ctx);
}

23
src/browser/routes/inspect.ts
View File

@@ -281,27 +281,4 @@ export function registerBrowserInspectRoutes(
jsonError(res, 500, String(err));
}
});
app.post("/click", async (req, res) => {
const ref = toStringOrEmpty((req.body as { ref?: unknown })?.ref);
const targetId = toStringOrEmpty(
(req.body as { targetId?: unknown })?.targetId,
);
if (!ref) return jsonError(res, 400, "ref is required");
try {
const tab = await ctx.ensureTabAvailable(targetId || undefined);
await clickViaPlaywright({
cdpPort: ctx.state().cdpPort,
targetId: tab.targetId,
ref,
});
res.json({ ok: true, targetId: tab.targetId, url: tab.url });
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
}

65
src/browser/routes/tool.ts
View File

@@ -1,65 +0,0 @@
import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { handleBrowserToolCore } from "./tool-core.js";
import { handleBrowserToolExtra } from "./tool-extra.js";
import { jsonError, toStringOrEmpty } from "./utils.js";
type ToolRequestBody = {
name?: unknown;
args?: unknown;
targetId?: unknown;
};
function toolArgs(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
return value as Record<string, unknown>;
}
export function registerBrowserToolRoutes(
app: express.Express,
ctx: BrowserRouteContext,
) {
app.post("/tool", async (req, res) => {
const body = req.body as ToolRequestBody;
const name = toStringOrEmpty(body?.name);
if (!name) return jsonError(res, 400, "name is required");
const args = toolArgs(body?.args);
const targetId = toStringOrEmpty(body?.targetId || args?.targetId);
try {
let cdpPort: number;
try {
cdpPort = ctx.state().cdpPort;
} catch {
return jsonError(res, 503, "browser server not started");
}
const handledCore = await handleBrowserToolCore({
name,
args,
targetId,
cdpPort,
ctx,
res,
});
if (handledCore) return;
const handledExtra = await handleBrowserToolExtra({
name,
args,
targetId,
cdpPort,
ctx,
res,
});
if (handledExtra) return;
return jsonError(res, 400, "unknown tool name");
} catch (err) {
const mapped = ctx.mapTabError(err);
if (mapped) return jsonError(res, mapped.status, mapped.message);
jsonError(res, 500, String(err));
}
});
}