let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(browser): trim observe endpoints

Browse Source
This commit is contained in:
Peter Steinberger
2025-12-20 02:07:27 +00:00
parent 0e94c6b025
commit 479720c169
11 changed files with 12 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
docs/browser.md
View File

@@ -121,6 +121,11 @@ Inspection:
- `GET /dom`
- `GET /snapshot` (`aria` | `domSnapshot` | `ai`)
Debug-only endpoints (intentionally omitted for now):
- network request log (privacy)
- tracing export (large + sensitive)
- locator generation (dev convenience)
Actions:
- `POST /navigate`, `POST /back`
- `POST /resize`
@@ -131,12 +136,10 @@ Actions:
- `POST /wait` (time/text/textGone)
- `POST /evaluate` (function + optional ref)
- `POST /run` (function(page) → result)
- `GET /console`, `GET /network`
- `POST /trace/start`, `POST /trace/stop`
- `GET /console`
- `POST /pdf`
- `POST /verify/element`, `POST /verify/text`, `POST /verify/list`, `POST /verify/value`
- `POST /mouse/move`, `POST /mouse/click`, `POST /mouse/drag`
- `POST /locator` (generate Playwright locator)
### "Is it open or closed?"
@@ -201,9 +204,6 @@ Actions:
- `clawdis browser evaluate --fn '(el) => el.textContent' --ref 7`
- `clawdis browser run --code '(page) => page.title()'`
- `clawdis browser console --level error`
- `clawdis browser network --include-static`
- `clawdis browser trace-start`
- `clawdis browser trace-stop`
- `clawdis browser pdf`
- `clawdis browser verify-element --role button --name "Submit"`
- `clawdis browser verify-text "Welcome"`
@@ -212,7 +212,6 @@ Actions:
- `clawdis browser mouse-move --x 120 --y 240`
- `clawdis browser mouse-click --x 120 --y 240`
- `clawdis browser mouse-drag --start-x 10 --start-y 20 --end-x 200 --end-y 300`
- `clawdis browser locator 77`
Notes:
- `upload` and `dialog` only work when a file chooser or dialog is present.

62
src/browser/client-actions-observe.ts
View File

@@ -3,10 +3,7 @@ import type {
BrowserActionPathResult,
} from "./client-actions-types.js";
import { fetchBrowserJson } from "./client-fetch.js";
import type {
BrowserConsoleMessage,
BrowserNetworkRequest,
} from "./pw-session.js";
import type { BrowserConsoleMessage } from "./pw-session.js";
export async function browserConsoleMessages(
baseUrl: string,
@@ -23,48 +20,6 @@ export async function browserConsoleMessages(
}>(`${baseUrl}/console${suffix}`, { timeoutMs: 20000 });
}
export async function browserNetworkRequests(
baseUrl: string,
opts: { includeStatic?: boolean; targetId?: string } = {},
): Promise<{ ok: true; requests: BrowserNetworkRequest[]; targetId: string }> {
const q = new URLSearchParams();
if (opts.includeStatic) q.set("includeStatic", "true");
if (opts.targetId) q.set("targetId", opts.targetId);
const suffix = q.toString() ? `?${q.toString()}` : "";
return await fetchBrowserJson<{
ok: true;
requests: BrowserNetworkRequest[];
targetId: string;
}>(`${baseUrl}/network${suffix}`, { timeoutMs: 20000 });
}
export async function browserStartTracing(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionOk> {
return await fetchBrowserJson<BrowserActionOk>(`${baseUrl}/trace/start`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
});
}
export async function browserStopTracing(
baseUrl: string,
opts: { targetId?: string } = {},
): Promise<BrowserActionPathResult> {
return await fetchBrowserJson<BrowserActionPathResult>(
`${baseUrl}/trace/stop`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ targetId: opts.targetId }),
timeoutMs: 20000,
},
);
}
export async function browserPdfSave(
baseUrl: string,
opts: { targetId?: string } = {},
@@ -190,18 +145,3 @@ export async function browserMouseDrag(
timeoutMs: 20000,
});
}
export async function browserGenerateLocator(
baseUrl: string,
opts: { ref: string },
): Promise<{ ok: true; locator: string }> {
return await fetchBrowserJson<{ ok: true; locator: string }>(
`${baseUrl}/locator`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ ref: opts.ref }),
timeoutMs: 20000,
},
);
}

5
src/browser/pw-ai.ts
View File

@@ -1,6 +1,5 @@
export {
type BrowserConsoleMessage,
type BrowserNetworkRequest,
closePlaywrightBrowserConnection,
ensurePageState,
getPageForTargetId,
@@ -32,14 +31,10 @@ export {
} from "./pw-tools-core.js";
export {
generateLocatorForRef,
getConsoleMessagesViaPlaywright,
getNetworkRequestsViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
startTracingViaPlaywright,
stopTracingViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
verifyTextVisibleViaPlaywright,

42
src/browser/pw-session.ts
View File

@@ -3,7 +3,6 @@ import type {
BrowserContext,
ConsoleMessage,
Page,
Request,
} from "playwright-core";
import { chromium } from "playwright-core";
@@ -15,14 +14,12 @@ export type BrowserConsoleMessage = {
};
export type BrowserNetworkRequest = {
requestId?: string;
url: string;
method: string;
resourceType?: string;
status?: number;
ok?: boolean;
fromCache?: boolean;
failureText?: string;
timestamp: string;
resourceType?: string;
timestamp?: string;
};
type SnapshotForAIResult = { full: string; incremental?: string };
@@ -47,8 +44,6 @@ type ConnectedBrowser = {
type PageState = {
console: BrowserConsoleMessage[];
network: BrowserNetworkRequest[];
requestMap: Map<Request, BrowserNetworkRequest>;
};
const pageStates = new WeakMap<Page, PageState>();
@@ -56,7 +51,6 @@ const observedContexts = new WeakSet<BrowserContext>();
const observedPages = new WeakSet<Page>();
const MAX_CONSOLE_MESSAGES = 500;
const MAX_NETWORK_REQUESTS = 1000;
let cached: ConnectedBrowser | null = null;
let connecting: Promise<ConnectedBrowser> | null = null;
@@ -71,8 +65,6 @@ export function ensurePageState(page: Page): PageState {
const state: PageState = {
console: [],
network: [],
requestMap: new Map(),
};
pageStates.set(page, state);
@@ -88,34 +80,6 @@ export function ensurePageState(page: Page): PageState {
state.console.push(entry);
if (state.console.length > MAX_CONSOLE_MESSAGES) state.console.shift();
});
page.on("request", (req: Request) => {
const entry: BrowserNetworkRequest = {
url: req.url(),
method: req.method(),
resourceType: req.resourceType(),
timestamp: new Date().toISOString(),
};
state.network.push(entry);
state.requestMap.set(req, entry);
if (state.network.length > MAX_NETWORK_REQUESTS) state.network.shift();
});
page.on("requestfinished", async (req: Request) => {
const entry = state.requestMap.get(req);
if (!entry) return;
const response = await req.response().catch(() => null);
if (response) {
entry.status = response.status();
entry.ok = response.ok();
entry.fromCache = response.fromServiceWorker();
}
state.requestMap.delete(req);
});
page.on("requestfailed", (req: Request) => {
const entry = state.requestMap.get(req);
if (!entry) return;
entry.failureText = req.failure()?.errorText;
state.requestMap.delete(req);
});
page.on("close", () => {
pageStates.delete(page);
observedPages.delete(page);

60
src/browser/pw-tools-observe.ts
View File

@@ -1,20 +1,10 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import {
type BrowserConsoleMessage,
type BrowserNetworkRequest,
ensurePageState,
getPageForTargetId,
refLocator,
} from "./pw-session.js";
const STATIC_RESOURCE_TYPES = new Set(["image", "font", "stylesheet", "media"]);
const tracingContexts = new WeakSet<object>();
function consolePriority(level: string) {
switch (level) {
case "error":
@@ -31,39 +21,6 @@ function consolePriority(level: string) {
}
}
export async function startTracingViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
}): Promise<void> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const context = page.context();
if (tracingContexts.has(context)) throw new Error("Tracing already started");
await context.tracing.start({
screenshots: true,
snapshots: true,
sources: true,
});
tracingContexts.add(context);
}
export async function stopTracingViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
}): Promise<{ buffer: Buffer }> {
const page = await getPageForTargetId(opts);
ensurePageState(page);
const context = page.context();
if (!tracingContexts.has(context)) throw new Error("Tracing not started");
const fileName = `clawd-trace-${crypto.randomUUID()}.zip`;
const filePath = path.join(os.tmpdir(), fileName);
await context.tracing.stop({ path: filePath });
tracingContexts.delete(context);
const buffer = await fs.readFile(filePath);
await fs.rm(filePath).catch(() => {});
return { buffer };
}
export async function getConsoleMessagesViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -76,19 +33,6 @@ export async function getConsoleMessagesViaPlaywright(opts: {
return state.console.filter((msg) => consolePriority(msg.type) >= min);
}
export async function getNetworkRequestsViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
includeStatic?: boolean;
}): Promise<BrowserNetworkRequest[]> {
const page = await getPageForTargetId(opts);
const state = ensurePageState(page);
if (opts.includeStatic) return [...state.network];
return state.network.filter(
(req) => !req.resourceType || !STATIC_RESOURCE_TYPES.has(req.resourceType),
);
}
export async function mouseMoveViaPlaywright(opts: {
cdpPort: number;
targetId?: string;
@@ -194,7 +138,3 @@ export async function verifyValueViaPlaywright(opts: {
if (value !== opts.value)
throw new Error(`expected ${opts.value}, got ${value}`);
}
export function generateLocatorForRef(ref: string) {
return `locator('aria-ref=${ref}')`;
}

48
src/browser/routes/actions-extra.test.ts
View File

@@ -3,19 +3,11 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { BrowserRouteContext } from "../server-context.js";
const pw = vi.hoisted(() => ({
generateLocatorForRef: vi
.fn()
.mockImplementation((ref: string) => `locator('aria-ref=${ref}')`),
getConsoleMessagesViaPlaywright: vi.fn().mockResolvedValue([]),
getNetworkRequestsViaPlaywright: vi.fn().mockResolvedValue([]),
mouseClickViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseDragViaPlaywright: vi.fn().mockResolvedValue(undefined),
mouseMoveViaPlaywright: vi.fn().mockResolvedValue(undefined),
pdfViaPlaywright: vi.fn().mockResolvedValue({ buffer: Buffer.from("pdf") }),
startTracingViaPlaywright: vi.fn().mockResolvedValue(undefined),
stopTracingViaPlaywright: vi
.fn()
.mockResolvedValue({ buffer: Buffer.from("trace") }),
verifyElementVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
verifyListVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
verifyTextVisibleViaPlaywright: vi.fn().mockResolvedValue(undefined),
@@ -113,24 +105,6 @@ describe("handleBrowserActionExtra", () => {
},
expectBody: { ok: true, messages: [], targetId: "tab1" },
},
{
action: "network" as const,
args: { includeStatic: true },
fn: pw.getNetworkRequestsViaPlaywright,
expectArgs: {
cdpPort: 18792,
targetId: "tab1",
includeStatic: true,
},
expectBody: { ok: true, requests: [], targetId: "tab1" },
},
{
action: "traceStart" as const,
args: {},
fn: pw.startTracingViaPlaywright,
expectArgs: { cdpPort: 18792, targetId: "tab1" },
expectBody: { ok: true },
},
{
action: "verifyElement" as const,
args: { role: "button", accessibleName: "Submit" },
@@ -209,13 +183,6 @@ describe("handleBrowserActionExtra", () => {
},
expectBody: { ok: true },
},
{
action: "locator" as const,
args: { ref: "99" },
fn: pw.generateLocatorForRef,
expectArgs: "99",
expectBody: { ok: true, locator: "locator('aria-ref=99')" },
},
];
for (const item of cases) {
@@ -226,7 +193,7 @@ describe("handleBrowserActionExtra", () => {
}
});
it("stores PDF and trace outputs", async () => {
it("stores PDF output", async () => {
const { res: pdfRes } = await callAction("pdf");
expect(pw.pdfViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
@@ -240,18 +207,5 @@ describe("handleBrowserActionExtra", () => {
targetId: "tab1",
url: baseTab.url,
});
media.saveMediaBuffer.mockResolvedValueOnce({ path: "/tmp/fake.zip" });
const { res: traceRes } = await callAction("traceStop");
expect(pw.stopTracingViaPlaywright).toHaveBeenCalledWith({
cdpPort: 18792,
targetId: "tab1",
});
expect(traceRes.body).toMatchObject({
ok: true,
path: "/tmp/fake.zip",
targetId: "tab1",
url: baseTab.url,
});
});
});

60
src/browser/routes/actions-extra.ts
View File

@@ -4,15 +4,11 @@ import type express from "express";
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
import {
generateLocatorForRef,
getConsoleMessagesViaPlaywright,
getNetworkRequestsViaPlaywright,
mouseClickViaPlaywright,
mouseDragViaPlaywright,
mouseMoveViaPlaywright,
pdfViaPlaywright,
startTracingViaPlaywright,
stopTracingViaPlaywright,
verifyElementVisibleViaPlaywright,
verifyListVisibleViaPlaywright,
verifyTextVisibleViaPlaywright,
@@ -21,7 +17,6 @@ import {
import type { BrowserRouteContext } from "../server-context.js";
import {
jsonError,
toBoolean,
toNumber,
toStringArray,
toStringOrEmpty,
@@ -37,14 +32,10 @@ function normalizeMouseButton(value: unknown): MouseButton | undefined {
export type BrowserActionExtra =
| "console"
| "locator"
| "mouseClick"
| "mouseDrag"
| "mouseMove"
| "network"
| "pdf"
| "traceStart"
| "traceStop"
| "verifyElement"
| "verifyList"
| "verifyText"
@@ -77,17 +68,6 @@ export async function handleBrowserActionExtra(
res.json({ ok: true, messages, targetId: tab.targetId });
return true;
}
case "network": {
const includeStatic = toBoolean(args.includeStatic) ?? false;
const tab = await ctx.ensureTabAvailable(target);
const requests = await getNetworkRequestsViaPlaywright({
cdpPort,
targetId: tab.targetId,
includeStatic,
});
res.json({ ok: true, requests, targetId: tab.targetId });
return true;
}
case "pdf": {
const tab = await ctx.ensureTabAvailable(target);
const pdf = await pdfViaPlaywright({
@@ -109,36 +89,6 @@ export async function handleBrowserActionExtra(
});
return true;
}
case "traceStart": {
const tab = await ctx.ensureTabAvailable(target);
await startTracingViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
res.json({ ok: true });
return true;
}
case "traceStop": {
const tab = await ctx.ensureTabAvailable(target);
const trace = await stopTracingViaPlaywright({
cdpPort,
targetId: tab.targetId,
});
await ensureMediaDir();
const saved = await saveMediaBuffer(
trace.buffer,
"application/zip",
"browser",
trace.buffer.byteLength,
);
res.json({
ok: true,
path: path.resolve(saved.path),
targetId: tab.targetId,
url: tab.url,
});
return true;
}
case "verifyElement": {
const role = toStringOrEmpty(args.role);
const accessibleName = toStringOrEmpty(args.accessibleName);
@@ -269,16 +219,6 @@ export async function handleBrowserActionExtra(
res.json({ ok: true });
return true;
}
case "locator": {
const ref = toStringOrEmpty(args.ref);
if (!ref) {
jsonError(res, 400, "ref is required");
return true;
}
const locator = generateLocatorForRef(ref);
res.json({ ok: true, locator });
return true;
}
default:
return false;
}

25
src/browser/routes/actions.ts
View File

@@ -3,7 +3,7 @@ import type express from "express";
import type { BrowserRouteContext } from "../server-context.js";
import { handleBrowserActionCore } from "./actions-core.js";
import { handleBrowserActionExtra } from "./actions-extra.js";
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
import { jsonError, toStringOrEmpty } from "./utils.js";
function readBody(req: express.Request): Record<string, unknown> {
const body = req.body as Record<string, unknown> | undefined;
@@ -176,24 +176,6 @@ export function registerBrowserActionRoutes(
await runExtraAction(ctx, res, "console", args, targetId);
});
app.get("/network", async (req, res) => {
const targetId = readTargetId(req.query.targetId);
const includeStatic = toBoolean(req.query.includeStatic) ?? false;
await runExtraAction(ctx, res, "network", { includeStatic }, targetId);
});
app.post("/trace/start", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStart", body, targetId);
});
app.post("/trace/stop", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "traceStop", body, targetId);
});
app.post("/pdf", async (req, res) => {
const body = readBody(req);
const targetId = readTargetId(body.targetId);
@@ -241,9 +223,4 @@ export function registerBrowserActionRoutes(
const targetId = readTargetId(body.targetId);
await runExtraAction(ctx, res, "mouseDrag", body, targetId);
});
app.post("/locator", async (req, res) => {
const body = readBody(req);
await runExtraAction(ctx, res, "locator", body, "");
});
}

6
src/browser/server.test.ts
View File

@@ -104,9 +104,7 @@ vi.mock("./pw-ai.js", () => ({
evaluateViaPlaywright: vi.fn(async () => "ok"),
fileUploadViaPlaywright: vi.fn(async () => {}),
fillFormViaPlaywright: vi.fn(async () => {}),
generateLocatorForRef: vi.fn((ref: string) => `locator('aria-ref=${ref}')`),
getConsoleMessagesViaPlaywright: vi.fn(async () => []),
getNetworkRequestsViaPlaywright: vi.fn(async () => []),
handleDialogViaPlaywright: vi.fn(async () => ({
message: "ok",
type: "alert",
@@ -123,10 +121,6 @@ vi.mock("./pw-ai.js", () => ({
runCodeViaPlaywright: vi.fn(async () => "ok"),
selectOptionViaPlaywright: vi.fn(async () => {}),
snapshotAiViaPlaywright: vi.fn(async () => ({ snapshot: "ok" })),
startTracingViaPlaywright: vi.fn(async () => {}),
stopTracingViaPlaywright: vi.fn(async () => ({
buffer: Buffer.from("trace"),
})),
takeScreenshotViaPlaywright: vi.fn(async () => ({
buffer: Buffer.from("png"),
})),

92
src/cli/browser-cli-actions-observe.ts
View File

@@ -2,14 +2,10 @@ import type { Command } from "commander";
import { resolveBrowserControlUrl } from "../browser/client.js";
import {
browserConsoleMessages,
browserGenerateLocator,
browserMouseClick,
browserMouseDrag,
browserMouseMove,
browserNetworkRequests,
browserPdfSave,
browserStartTracing,
browserStopTracing,
browserVerifyElementVisible,
browserVerifyListVisible,
browserVerifyTextVisible,
@@ -47,74 +43,6 @@ export function registerBrowserActionObserveCommands(
}
});
browser
.command("network")
.description("Get recent network requests")
.option("--include-static", "Include static assets", false)
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const result = await browserNetworkRequests(baseUrl, {
includeStatic: Boolean(opts.includeStatic),
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(JSON.stringify(result.requests, null, 2));
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("trace-start")
.description("Start Playwright tracing")
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const result = await browserStartTracing(baseUrl, {
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log("trace started");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("trace-stop")
.description("Stop tracing and save a trace.zip")
.option("--target-id <id>", "CDP target id (or unique prefix)")
.action(async (opts, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const result = await browserStopTracing(baseUrl, {
targetId: opts.targetId?.trim() || undefined,
});
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(`trace: ${result.path}`);
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
browser
.command("pdf")
.description("Save page as PDF")
@@ -356,24 +284,4 @@ export function registerBrowserActionObserveCommands(
defaultRuntime.exit(1);
}
});
browser
.command("locator")
.description("Generate a Playwright locator for a ref")
.argument("<ref>", "Ref id from ai snapshot")
.action(async (ref: string, cmd) => {
const parent = parentOpts(cmd);
const baseUrl = resolveBrowserControlUrl(parent?.url);
try {
const result = await browserGenerateLocator(baseUrl, { ref });
if (parent?.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
defaultRuntime.log(result.locator);
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

4
src/cli/browser-cli-examples.ts
View File

@@ -33,9 +33,6 @@ export const browserActionExamples = [
"clawdis browser evaluate --fn '(el) => el.textContent' --ref 7",
"clawdis browser run --code '(page) => page.title()'",
"clawdis browser console --level error",
"clawdis browser network --include-static",
"clawdis browser trace-start",
"clawdis browser trace-stop",
"clawdis browser pdf",
'clawdis browser verify-element --role button --name "Submit"',
'clawdis browser verify-text "Welcome"',
@@ -44,5 +41,4 @@ export const browserActionExamples = [
"clawdis browser mouse-move --x 120 --y 240",
"clawdis browser mouse-click --x 120 --y 240",
"clawdis browser mouse-drag --start-x 10 --start-y 20 --end-x 200 --end-y 300",
"clawdis browser locator 77",
];