From d4f7dc067ecb9992d29b2ec6eefe50839a4ac64c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 12 Jan 2026 19:40:52 +0000 Subject: [PATCH] feat(browser): add downloads + response bodies --- src/browser/client-actions-core.ts | 60 ++++++ src/browser/client-actions-observe.ts | 44 +++++ src/browser/pw-ai.ts | 3 + src/browser/pw-session.ts | 2 + src/browser/pw-tools-core.ts | 264 ++++++++++++++++++++++++- src/browser/routes/agent.ts | 76 +++++++ src/cli/browser-cli-actions-input.ts | 72 +++++++ src/cli/browser-cli-actions-observe.ts | 41 ++++ 8 files changed, 560 insertions(+), 2 deletions(-) diff --git a/src/browser/client-actions-core.ts b/src/browser/client-actions-core.ts index 80d21a890..58b260eda 100644 --- a/src/browser/client-actions-core.ts +++ b/src/browser/client-actions-core.ts @@ -79,6 +79,12 @@ export type BrowserActResponse = { result?: unknown; }; +export type BrowserDownloadPayload = { + url: string; + suggestedFilename: string; + path: string; +}; + export async function browserNavigate( baseUrl: string, opts: { url: string; targetId?: string; profile?: string }, @@ -153,6 +159,60 @@ export async function browserArmFileChooser( ); } +export async function browserWaitForDownload( + baseUrl: string, + opts: { + path?: string; + targetId?: string; + timeoutMs?: number; + profile?: string; + }, +): Promise<{ ok: true; targetId: string; download: BrowserDownloadPayload }> { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson<{ + ok: true; + targetId: string; + download: BrowserDownloadPayload; + }>(`${baseUrl}/wait/download${q}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + path: opts.path, + timeoutMs: opts.timeoutMs, + }), + timeoutMs: 20000, + }); +} + +export async function browserDownload( + baseUrl: string, + opts: { + ref: string; + path: string; + targetId?: string; + timeoutMs?: number; + profile?: string; + }, +): Promise<{ ok: true; targetId: string; download: BrowserDownloadPayload }> { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson<{ + ok: true; + targetId: string; + download: BrowserDownloadPayload; + }>(`${baseUrl}/download${q}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + ref: opts.ref, + path: opts.path, + timeoutMs: opts.timeoutMs, + }), + timeoutMs: 20000, + }); +} + export async function browserAct( baseUrl: string, req: BrowserActRequest, diff --git a/src/browser/client-actions-observe.ts b/src/browser/client-actions-observe.ts index b4db4aad2..651f597c9 100644 --- a/src/browser/client-actions-observe.ts +++ b/src/browser/client-actions-observe.ts @@ -138,3 +138,47 @@ export async function browserHighlight( }, ); } + +export async function browserResponseBody( + baseUrl: string, + opts: { + url: string; + targetId?: string; + timeoutMs?: number; + maxChars?: number; + profile?: string; + }, +): Promise<{ + ok: true; + targetId: string; + response: { + url: string; + status?: number; + headers?: Record; + body: string; + truncated?: boolean; + }; +}> { + const q = buildProfileQuery(opts.profile); + return await fetchBrowserJson<{ + ok: true; + targetId: string; + response: { + url: string; + status?: number; + headers?: Record; + body: string; + truncated?: boolean; + }; + }>(`${baseUrl}/response/body${q}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetId: opts.targetId, + url: opts.url, + timeoutMs: opts.timeoutMs, + maxChars: opts.maxChars, + }), + timeoutMs: 20000, + }); +} diff --git a/src/browser/pw-ai.ts b/src/browser/pw-ai.ts index 6ea78c72e..a0367d4cc 100644 --- a/src/browser/pw-ai.ts +++ b/src/browser/pw-ai.ts @@ -15,6 +15,7 @@ export { cookiesClearViaPlaywright, cookiesGetViaPlaywright, cookiesSetViaPlaywright, + downloadViaPlaywright, dragViaPlaywright, emulateMediaViaPlaywright, evaluateViaPlaywright, @@ -28,6 +29,7 @@ export { pdfViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, + responseBodyViaPlaywright, selectOptionViaPlaywright, setDeviceViaPlaywright, setExtraHTTPHeadersViaPlaywright, @@ -46,5 +48,6 @@ export { traceStartViaPlaywright, traceStopViaPlaywright, typeViaPlaywright, + waitForDownloadViaPlaywright, waitForViaPlaywright, } from "./pw-tools-core.js"; diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index b3a1227f8..1a3c569d7 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -63,6 +63,7 @@ type PageState = { nextRequestId: number; armIdUpload: number; armIdDialog: number; + armIdDownload: number; /** * Role-based refs from the last role snapshot (e.g. e1/e2). * These refs are NOT Playwright's `aria-ref` values. @@ -103,6 +104,7 @@ export function ensurePageState(page: Page): PageState { nextRequestId: 0, armIdUpload: 0, armIdDialog: 0, + armIdDownload: 0, }; pageStates.set(page, state); diff --git a/src/browser/pw-tools-core.ts b/src/browser/pw-tools-core.ts index c16ba3ac6..8f66fb252 100644 --- a/src/browser/pw-tools-core.ts +++ b/src/browser/pw-tools-core.ts @@ -1,3 +1,7 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + import type { CDPSession, Page } from "playwright-core"; import { devices as playwrightDevices } from "playwright-core"; import type { BrowserFormField } from "./client-actions-core.js"; @@ -20,6 +24,7 @@ import { let nextUploadArmId = 0; let nextDialogArmId = 0; +let nextDownloadArmId = 0; function requireRef(value: unknown): string { const raw = typeof value === "string" ? value.trim() : ""; @@ -29,6 +34,71 @@ function requireRef(value: unknown): string { return ref; } +function buildTempDownloadPath(fileName: string): string { + const id = crypto.randomUUID(); + const safeName = fileName.trim() ? fileName.trim() : "download.bin"; + return path.join("/tmp/clawdbot/downloads", `${id}-${safeName}`); +} + +function normalizeTimeoutMs(timeoutMs: number | undefined, fallback: number) { + return Math.max(500, Math.min(120_000, timeoutMs ?? fallback)); +} + +function createPageDownloadWaiter(page: Page, timeoutMs: number) { + let done = false; + let timer: NodeJS.Timeout | undefined; + let handler: ((download: unknown) => void) | undefined; + + const cleanup = () => { + if (timer) clearTimeout(timer); + timer = undefined; + if (handler) { + page.off("download", handler as never); + handler = undefined; + } + }; + + const promise = new Promise((resolve, reject) => { + handler = (download: unknown) => { + if (done) return; + done = true; + cleanup(); + resolve(download); + }; + + page.on("download", handler as never); + timer = setTimeout(() => { + if (done) return; + done = true; + cleanup(); + reject(new Error("Timeout waiting for download")); + }, timeoutMs); + }); + + return { + promise, + cancel: () => { + if (done) return; + done = true; + cleanup(); + }, + }; +} + +function matchUrlPattern(pattern: string, url: string): boolean { + const p = pattern.trim(); + if (!p) return false; + if (p === url) return true; + if (p.includes("*")) { + const escaped = p.replace(/[|\\{}()[\]^$+?.]/g, "\\$&"); + const regex = new RegExp( + `^${escaped.replace(/\*\*/g, ".*").replace(/\*/g, ".*")}$`, + ); + return regex.test(url); + } + return url.includes(p); +} + function toAIFriendlyError(error: unknown, selector: string): Error { const message = error instanceof Error ? error.message : String(error); @@ -883,7 +953,7 @@ export async function armDialogViaPlaywright(opts: { }): Promise { const page = await getPageForTargetId(opts); const state = ensurePageState(page); - const timeout = Math.max(500, Math.min(120_000, opts.timeoutMs ?? 120_000)); + const timeout = normalizeTimeoutMs(opts.timeoutMs, 120_000); state.armIdDialog = nextDialogArmId += 1; const armId = state.armIdDialog; @@ -900,6 +970,196 @@ export async function armDialogViaPlaywright(opts: { }); } +export async function waitForDownloadViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + path?: string; + timeoutMs?: number; +}): Promise<{ + url: string; + suggestedFilename: string; + path: string; +}> { + const page = await getPageForTargetId(opts); + const state = ensurePageState(page); + const timeout = normalizeTimeoutMs(opts.timeoutMs, 120_000); + + state.armIdDownload = nextDownloadArmId += 1; + const armId = state.armIdDownload; + + const waiter = createPageDownloadWaiter(page, timeout); + try { + const download = (await waiter.promise) as { + url?: () => string; + suggestedFilename?: () => string; + saveAs?: (outPath: string) => Promise; + }; + if (state.armIdDownload !== armId) { + throw new Error("Download was superseded by another waiter"); + } + const suggested = download.suggestedFilename?.() || "download.bin"; + const outPath = opts.path?.trim() || buildTempDownloadPath(suggested); + await fs.mkdir(path.dirname(outPath), { recursive: true }); + await download.saveAs?.(outPath); + return { + url: download.url?.() || "", + suggestedFilename: suggested, + path: path.resolve(outPath), + }; + } catch (err) { + waiter.cancel(); + throw err; + } +} + +export async function downloadViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + ref: string; + path: string; + timeoutMs?: number; +}): Promise<{ + url: string; + suggestedFilename: string; + path: string; +}> { + const page = await getPageForTargetId(opts); + const state = ensurePageState(page); + const timeout = normalizeTimeoutMs(opts.timeoutMs, 120_000); + + const ref = requireRef(opts.ref); + const outPath = String(opts.path ?? "").trim(); + if (!outPath) throw new Error("path is required"); + + state.armIdDownload = nextDownloadArmId += 1; + const armId = state.armIdDownload; + + const waiter = createPageDownloadWaiter(page, timeout); + try { + const locator = refLocator(page, ref); + try { + await locator.click({ timeout }); + } catch (err) { + throw toAIFriendlyError(err, ref); + } + + const download = (await waiter.promise) as { + url?: () => string; + suggestedFilename?: () => string; + saveAs?: (outPath: string) => Promise; + }; + if (state.armIdDownload !== armId) { + throw new Error("Download was superseded by another waiter"); + } + const suggested = download.suggestedFilename?.() || "download.bin"; + await fs.mkdir(path.dirname(outPath), { recursive: true }); + await download.saveAs?.(outPath); + return { + url: download.url?.() || "", + suggestedFilename: suggested, + path: path.resolve(outPath), + }; + } catch (err) { + waiter.cancel(); + throw err; + } +} + +export async function responseBodyViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + url: string; + timeoutMs?: number; + maxChars?: number; +}): Promise<{ + url: string; + status?: number; + headers?: Record; + body: string; + truncated?: boolean; +}> { + const pattern = String(opts.url ?? "").trim(); + if (!pattern) throw new Error("url is required"); + const maxChars = + typeof opts.maxChars === "number" && Number.isFinite(opts.maxChars) + ? Math.max(1, Math.min(5_000_000, Math.floor(opts.maxChars))) + : 200_000; + const timeout = normalizeTimeoutMs(opts.timeoutMs, 20_000); + + const page = await getPageForTargetId(opts); + ensurePageState(page); + + const promise = new Promise((resolve, reject) => { + let done = false; + let timer: NodeJS.Timeout | undefined; + let handler: ((resp: unknown) => void) | undefined; + + const cleanup = () => { + if (timer) clearTimeout(timer); + timer = undefined; + if (handler) page.off("response", handler as never); + }; + + handler = (resp: unknown) => { + if (done) return; + const r = resp as { url?: () => string }; + const u = r.url?.() || ""; + if (!matchUrlPattern(pattern, u)) return; + done = true; + cleanup(); + resolve(resp); + }; + + page.on("response", handler as never); + timer = setTimeout(() => { + if (done) return; + done = true; + cleanup(); + reject( + new Error( + `Response not found for url pattern "${pattern}". Run 'clawdbot browser requests' to inspect recent network activity.`, + ), + ); + }, timeout); + }); + + const resp = (await promise) as { + url?: () => string; + status?: () => number; + headers?: () => Record; + body?: () => Promise; + text?: () => Promise; + }; + + const url = resp.url?.() || ""; + const status = resp.status?.(); + const headers = resp.headers?.(); + + let bodyText = ""; + try { + if (typeof resp.text === "function") { + bodyText = await resp.text(); + } else if (typeof resp.body === "function") { + const buf = await resp.body(); + bodyText = new TextDecoder("utf-8").decode(buf); + } + } catch (err) { + throw new Error( + `Failed to read response body for "${url}": ${String(err)}`, + ); + } + + const trimmed = + bodyText.length > maxChars ? bodyText.slice(0, maxChars) : bodyText; + return { + url, + status, + headers, + body: trimmed, + truncated: bodyText.length > maxChars ? true : undefined, + }; +} + export async function navigateViaPlaywright(opts: { cdpUrl: string; targetId?: string; @@ -930,7 +1190,7 @@ export async function waitForViaPlaywright(opts: { }): Promise { const page = await getPageForTargetId(opts); ensurePageState(page); - const timeout = Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)); + const timeout = normalizeTimeoutMs(opts.timeoutMs, 20_000); if (typeof opts.timeMs === "number" && Number.isFinite(opts.timeMs)) { await page.waitForTimeout(Math.max(0, opts.timeMs)); diff --git a/src/browser/routes/agent.ts b/src/browser/routes/agent.ts index c9b737e6a..2de5e21eb 100644 --- a/src/browser/routes/agent.ts +++ b/src/browser/routes/agent.ts @@ -481,6 +481,82 @@ export function registerBrowserAgentRoutes( } }); + app.post("/wait/download", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const out = toStringOrEmpty(body.path) || undefined; + const timeoutMs = toNumber(body.timeoutMs); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "wait for download"); + if (!pw) return; + const result = await pw.waitForDownloadViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + path: out, + timeoutMs: timeoutMs ?? undefined, + }); + res.json({ ok: true, targetId: tab.targetId, download: result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/download", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const ref = toStringOrEmpty(body.ref); + const out = toStringOrEmpty(body.path); + const timeoutMs = toNumber(body.timeoutMs); + if (!ref) return jsonError(res, 400, "ref is required"); + if (!out) return jsonError(res, 400, "path is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "download"); + if (!pw) return; + const result = await pw.downloadViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + ref, + path: out, + timeoutMs: timeoutMs ?? undefined, + }); + res.json({ ok: true, targetId: tab.targetId, download: result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + + app.post("/response/body", async (req, res) => { + const profileCtx = resolveProfileContext(req, res, ctx); + if (!profileCtx) return; + const body = readBody(req); + const targetId = toStringOrEmpty(body.targetId) || undefined; + const url = toStringOrEmpty(body.url); + const timeoutMs = toNumber(body.timeoutMs); + const maxChars = toNumber(body.maxChars); + if (!url) return jsonError(res, 400, "url is required"); + try { + const tab = await profileCtx.ensureTabAvailable(targetId); + const pw = await requirePwAi(res, "response body"); + if (!pw) return; + const result = await pw.responseBodyViaPlaywright({ + cdpUrl: profileCtx.profile.cdpUrl, + targetId: tab.targetId, + url, + timeoutMs: timeoutMs ?? undefined, + maxChars: maxChars ?? undefined, + }); + res.json({ ok: true, targetId: tab.targetId, response: result }); + } catch (err) { + handleRouteError(ctx, res, err); + } + }); + app.get("/console", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) return; diff --git a/src/cli/browser-cli-actions-input.ts b/src/cli/browser-cli-actions-input.ts index 22882fd40..7fd05d49e 100644 --- a/src/cli/browser-cli-actions-input.ts +++ b/src/cli/browser-cli-actions-input.ts @@ -4,7 +4,9 @@ import { browserAct, browserArmDialog, browserArmFileChooser, + browserDownload, browserNavigate, + browserWaitForDownload, } from "../browser/client-actions.js"; import type { BrowserFormField } from "../browser/client-actions-core.js"; import { danger } from "../globals.js"; @@ -374,6 +376,76 @@ export function registerBrowserActionInputCommands( } }); + browser + .command("waitfordownload") + .description("Wait for the next download (and save it)") + .argument("[path]", "Save path (default: /tmp/clawdbot/downloads/...)") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the next download (default: 120000)", + (v: string) => Number(v), + ) + .action(async (outPath: string | undefined, opts, cmd) => { + const parent = parentOpts(cmd); + const baseUrl = resolveBrowserControlUrl(parent?.url); + const profile = parent?.browserProfile; + try { + const result = await browserWaitForDownload(baseUrl, { + path: outPath?.trim() || undefined, + targetId: opts.targetId?.trim() || undefined, + timeoutMs: Number.isFinite(opts.timeoutMs) + ? opts.timeoutMs + : undefined, + profile, + }); + if (parent?.json) { + defaultRuntime.log(JSON.stringify(result, null, 2)); + return; + } + defaultRuntime.log(`downloaded: ${result.download.path}`); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + + browser + .command("download") + .description("Click a ref and save the resulting download") + .argument("", "Ref id from snapshot to click") + .argument("", "Save path") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the download to start (default: 120000)", + (v: string) => Number(v), + ) + .action(async (ref: string, outPath: string, opts, cmd) => { + const parent = parentOpts(cmd); + const baseUrl = resolveBrowserControlUrl(parent?.url); + const profile = parent?.browserProfile; + try { + const result = await browserDownload(baseUrl, { + ref, + path: outPath, + targetId: opts.targetId?.trim() || undefined, + timeoutMs: Number.isFinite(opts.timeoutMs) + ? opts.timeoutMs + : undefined, + profile, + }); + if (parent?.json) { + defaultRuntime.log(JSON.stringify(result, null, 2)); + return; + } + defaultRuntime.log(`downloaded: ${result.download.path}`); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); + browser .command("fill") .description("Fill a form with JSON field descriptors") diff --git a/src/cli/browser-cli-actions-observe.ts b/src/cli/browser-cli-actions-observe.ts index 0eade88a6..e1f674c23 100644 --- a/src/cli/browser-cli-actions-observe.ts +++ b/src/cli/browser-cli-actions-observe.ts @@ -3,6 +3,7 @@ import { resolveBrowserControlUrl } from "../browser/client.js"; import { browserConsoleMessages, browserPdfSave, + browserResponseBody, } from "../browser/client-actions.js"; import { danger } from "../globals.js"; import { defaultRuntime } from "../runtime.js"; @@ -61,4 +62,44 @@ export function registerBrowserActionObserveCommands( defaultRuntime.exit(1); } }); + + browser + .command("responsebody") + .description("Wait for a network response and return its body") + .argument("", "URL (exact, substring, or glob like **/api)") + .option("--target-id ", "CDP target id (or unique prefix)") + .option( + "--timeout-ms ", + "How long to wait for the response (default: 20000)", + (v: string) => Number(v), + ) + .option( + "--max-chars ", + "Max body chars to return (default: 200000)", + (v: string) => Number(v), + ) + .action(async (url: string, opts, cmd) => { + const parent = parentOpts(cmd); + const baseUrl = resolveBrowserControlUrl(parent?.url); + const profile = parent?.browserProfile; + try { + const result = await browserResponseBody(baseUrl, { + url, + targetId: opts.targetId?.trim() || undefined, + timeoutMs: Number.isFinite(opts.timeoutMs) + ? opts.timeoutMs + : undefined, + maxChars: Number.isFinite(opts.maxChars) ? opts.maxChars : undefined, + profile, + }); + if (parent?.json) { + defaultRuntime.log(JSON.stringify(result, null, 2)); + return; + } + defaultRuntime.log(result.response.body); + } catch (err) { + defaultRuntime.error(danger(String(err))); + defaultRuntime.exit(1); + } + }); }