feat: add node browser proxy routing
This commit is contained in:
@@ -35,7 +35,7 @@ const BROWSER_TOOL_ACTIONS = [
|
||||
"act",
|
||||
] as const;
|
||||
|
||||
const BROWSER_TARGETS = ["sandbox", "host", "custom"] as const;
|
||||
const BROWSER_TARGETS = ["sandbox", "host", "custom", "node"] as const;
|
||||
|
||||
const BROWSER_SNAPSHOT_FORMATS = ["aria", "ai"] as const;
|
||||
const BROWSER_SNAPSHOT_MODES = ["efficient"] as const;
|
||||
@@ -84,6 +84,7 @@ const BrowserActSchema = Type.Object({
|
||||
export const BrowserToolSchema = Type.Object({
|
||||
action: stringEnum(BROWSER_TOOL_ACTIONS),
|
||||
target: optionalStringEnum(BROWSER_TARGETS),
|
||||
node: Type.Optional(Type.String()),
|
||||
profile: Type.Optional(Type.String()),
|
||||
controlUrl: Type.Optional(Type.String()),
|
||||
targetUrl: Type.Optional(Type.String()),
|
||||
|
||||
@@ -49,6 +49,25 @@ const browserConfigMocks = vi.hoisted(() => ({
|
||||
}));
|
||||
vi.mock("../../browser/config.js", () => browserConfigMocks);
|
||||
|
||||
const nodesUtilsMocks = vi.hoisted(() => ({
|
||||
listNodes: vi.fn(async () => []),
|
||||
}));
|
||||
vi.mock("./nodes-utils.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("./nodes-utils.js")>("./nodes-utils.js");
|
||||
return {
|
||||
...actual,
|
||||
listNodes: nodesUtilsMocks.listNodes,
|
||||
};
|
||||
});
|
||||
|
||||
const gatewayMocks = vi.hoisted(() => ({
|
||||
callGatewayTool: vi.fn(async () => ({
|
||||
ok: true,
|
||||
payload: { result: { ok: true, running: true } },
|
||||
})),
|
||||
}));
|
||||
vi.mock("./gateway.js", () => gatewayMocks);
|
||||
|
||||
const configMocks = vi.hoisted(() => ({
|
||||
loadConfig: vi.fn(() => ({ browser: {} })),
|
||||
}));
|
||||
@@ -72,6 +91,7 @@ describe("browser tool snapshot maxChars", () => {
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
configMocks.loadConfig.mockReturnValue({ browser: {} });
|
||||
nodesUtilsMocks.listNodes.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it("applies the default ai snapshot limit", async () => {
|
||||
@@ -175,6 +195,70 @@ describe("browser tool snapshot maxChars", () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes to node proxy when target=node", async () => {
|
||||
nodesUtilsMocks.listNodes.mockResolvedValue([
|
||||
{
|
||||
nodeId: "node-1",
|
||||
displayName: "Browser Node",
|
||||
connected: true,
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy"],
|
||||
},
|
||||
]);
|
||||
const tool = createBrowserTool();
|
||||
await tool.execute?.(null, { action: "status", target: "node" });
|
||||
|
||||
expect(gatewayMocks.callGatewayTool).toHaveBeenCalledWith(
|
||||
"node.invoke",
|
||||
{ timeoutMs: 20000 },
|
||||
expect.objectContaining({
|
||||
nodeId: "node-1",
|
||||
command: "browser.proxy",
|
||||
}),
|
||||
);
|
||||
expect(browserClientMocks.browserStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps sandbox control url when node proxy is available", async () => {
|
||||
nodesUtilsMocks.listNodes.mockResolvedValue([
|
||||
{
|
||||
nodeId: "node-1",
|
||||
displayName: "Browser Node",
|
||||
connected: true,
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy"],
|
||||
},
|
||||
]);
|
||||
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
|
||||
await tool.execute?.(null, { action: "status" });
|
||||
|
||||
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
|
||||
"http://127.0.0.1:9999",
|
||||
expect.objectContaining({ profile: undefined }),
|
||||
);
|
||||
expect(gatewayMocks.callGatewayTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps chrome profile on host when node proxy is available", async () => {
|
||||
nodesUtilsMocks.listNodes.mockResolvedValue([
|
||||
{
|
||||
nodeId: "node-1",
|
||||
displayName: "Browser Node",
|
||||
connected: true,
|
||||
caps: ["browser"],
|
||||
commands: ["browser.proxy"],
|
||||
},
|
||||
]);
|
||||
const tool = createBrowserTool();
|
||||
await tool.execute?.(null, { action: "status", profile: "chrome" });
|
||||
|
||||
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
|
||||
"http://127.0.0.1:18791",
|
||||
expect.objectContaining({ profile: "chrome" }),
|
||||
);
|
||||
expect(gatewayMocks.callGatewayTool).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser tool snapshot labels", () => {
|
||||
|
||||
@@ -18,11 +18,173 @@ import {
|
||||
browserPdfSave,
|
||||
browserScreenshotAction,
|
||||
} from "../../browser/client-actions.js";
|
||||
import crypto from "node:crypto";
|
||||
|
||||
import { resolveBrowserConfig } from "../../browser/config.js";
|
||||
import { DEFAULT_AI_SNAPSHOT_MAX_CHARS } from "../../browser/constants.js";
|
||||
import { loadConfig } from "../../config/config.js";
|
||||
import { saveMediaBuffer } from "../../media/store.js";
|
||||
import { listNodes, resolveNodeIdFromList, type NodeListNode } from "./nodes-utils.js";
|
||||
import { BrowserToolSchema } from "./browser-tool.schema.js";
|
||||
import { type AnyAgentTool, imageResultFromFile, jsonResult, readStringParam } from "./common.js";
|
||||
import { callGatewayTool } from "./gateway.js";
|
||||
|
||||
type BrowserProxyFile = {
|
||||
path: string;
|
||||
base64: string;
|
||||
mimeType?: string;
|
||||
};
|
||||
|
||||
type BrowserProxyResult = {
|
||||
result: unknown;
|
||||
files?: BrowserProxyFile[];
|
||||
};
|
||||
|
||||
const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000;
|
||||
|
||||
type BrowserNodeTarget = {
|
||||
nodeId: string;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
function isBrowserNode(node: NodeListNode) {
|
||||
const caps = Array.isArray(node.caps) ? node.caps : [];
|
||||
const commands = Array.isArray(node.commands) ? node.commands : [];
|
||||
return caps.includes("browser") || commands.includes("browser.proxy");
|
||||
}
|
||||
|
||||
async function resolveBrowserNodeTarget(params: {
|
||||
requestedNode?: string;
|
||||
target?: "sandbox" | "host" | "custom" | "node";
|
||||
controlUrl?: string;
|
||||
defaultControlUrl?: string;
|
||||
}): Promise<BrowserNodeTarget | null> {
|
||||
const cfg = loadConfig();
|
||||
const policy = cfg.gateway?.nodes?.browser;
|
||||
const mode = policy?.mode ?? "auto";
|
||||
if (mode === "off") {
|
||||
if (params.target === "node" || params.requestedNode) {
|
||||
throw new Error("Node browser proxy is disabled (gateway.nodes.browser.mode=off).");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
if (params.defaultControlUrl?.trim() && params.target !== "node" && !params.requestedNode) {
|
||||
return null;
|
||||
}
|
||||
if (params.controlUrl?.trim()) return null;
|
||||
if (params.target && params.target !== "node") return null;
|
||||
if (mode === "manual" && params.target !== "node" && !params.requestedNode) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const nodes = await listNodes({});
|
||||
const browserNodes = nodes.filter((node) => node.connected && isBrowserNode(node));
|
||||
if (browserNodes.length === 0) {
|
||||
if (params.target === "node" || params.requestedNode) {
|
||||
throw new Error("No connected browser-capable nodes.");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const requested = params.requestedNode?.trim() || policy?.node?.trim();
|
||||
if (requested) {
|
||||
const nodeId = resolveNodeIdFromList(browserNodes, requested, false);
|
||||
const node = browserNodes.find((entry) => entry.nodeId === nodeId);
|
||||
return { nodeId, label: node?.displayName ?? node?.remoteIp ?? nodeId };
|
||||
}
|
||||
|
||||
if (params.target === "node") {
|
||||
if (browserNodes.length === 1) {
|
||||
const node = browserNodes[0]!;
|
||||
return { nodeId: node.nodeId, label: node.displayName ?? node.remoteIp ?? node.nodeId };
|
||||
}
|
||||
throw new Error(
|
||||
`Multiple browser-capable nodes connected (${browserNodes.length}). Set gateway.nodes.browser.node or pass node=<id>.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (mode === "manual") return null;
|
||||
|
||||
if (browserNodes.length === 1) {
|
||||
const node = browserNodes[0]!;
|
||||
return { nodeId: node.nodeId, label: node.displayName ?? node.remoteIp ?? node.nodeId };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function callBrowserProxy(params: {
|
||||
nodeId: string;
|
||||
method: string;
|
||||
path: string;
|
||||
query?: Record<string, string | number | boolean | undefined>;
|
||||
body?: unknown;
|
||||
timeoutMs?: number;
|
||||
profile?: string;
|
||||
}): Promise<BrowserProxyResult> {
|
||||
const gatewayTimeoutMs =
|
||||
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
|
||||
? Math.max(1, Math.floor(params.timeoutMs))
|
||||
: DEFAULT_BROWSER_PROXY_TIMEOUT_MS;
|
||||
const payload = (await callGatewayTool(
|
||||
"node.invoke",
|
||||
{ timeoutMs: gatewayTimeoutMs },
|
||||
{
|
||||
nodeId: params.nodeId,
|
||||
command: "browser.proxy",
|
||||
params: {
|
||||
method: params.method,
|
||||
path: params.path,
|
||||
query: params.query,
|
||||
body: params.body,
|
||||
timeoutMs: params.timeoutMs,
|
||||
profile: params.profile,
|
||||
},
|
||||
idempotencyKey: crypto.randomUUID(),
|
||||
},
|
||||
)) as {
|
||||
ok?: boolean;
|
||||
payload?: BrowserProxyResult;
|
||||
payloadJSON?: string | null;
|
||||
};
|
||||
const parsed =
|
||||
payload?.payload ??
|
||||
(typeof payload?.payloadJSON === "string" && payload.payloadJSON
|
||||
? (JSON.parse(payload.payloadJSON) as BrowserProxyResult)
|
||||
: null);
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
throw new Error("browser proxy failed");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function persistProxyFiles(files: BrowserProxyFile[] | undefined) {
|
||||
if (!files || files.length === 0) return new Map<string, string>();
|
||||
const mapping = new Map<string, string>();
|
||||
for (const file of files) {
|
||||
const buffer = Buffer.from(file.base64, "base64");
|
||||
const saved = await saveMediaBuffer(buffer, file.mimeType, "browser", buffer.byteLength);
|
||||
mapping.set(file.path, saved.path);
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
function applyProxyPaths(result: unknown, mapping: Map<string, string>) {
|
||||
if (!result || typeof result !== "object") return;
|
||||
const obj = result as Record<string, unknown>;
|
||||
if (typeof obj.path === "string" && mapping.has(obj.path)) {
|
||||
obj.path = mapping.get(obj.path);
|
||||
}
|
||||
if (typeof obj.imagePath === "string" && mapping.has(obj.imagePath)) {
|
||||
obj.imagePath = mapping.get(obj.imagePath);
|
||||
}
|
||||
const download = obj.download;
|
||||
if (download && typeof download === "object") {
|
||||
const d = download as Record<string, unknown>;
|
||||
if (typeof d.path === "string" && mapping.has(d.path)) {
|
||||
d.path = mapping.get(d.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resolveBrowserBaseUrl(params: {
|
||||
target?: "sandbox" | "host" | "custom";
|
||||
@@ -127,11 +289,12 @@ export function createBrowserTool(opts?: {
|
||||
"Control the browser via Clawdbot's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions).",
|
||||
'Profiles: use profile="chrome" for Chrome extension relay takeover (your existing Chrome tabs). Use profile="clawd" for the isolated clawd-managed browser.',
|
||||
'If the user mentions the Chrome extension / Browser Relay / toolbar button / “attach tab”, ALWAYS use profile="chrome" (do not ask which profile).',
|
||||
'When a node-hosted browser proxy is available, the tool may auto-route to it. Pin a node with node=<id|name> or target="node".',
|
||||
"Chrome extension relay needs an attached tab: user must click the Clawdbot Browser Relay toolbar icon on the tab (badge ON). If no tab is connected, ask them to attach it.",
|
||||
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc).",
|
||||
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.',
|
||||
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
|
||||
`target selects browser location (sandbox|host|custom). Default: ${targetDefault}.`,
|
||||
`target selects browser location (sandbox|host|custom|node). Default: ${targetDefault}.`,
|
||||
"controlUrl implies target=custom (remote control server).",
|
||||
hostHint,
|
||||
allowlistHint,
|
||||
@@ -142,49 +305,184 @@ export function createBrowserTool(opts?: {
|
||||
const action = readStringParam(params, "action", { required: true });
|
||||
const controlUrl = readStringParam(params, "controlUrl");
|
||||
const profile = readStringParam(params, "profile");
|
||||
let target = readStringParam(params, "target") as "sandbox" | "host" | "custom" | undefined;
|
||||
if (profile === "chrome" && !target && !controlUrl?.trim()) {
|
||||
// Chrome extension relay takeover is a host Chrome feature; default to host even in sandboxed sessions.
|
||||
const requestedNode = readStringParam(params, "node");
|
||||
let target = readStringParam(params, "target") as
|
||||
| "sandbox"
|
||||
| "host"
|
||||
| "custom"
|
||||
| "node"
|
||||
| undefined;
|
||||
|
||||
if (controlUrl?.trim() && (target === "node" || requestedNode)) {
|
||||
throw new Error('controlUrl is not supported with target="node".');
|
||||
}
|
||||
if (target === "custom" && requestedNode) {
|
||||
throw new Error('node is not supported with target="custom".');
|
||||
}
|
||||
|
||||
if (!target && !controlUrl?.trim() && !requestedNode && profile === "chrome") {
|
||||
// Chrome extension relay takeover is a host Chrome feature; prefer host unless explicitly targeting a node.
|
||||
target = "host";
|
||||
}
|
||||
const baseUrl = resolveBrowserBaseUrl({
|
||||
|
||||
const nodeTarget = await resolveBrowserNodeTarget({
|
||||
requestedNode: requestedNode ?? undefined,
|
||||
target,
|
||||
controlUrl,
|
||||
defaultControlUrl: opts?.defaultControlUrl,
|
||||
allowHostControl: opts?.allowHostControl,
|
||||
allowedControlUrls: opts?.allowedControlUrls,
|
||||
allowedControlHosts: opts?.allowedControlHosts,
|
||||
allowedControlPorts: opts?.allowedControlPorts,
|
||||
});
|
||||
|
||||
const resolvedTarget = target === "node" ? undefined : target;
|
||||
const baseUrl = nodeTarget
|
||||
? ""
|
||||
: resolveBrowserBaseUrl({
|
||||
target: resolvedTarget,
|
||||
controlUrl,
|
||||
defaultControlUrl: opts?.defaultControlUrl,
|
||||
allowHostControl: opts?.allowHostControl,
|
||||
allowedControlUrls: opts?.allowedControlUrls,
|
||||
allowedControlHosts: opts?.allowedControlHosts,
|
||||
allowedControlPorts: opts?.allowedControlPorts,
|
||||
});
|
||||
|
||||
const proxyRequest = nodeTarget
|
||||
? async (opts: {
|
||||
method: string;
|
||||
path: string;
|
||||
query?: Record<string, string | number | boolean | undefined>;
|
||||
body?: unknown;
|
||||
timeoutMs?: number;
|
||||
profile?: string;
|
||||
}) => {
|
||||
const proxy = await callBrowserProxy({
|
||||
nodeId: nodeTarget.nodeId,
|
||||
method: opts.method,
|
||||
path: opts.path,
|
||||
query: opts.query,
|
||||
body: opts.body,
|
||||
timeoutMs: opts.timeoutMs,
|
||||
profile: opts.profile,
|
||||
});
|
||||
const mapping = await persistProxyFiles(proxy.files);
|
||||
applyProxyPaths(proxy.result, mapping);
|
||||
return proxy.result;
|
||||
}
|
||||
: null;
|
||||
|
||||
switch (action) {
|
||||
case "status":
|
||||
if (proxyRequest) {
|
||||
return jsonResult(
|
||||
await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/",
|
||||
profile,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return jsonResult(await browserStatus(baseUrl, { profile }));
|
||||
case "start":
|
||||
if (proxyRequest) {
|
||||
await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/start",
|
||||
profile,
|
||||
});
|
||||
return jsonResult(
|
||||
await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/",
|
||||
profile,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await browserStart(baseUrl, { profile });
|
||||
return jsonResult(await browserStatus(baseUrl, { profile }));
|
||||
case "stop":
|
||||
if (proxyRequest) {
|
||||
await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/stop",
|
||||
profile,
|
||||
});
|
||||
return jsonResult(
|
||||
await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/",
|
||||
profile,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await browserStop(baseUrl, { profile });
|
||||
return jsonResult(await browserStatus(baseUrl, { profile }));
|
||||
case "profiles":
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/profiles",
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult({ profiles: await browserProfiles(baseUrl) });
|
||||
case "tabs":
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/tabs",
|
||||
profile,
|
||||
});
|
||||
const tabs = (result as { tabs?: unknown[] }).tabs ?? [];
|
||||
return jsonResult({ tabs });
|
||||
}
|
||||
return jsonResult({ tabs: await browserTabs(baseUrl, { profile }) });
|
||||
case "open": {
|
||||
const targetUrl = readStringParam(params, "targetUrl", {
|
||||
required: true,
|
||||
});
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/tabs/open",
|
||||
profile,
|
||||
body: { url: targetUrl },
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult(await browserOpenTab(baseUrl, targetUrl, { profile }));
|
||||
}
|
||||
case "focus": {
|
||||
const targetId = readStringParam(params, "targetId", {
|
||||
required: true,
|
||||
});
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/tabs/focus",
|
||||
profile,
|
||||
body: { targetId },
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
await browserFocusTab(baseUrl, targetId, { profile });
|
||||
return jsonResult({ ok: true });
|
||||
}
|
||||
case "close": {
|
||||
const targetId = readStringParam(params, "targetId");
|
||||
if (proxyRequest) {
|
||||
const result = targetId
|
||||
? await proxyRequest({
|
||||
method: "DELETE",
|
||||
path: `/tabs/${encodeURIComponent(targetId)}`,
|
||||
profile,
|
||||
})
|
||||
: await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/act",
|
||||
profile,
|
||||
body: { kind: "close" },
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
if (targetId) await browserCloseTab(baseUrl, targetId, { profile });
|
||||
else await browserAct(baseUrl, { kind: "close" }, { profile });
|
||||
return jsonResult({ ok: true });
|
||||
@@ -232,21 +530,41 @@ export function createBrowserTool(opts?: {
|
||||
: undefined;
|
||||
const selector = typeof params.selector === "string" ? params.selector.trim() : undefined;
|
||||
const frame = typeof params.frame === "string" ? params.frame.trim() : undefined;
|
||||
const snapshot = await browserSnapshot(baseUrl, {
|
||||
format,
|
||||
targetId,
|
||||
limit,
|
||||
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
|
||||
refs,
|
||||
interactive,
|
||||
compact,
|
||||
depth,
|
||||
selector,
|
||||
frame,
|
||||
labels,
|
||||
mode,
|
||||
profile,
|
||||
});
|
||||
const snapshot = proxyRequest
|
||||
? ((await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/snapshot",
|
||||
profile,
|
||||
query: {
|
||||
format,
|
||||
targetId,
|
||||
limit,
|
||||
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
|
||||
refs,
|
||||
interactive,
|
||||
compact,
|
||||
depth,
|
||||
selector,
|
||||
frame,
|
||||
labels,
|
||||
mode,
|
||||
},
|
||||
})) as Awaited<ReturnType<typeof browserSnapshot>>)
|
||||
: await browserSnapshot(baseUrl, {
|
||||
format,
|
||||
targetId,
|
||||
limit,
|
||||
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
|
||||
refs,
|
||||
interactive,
|
||||
compact,
|
||||
depth,
|
||||
selector,
|
||||
frame,
|
||||
labels,
|
||||
mode,
|
||||
profile,
|
||||
});
|
||||
if (snapshot.format === "ai") {
|
||||
if (labels && snapshot.imagePath) {
|
||||
return await imageResultFromFile({
|
||||
@@ -269,14 +587,27 @@ export function createBrowserTool(opts?: {
|
||||
const ref = readStringParam(params, "ref");
|
||||
const element = readStringParam(params, "element");
|
||||
const type = params.type === "jpeg" ? "jpeg" : "png";
|
||||
const result = await browserScreenshotAction(baseUrl, {
|
||||
targetId,
|
||||
fullPage,
|
||||
ref,
|
||||
element,
|
||||
type,
|
||||
profile,
|
||||
});
|
||||
const result = proxyRequest
|
||||
? ((await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/screenshot",
|
||||
profile,
|
||||
body: {
|
||||
targetId,
|
||||
fullPage,
|
||||
ref,
|
||||
element,
|
||||
type,
|
||||
},
|
||||
})) as Awaited<ReturnType<typeof browserScreenshotAction>>)
|
||||
: await browserScreenshotAction(baseUrl, {
|
||||
targetId,
|
||||
fullPage,
|
||||
ref,
|
||||
element,
|
||||
type,
|
||||
profile,
|
||||
});
|
||||
return await imageResultFromFile({
|
||||
label: "browser:screenshot",
|
||||
path: result.path,
|
||||
@@ -288,6 +619,18 @@ export function createBrowserTool(opts?: {
|
||||
required: true,
|
||||
});
|
||||
const targetId = readStringParam(params, "targetId");
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/navigate",
|
||||
profile,
|
||||
body: {
|
||||
url: targetUrl,
|
||||
targetId,
|
||||
},
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult(
|
||||
await browserNavigate(baseUrl, {
|
||||
url: targetUrl,
|
||||
@@ -299,11 +642,30 @@ export function createBrowserTool(opts?: {
|
||||
case "console": {
|
||||
const level = typeof params.level === "string" ? params.level.trim() : undefined;
|
||||
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/console",
|
||||
profile,
|
||||
query: {
|
||||
level,
|
||||
targetId,
|
||||
},
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult(await browserConsoleMessages(baseUrl, { level, targetId, profile }));
|
||||
}
|
||||
case "pdf": {
|
||||
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
|
||||
const result = await browserPdfSave(baseUrl, { targetId, profile });
|
||||
const result = proxyRequest
|
||||
? ((await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/pdf",
|
||||
profile,
|
||||
body: { targetId },
|
||||
})) as Awaited<ReturnType<typeof browserPdfSave>>)
|
||||
: await browserPdfSave(baseUrl, { targetId, profile });
|
||||
return {
|
||||
content: [{ type: "text", text: `FILE:${result.path}` }],
|
||||
details: result,
|
||||
@@ -320,6 +682,22 @@ export function createBrowserTool(opts?: {
|
||||
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
|
||||
? params.timeoutMs
|
||||
: undefined;
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/hooks/file-chooser",
|
||||
profile,
|
||||
body: {
|
||||
paths,
|
||||
ref,
|
||||
inputRef,
|
||||
element,
|
||||
targetId,
|
||||
timeoutMs,
|
||||
},
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult(
|
||||
await browserArmFileChooser(baseUrl, {
|
||||
paths,
|
||||
@@ -340,6 +718,20 @@ export function createBrowserTool(opts?: {
|
||||
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
|
||||
? params.timeoutMs
|
||||
: undefined;
|
||||
if (proxyRequest) {
|
||||
const result = await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/hooks/dialog",
|
||||
profile,
|
||||
body: {
|
||||
accept,
|
||||
promptText,
|
||||
targetId,
|
||||
timeoutMs,
|
||||
},
|
||||
});
|
||||
return jsonResult(result);
|
||||
}
|
||||
return jsonResult(
|
||||
await browserArmDialog(baseUrl, {
|
||||
accept,
|
||||
@@ -356,14 +748,29 @@ export function createBrowserTool(opts?: {
|
||||
throw new Error("request required");
|
||||
}
|
||||
try {
|
||||
const result = await browserAct(baseUrl, request as Parameters<typeof browserAct>[1], {
|
||||
profile,
|
||||
});
|
||||
const result = proxyRequest
|
||||
? await proxyRequest({
|
||||
method: "POST",
|
||||
path: "/act",
|
||||
profile,
|
||||
body: request,
|
||||
})
|
||||
: await browserAct(baseUrl, request as Parameters<typeof browserAct>[1], {
|
||||
profile,
|
||||
});
|
||||
return jsonResult(result);
|
||||
} catch (err) {
|
||||
const msg = String(err);
|
||||
if (msg.includes("404:") && msg.includes("tab not found") && profile === "chrome") {
|
||||
const tabs = await browserTabs(baseUrl, { profile }).catch(() => []);
|
||||
const tabs = proxyRequest
|
||||
? ((
|
||||
(await proxyRequest({
|
||||
method: "GET",
|
||||
path: "/tabs",
|
||||
profile,
|
||||
})) as { tabs?: unknown[] }
|
||||
).tabs ?? [])
|
||||
: await browserTabs(baseUrl, { profile }).catch(() => []);
|
||||
if (!tabs.length) {
|
||||
throw new Error(
|
||||
"No Chrome tabs are attached via the Clawdbot Browser Relay extension. Click the toolbar icon on the tab you want to control (badge ON), then retry.",
|
||||
|
||||
Reference in New Issue
Block a user