feat: add node browser proxy routing

This commit is contained in:
Peter Steinberger
2026-01-24 04:19:43 +00:00
parent dd06028827
commit c3cb26f7ca
14 changed files with 834 additions and 39 deletions

View File

@@ -35,7 +35,7 @@ const BROWSER_TOOL_ACTIONS = [
"act",
] as const;
const BROWSER_TARGETS = ["sandbox", "host", "custom"] as const;
const BROWSER_TARGETS = ["sandbox", "host", "custom", "node"] as const;
const BROWSER_SNAPSHOT_FORMATS = ["aria", "ai"] as const;
const BROWSER_SNAPSHOT_MODES = ["efficient"] as const;
@@ -84,6 +84,7 @@ const BrowserActSchema = Type.Object({
export const BrowserToolSchema = Type.Object({
action: stringEnum(BROWSER_TOOL_ACTIONS),
target: optionalStringEnum(BROWSER_TARGETS),
node: Type.Optional(Type.String()),
profile: Type.Optional(Type.String()),
controlUrl: Type.Optional(Type.String()),
targetUrl: Type.Optional(Type.String()),

View File

@@ -49,6 +49,25 @@ const browserConfigMocks = vi.hoisted(() => ({
}));
vi.mock("../../browser/config.js", () => browserConfigMocks);
const nodesUtilsMocks = vi.hoisted(() => ({
listNodes: vi.fn(async () => []),
}));
vi.mock("./nodes-utils.js", async () => {
const actual = await vi.importActual<typeof import("./nodes-utils.js")>("./nodes-utils.js");
return {
...actual,
listNodes: nodesUtilsMocks.listNodes,
};
});
const gatewayMocks = vi.hoisted(() => ({
callGatewayTool: vi.fn(async () => ({
ok: true,
payload: { result: { ok: true, running: true } },
})),
}));
vi.mock("./gateway.js", () => gatewayMocks);
const configMocks = vi.hoisted(() => ({
loadConfig: vi.fn(() => ({ browser: {} })),
}));
@@ -72,6 +91,7 @@ describe("browser tool snapshot maxChars", () => {
afterEach(() => {
vi.clearAllMocks();
configMocks.loadConfig.mockReturnValue({ browser: {} });
nodesUtilsMocks.listNodes.mockResolvedValue([]);
});
it("applies the default ai snapshot limit", async () => {
@@ -175,6 +195,70 @@ describe("browser tool snapshot maxChars", () => {
}),
);
});
it("routes to node proxy when target=node", async () => {
nodesUtilsMocks.listNodes.mockResolvedValue([
{
nodeId: "node-1",
displayName: "Browser Node",
connected: true,
caps: ["browser"],
commands: ["browser.proxy"],
},
]);
const tool = createBrowserTool();
await tool.execute?.(null, { action: "status", target: "node" });
expect(gatewayMocks.callGatewayTool).toHaveBeenCalledWith(
"node.invoke",
{ timeoutMs: 20000 },
expect.objectContaining({
nodeId: "node-1",
command: "browser.proxy",
}),
);
expect(browserClientMocks.browserStatus).not.toHaveBeenCalled();
});
it("keeps sandbox control url when node proxy is available", async () => {
nodesUtilsMocks.listNodes.mockResolvedValue([
{
nodeId: "node-1",
displayName: "Browser Node",
connected: true,
caps: ["browser"],
commands: ["browser.proxy"],
},
]);
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
await tool.execute?.(null, { action: "status" });
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
"http://127.0.0.1:9999",
expect.objectContaining({ profile: undefined }),
);
expect(gatewayMocks.callGatewayTool).not.toHaveBeenCalled();
});
it("keeps chrome profile on host when node proxy is available", async () => {
nodesUtilsMocks.listNodes.mockResolvedValue([
{
nodeId: "node-1",
displayName: "Browser Node",
connected: true,
caps: ["browser"],
commands: ["browser.proxy"],
},
]);
const tool = createBrowserTool();
await tool.execute?.(null, { action: "status", profile: "chrome" });
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
expect.objectContaining({ profile: "chrome" }),
);
expect(gatewayMocks.callGatewayTool).not.toHaveBeenCalled();
});
});
describe("browser tool snapshot labels", () => {

View File

@@ -18,11 +18,173 @@ import {
browserPdfSave,
browserScreenshotAction,
} from "../../browser/client-actions.js";
import crypto from "node:crypto";
import { resolveBrowserConfig } from "../../browser/config.js";
import { DEFAULT_AI_SNAPSHOT_MAX_CHARS } from "../../browser/constants.js";
import { loadConfig } from "../../config/config.js";
import { saveMediaBuffer } from "../../media/store.js";
import { listNodes, resolveNodeIdFromList, type NodeListNode } from "./nodes-utils.js";
import { BrowserToolSchema } from "./browser-tool.schema.js";
import { type AnyAgentTool, imageResultFromFile, jsonResult, readStringParam } from "./common.js";
import { callGatewayTool } from "./gateway.js";
type BrowserProxyFile = {
path: string;
base64: string;
mimeType?: string;
};
type BrowserProxyResult = {
result: unknown;
files?: BrowserProxyFile[];
};
const DEFAULT_BROWSER_PROXY_TIMEOUT_MS = 20_000;
type BrowserNodeTarget = {
nodeId: string;
label?: string;
};
function isBrowserNode(node: NodeListNode) {
const caps = Array.isArray(node.caps) ? node.caps : [];
const commands = Array.isArray(node.commands) ? node.commands : [];
return caps.includes("browser") || commands.includes("browser.proxy");
}
async function resolveBrowserNodeTarget(params: {
requestedNode?: string;
target?: "sandbox" | "host" | "custom" | "node";
controlUrl?: string;
defaultControlUrl?: string;
}): Promise<BrowserNodeTarget | null> {
const cfg = loadConfig();
const policy = cfg.gateway?.nodes?.browser;
const mode = policy?.mode ?? "auto";
if (mode === "off") {
if (params.target === "node" || params.requestedNode) {
throw new Error("Node browser proxy is disabled (gateway.nodes.browser.mode=off).");
}
return null;
}
if (params.defaultControlUrl?.trim() && params.target !== "node" && !params.requestedNode) {
return null;
}
if (params.controlUrl?.trim()) return null;
if (params.target && params.target !== "node") return null;
if (mode === "manual" && params.target !== "node" && !params.requestedNode) {
return null;
}
const nodes = await listNodes({});
const browserNodes = nodes.filter((node) => node.connected && isBrowserNode(node));
if (browserNodes.length === 0) {
if (params.target === "node" || params.requestedNode) {
throw new Error("No connected browser-capable nodes.");
}
return null;
}
const requested = params.requestedNode?.trim() || policy?.node?.trim();
if (requested) {
const nodeId = resolveNodeIdFromList(browserNodes, requested, false);
const node = browserNodes.find((entry) => entry.nodeId === nodeId);
return { nodeId, label: node?.displayName ?? node?.remoteIp ?? nodeId };
}
if (params.target === "node") {
if (browserNodes.length === 1) {
const node = browserNodes[0]!;
return { nodeId: node.nodeId, label: node.displayName ?? node.remoteIp ?? node.nodeId };
}
throw new Error(
`Multiple browser-capable nodes connected (${browserNodes.length}). Set gateway.nodes.browser.node or pass node=<id>.`,
);
}
if (mode === "manual") return null;
if (browserNodes.length === 1) {
const node = browserNodes[0]!;
return { nodeId: node.nodeId, label: node.displayName ?? node.remoteIp ?? node.nodeId };
}
return null;
}
async function callBrowserProxy(params: {
nodeId: string;
method: string;
path: string;
query?: Record<string, string | number | boolean | undefined>;
body?: unknown;
timeoutMs?: number;
profile?: string;
}): Promise<BrowserProxyResult> {
const gatewayTimeoutMs =
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
? Math.max(1, Math.floor(params.timeoutMs))
: DEFAULT_BROWSER_PROXY_TIMEOUT_MS;
const payload = (await callGatewayTool(
"node.invoke",
{ timeoutMs: gatewayTimeoutMs },
{
nodeId: params.nodeId,
command: "browser.proxy",
params: {
method: params.method,
path: params.path,
query: params.query,
body: params.body,
timeoutMs: params.timeoutMs,
profile: params.profile,
},
idempotencyKey: crypto.randomUUID(),
},
)) as {
ok?: boolean;
payload?: BrowserProxyResult;
payloadJSON?: string | null;
};
const parsed =
payload?.payload ??
(typeof payload?.payloadJSON === "string" && payload.payloadJSON
? (JSON.parse(payload.payloadJSON) as BrowserProxyResult)
: null);
if (!parsed || typeof parsed !== "object") {
throw new Error("browser proxy failed");
}
return parsed;
}
async function persistProxyFiles(files: BrowserProxyFile[] | undefined) {
if (!files || files.length === 0) return new Map<string, string>();
const mapping = new Map<string, string>();
for (const file of files) {
const buffer = Buffer.from(file.base64, "base64");
const saved = await saveMediaBuffer(buffer, file.mimeType, "browser", buffer.byteLength);
mapping.set(file.path, saved.path);
}
return mapping;
}
function applyProxyPaths(result: unknown, mapping: Map<string, string>) {
if (!result || typeof result !== "object") return;
const obj = result as Record<string, unknown>;
if (typeof obj.path === "string" && mapping.has(obj.path)) {
obj.path = mapping.get(obj.path);
}
if (typeof obj.imagePath === "string" && mapping.has(obj.imagePath)) {
obj.imagePath = mapping.get(obj.imagePath);
}
const download = obj.download;
if (download && typeof download === "object") {
const d = download as Record<string, unknown>;
if (typeof d.path === "string" && mapping.has(d.path)) {
d.path = mapping.get(d.path);
}
}
}
function resolveBrowserBaseUrl(params: {
target?: "sandbox" | "host" | "custom";
@@ -127,11 +289,12 @@ export function createBrowserTool(opts?: {
"Control the browser via Clawdbot's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions).",
'Profiles: use profile="chrome" for Chrome extension relay takeover (your existing Chrome tabs). Use profile="clawd" for the isolated clawd-managed browser.',
'If the user mentions the Chrome extension / Browser Relay / toolbar button / “attach tab”, ALWAYS use profile="chrome" (do not ask which profile).',
'When a node-hosted browser proxy is available, the tool may auto-route to it. Pin a node with node=<id|name> or target="node".',
"Chrome extension relay needs an attached tab: user must click the Clawdbot Browser Relay toolbar icon on the tab (badge ON). If no tab is connected, ask them to attach it.",
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc).",
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.',
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
`target selects browser location (sandbox|host|custom). Default: ${targetDefault}.`,
`target selects browser location (sandbox|host|custom|node). Default: ${targetDefault}.`,
"controlUrl implies target=custom (remote control server).",
hostHint,
allowlistHint,
@@ -142,49 +305,184 @@ export function createBrowserTool(opts?: {
const action = readStringParam(params, "action", { required: true });
const controlUrl = readStringParam(params, "controlUrl");
const profile = readStringParam(params, "profile");
let target = readStringParam(params, "target") as "sandbox" | "host" | "custom" | undefined;
if (profile === "chrome" && !target && !controlUrl?.trim()) {
// Chrome extension relay takeover is a host Chrome feature; default to host even in sandboxed sessions.
const requestedNode = readStringParam(params, "node");
let target = readStringParam(params, "target") as
| "sandbox"
| "host"
| "custom"
| "node"
| undefined;
if (controlUrl?.trim() && (target === "node" || requestedNode)) {
throw new Error('controlUrl is not supported with target="node".');
}
if (target === "custom" && requestedNode) {
throw new Error('node is not supported with target="custom".');
}
if (!target && !controlUrl?.trim() && !requestedNode && profile === "chrome") {
// Chrome extension relay takeover is a host Chrome feature; prefer host unless explicitly targeting a node.
target = "host";
}
const baseUrl = resolveBrowserBaseUrl({
const nodeTarget = await resolveBrowserNodeTarget({
requestedNode: requestedNode ?? undefined,
target,
controlUrl,
defaultControlUrl: opts?.defaultControlUrl,
allowHostControl: opts?.allowHostControl,
allowedControlUrls: opts?.allowedControlUrls,
allowedControlHosts: opts?.allowedControlHosts,
allowedControlPorts: opts?.allowedControlPorts,
});
const resolvedTarget = target === "node" ? undefined : target;
const baseUrl = nodeTarget
? ""
: resolveBrowserBaseUrl({
target: resolvedTarget,
controlUrl,
defaultControlUrl: opts?.defaultControlUrl,
allowHostControl: opts?.allowHostControl,
allowedControlUrls: opts?.allowedControlUrls,
allowedControlHosts: opts?.allowedControlHosts,
allowedControlPorts: opts?.allowedControlPorts,
});
const proxyRequest = nodeTarget
? async (opts: {
method: string;
path: string;
query?: Record<string, string | number | boolean | undefined>;
body?: unknown;
timeoutMs?: number;
profile?: string;
}) => {
const proxy = await callBrowserProxy({
nodeId: nodeTarget.nodeId,
method: opts.method,
path: opts.path,
query: opts.query,
body: opts.body,
timeoutMs: opts.timeoutMs,
profile: opts.profile,
});
const mapping = await persistProxyFiles(proxy.files);
applyProxyPaths(proxy.result, mapping);
return proxy.result;
}
: null;
switch (action) {
case "status":
if (proxyRequest) {
return jsonResult(
await proxyRequest({
method: "GET",
path: "/",
profile,
}),
);
}
return jsonResult(await browserStatus(baseUrl, { profile }));
case "start":
if (proxyRequest) {
await proxyRequest({
method: "POST",
path: "/start",
profile,
});
return jsonResult(
await proxyRequest({
method: "GET",
path: "/",
profile,
}),
);
}
await browserStart(baseUrl, { profile });
return jsonResult(await browserStatus(baseUrl, { profile }));
case "stop":
if (proxyRequest) {
await proxyRequest({
method: "POST",
path: "/stop",
profile,
});
return jsonResult(
await proxyRequest({
method: "GET",
path: "/",
profile,
}),
);
}
await browserStop(baseUrl, { profile });
return jsonResult(await browserStatus(baseUrl, { profile }));
case "profiles":
if (proxyRequest) {
const result = await proxyRequest({
method: "GET",
path: "/profiles",
});
return jsonResult(result);
}
return jsonResult({ profiles: await browserProfiles(baseUrl) });
case "tabs":
if (proxyRequest) {
const result = await proxyRequest({
method: "GET",
path: "/tabs",
profile,
});
const tabs = (result as { tabs?: unknown[] }).tabs ?? [];
return jsonResult({ tabs });
}
return jsonResult({ tabs: await browserTabs(baseUrl, { profile }) });
case "open": {
const targetUrl = readStringParam(params, "targetUrl", {
required: true,
});
if (proxyRequest) {
const result = await proxyRequest({
method: "POST",
path: "/tabs/open",
profile,
body: { url: targetUrl },
});
return jsonResult(result);
}
return jsonResult(await browserOpenTab(baseUrl, targetUrl, { profile }));
}
case "focus": {
const targetId = readStringParam(params, "targetId", {
required: true,
});
if (proxyRequest) {
const result = await proxyRequest({
method: "POST",
path: "/tabs/focus",
profile,
body: { targetId },
});
return jsonResult(result);
}
await browserFocusTab(baseUrl, targetId, { profile });
return jsonResult({ ok: true });
}
case "close": {
const targetId = readStringParam(params, "targetId");
if (proxyRequest) {
const result = targetId
? await proxyRequest({
method: "DELETE",
path: `/tabs/${encodeURIComponent(targetId)}`,
profile,
})
: await proxyRequest({
method: "POST",
path: "/act",
profile,
body: { kind: "close" },
});
return jsonResult(result);
}
if (targetId) await browserCloseTab(baseUrl, targetId, { profile });
else await browserAct(baseUrl, { kind: "close" }, { profile });
return jsonResult({ ok: true });
@@ -232,21 +530,41 @@ export function createBrowserTool(opts?: {
: undefined;
const selector = typeof params.selector === "string" ? params.selector.trim() : undefined;
const frame = typeof params.frame === "string" ? params.frame.trim() : undefined;
const snapshot = await browserSnapshot(baseUrl, {
format,
targetId,
limit,
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
refs,
interactive,
compact,
depth,
selector,
frame,
labels,
mode,
profile,
});
const snapshot = proxyRequest
? ((await proxyRequest({
method: "GET",
path: "/snapshot",
profile,
query: {
format,
targetId,
limit,
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
refs,
interactive,
compact,
depth,
selector,
frame,
labels,
mode,
},
})) as Awaited<ReturnType<typeof browserSnapshot>>)
: await browserSnapshot(baseUrl, {
format,
targetId,
limit,
...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
refs,
interactive,
compact,
depth,
selector,
frame,
labels,
mode,
profile,
});
if (snapshot.format === "ai") {
if (labels && snapshot.imagePath) {
return await imageResultFromFile({
@@ -269,14 +587,27 @@ export function createBrowserTool(opts?: {
const ref = readStringParam(params, "ref");
const element = readStringParam(params, "element");
const type = params.type === "jpeg" ? "jpeg" : "png";
const result = await browserScreenshotAction(baseUrl, {
targetId,
fullPage,
ref,
element,
type,
profile,
});
const result = proxyRequest
? ((await proxyRequest({
method: "POST",
path: "/screenshot",
profile,
body: {
targetId,
fullPage,
ref,
element,
type,
},
})) as Awaited<ReturnType<typeof browserScreenshotAction>>)
: await browserScreenshotAction(baseUrl, {
targetId,
fullPage,
ref,
element,
type,
profile,
});
return await imageResultFromFile({
label: "browser:screenshot",
path: result.path,
@@ -288,6 +619,18 @@ export function createBrowserTool(opts?: {
required: true,
});
const targetId = readStringParam(params, "targetId");
if (proxyRequest) {
const result = await proxyRequest({
method: "POST",
path: "/navigate",
profile,
body: {
url: targetUrl,
targetId,
},
});
return jsonResult(result);
}
return jsonResult(
await browserNavigate(baseUrl, {
url: targetUrl,
@@ -299,11 +642,30 @@ export function createBrowserTool(opts?: {
case "console": {
const level = typeof params.level === "string" ? params.level.trim() : undefined;
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
if (proxyRequest) {
const result = await proxyRequest({
method: "GET",
path: "/console",
profile,
query: {
level,
targetId,
},
});
return jsonResult(result);
}
return jsonResult(await browserConsoleMessages(baseUrl, { level, targetId, profile }));
}
case "pdf": {
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
const result = await browserPdfSave(baseUrl, { targetId, profile });
const result = proxyRequest
? ((await proxyRequest({
method: "POST",
path: "/pdf",
profile,
body: { targetId },
})) as Awaited<ReturnType<typeof browserPdfSave>>)
: await browserPdfSave(baseUrl, { targetId, profile });
return {
content: [{ type: "text", text: `FILE:${result.path}` }],
details: result,
@@ -320,6 +682,22 @@ export function createBrowserTool(opts?: {
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
? params.timeoutMs
: undefined;
if (proxyRequest) {
const result = await proxyRequest({
method: "POST",
path: "/hooks/file-chooser",
profile,
body: {
paths,
ref,
inputRef,
element,
targetId,
timeoutMs,
},
});
return jsonResult(result);
}
return jsonResult(
await browserArmFileChooser(baseUrl, {
paths,
@@ -340,6 +718,20 @@ export function createBrowserTool(opts?: {
typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)
? params.timeoutMs
: undefined;
if (proxyRequest) {
const result = await proxyRequest({
method: "POST",
path: "/hooks/dialog",
profile,
body: {
accept,
promptText,
targetId,
timeoutMs,
},
});
return jsonResult(result);
}
return jsonResult(
await browserArmDialog(baseUrl, {
accept,
@@ -356,14 +748,29 @@ export function createBrowserTool(opts?: {
throw new Error("request required");
}
try {
const result = await browserAct(baseUrl, request as Parameters<typeof browserAct>[1], {
profile,
});
const result = proxyRequest
? await proxyRequest({
method: "POST",
path: "/act",
profile,
body: request,
})
: await browserAct(baseUrl, request as Parameters<typeof browserAct>[1], {
profile,
});
return jsonResult(result);
} catch (err) {
const msg = String(err);
if (msg.includes("404:") && msg.includes("tab not found") && profile === "chrome") {
const tabs = await browserTabs(baseUrl, { profile }).catch(() => []);
const tabs = proxyRequest
? ((
(await proxyRequest({
method: "GET",
path: "/tabs",
profile,
})) as { tabs?: unknown[] }
).tabs ?? [])
: await browserTabs(baseUrl, { profile }).catch(() => []);
if (!tabs.length) {
throw new Error(
"No Chrome tabs are attached via the Clawdbot Browser Relay extension. Click the toolbar icon on the tab you want to control (badge ON), then retry.",