let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: route browser control via gateway/node

Browse Source
This commit is contained in:
Peter Steinberger
2026-01-27 03:23:42 +00:00
parent b151b8d196
commit e7fdccce39
91 changed files with 1909 additions and 1608 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/agents/clawdbot-tools.ts
View File

@@ -20,11 +20,8 @@ import { createWebFetchTool, createWebSearchTool } from "./tools/web-tools.js";
import { createTtsTool } from "./tools/tts-tool.js";
export function createClawdbotTools(options?: {
browserControlUrl?: string;
sandboxBrowserBridgeUrl?: string;
allowHostBrowserControl?: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
agentSessionKey?: string;
agentChannel?: GatewayMessageChannel;
agentAccountId?: string;
@@ -75,11 +72,8 @@ export function createClawdbotTools(options?: {
});
const tools: AnyAgentTool[] = [
createBrowserTool({
defaultControlUrl: options?.browserControlUrl,
sandboxBridgeUrl: options?.sandboxBrowserBridgeUrl,
allowHostControl: options?.allowHostBrowserControl,
allowedControlUrls: options?.allowedControlUrls,
allowedControlHosts: options?.allowedControlHosts,
allowedControlPorts: options?.allowedControlPorts,
}),
createCanvasTool(),
createNodesTool({

4
src/agents/pi-embedded-runner.buildembeddedsandboxinfo.test.ts
View File

@@ -127,7 +127,7 @@ describe("buildEmbeddedSandboxInfo", () => {
},
browserAllowHostControl: true,
browser: {
controlUrl: "http://localhost:9222",
bridgeUrl: "http://localhost:9222",
noVncUrl: "http://localhost:6080",
containerName: "clawdbot-sbx-browser-test",
},
@@ -138,7 +138,7 @@ describe("buildEmbeddedSandboxInfo", () => {
workspaceDir: "/tmp/clawdbot-sandbox",
workspaceAccess: "none",
agentWorkspaceMount: undefined,
browserControlUrl: "http://localhost:9222",
browserBridgeUrl: "http://localhost:9222",
browserNoVncUrl: "http://localhost:6080",
hostBrowserAllowed: true,
});

5
src/agents/pi-embedded-runner/sandbox-info.ts
View File

@@ -13,12 +13,9 @@ export function buildEmbeddedSandboxInfo(
workspaceDir: sandbox.workspaceDir,
workspaceAccess: sandbox.workspaceAccess,
agentWorkspaceMount: sandbox.workspaceAccess === "ro" ? "/agent" : undefined,
browserControlUrl: sandbox.browser?.controlUrl,
browserBridgeUrl: sandbox.browser?.bridgeUrl,
browserNoVncUrl: sandbox.browser?.noVncUrl,
hostBrowserAllowed: sandbox.browserAllowHostControl,
allowedControlUrls: sandbox.browserAllowedControlUrls,
allowedControlHosts: sandbox.browserAllowedControlHosts,
allowedControlPorts: sandbox.browserAllowedControlPorts,
...(elevatedAllowed
? {
elevated: {

5
src/agents/pi-embedded-runner/types.ts
View File

@@ -69,12 +69,9 @@ export type EmbeddedSandboxInfo = {
workspaceDir?: string;
workspaceAccess?: "none" | "ro" | "rw";
agentWorkspaceMount?: string;
browserControlUrl?: string;
browserBridgeUrl?: string;
browserNoVncUrl?: string;
hostBrowserAllowed?: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
elevated?: {
allowed: boolean;
defaultLevel: "on" | "off" | "ask" | "full";

1
src/agents/pi-tools.create-clawdbot-coding-tools.adds-claude-style-aliases-schemas-without-dropping.test.ts
View File

@@ -96,7 +96,6 @@ describe("createClawdbotCodingTools", () => {
};
expect(parameters.properties?.action).toBeDefined();
expect(parameters.properties?.target).toBeDefined();
expect(parameters.properties?.controlUrl).toBeDefined();
expect(parameters.properties?.targetUrl).toBeDefined();
expect(parameters.properties?.request).toBeDefined();
expect(parameters.required ?? []).toContain("action");

5
src/agents/pi-tools.ts
View File

@@ -294,11 +294,8 @@ export function createClawdbotCodingTools(options?: {
// Channel docking: include channel-defined agent tools (login, etc.).
...listChannelAgentTools({ cfg: options?.config }),
...createClawdbotTools({
browserControlUrl: sandbox?.browser?.controlUrl,
sandboxBrowserBridgeUrl: sandbox?.browser?.bridgeUrl,
allowHostBrowserControl: sandbox ? sandbox.browserAllowHostControl : true,
allowedControlUrls: sandbox?.browserAllowedControlUrls,
allowedControlHosts: sandbox?.browserAllowedControlHosts,
allowedControlPorts: sandbox?.browserAllowedControlPorts,
agentSessionKey: options?.sessionKey,
agentChannel: resolveGatewayMessageChannel(options?.messageProvider),
agentAccountId: options?.agentAccountId,

6
src/agents/sandbox/browser.ts
View File

@@ -40,13 +40,9 @@ function buildSandboxBrowserResolvedConfig(params: {
cdpPort: number;
headless: boolean;
}): ResolvedBrowserConfig {
const controlHost = "127.0.0.1";
const controlUrl = `http://${controlHost}:${params.controlPort}`;
const cdpHost = "127.0.0.1";
return {
enabled: true,
controlUrl,
controlHost,
controlPort: params.controlPort,
cdpProtocol: "http",
cdpHost,
@@ -204,7 +200,7 @@ export async function ensureSandboxBrowser(params: {
: undefined;
return {
controlUrl: resolvedBridge.baseUrl,
bridgeUrl: resolvedBridge.baseUrl,
noVncUrl,
containerName,
};

17
src/agents/sandbox/config.ts
View File

@@ -86,11 +86,6 @@ export function resolveSandboxBrowserConfig(params: {
}): SandboxBrowserConfig {
const agentBrowser = params.scope === "shared" ? undefined : params.agentBrowser;
const globalBrowser = params.globalBrowser;
const allowedControlUrls = agentBrowser?.allowedControlUrls ?? globalBrowser?.allowedControlUrls;
const allowedControlHosts =
agentBrowser?.allowedControlHosts ?? globalBrowser?.allowedControlHosts;
const allowedControlPorts =
agentBrowser?.allowedControlPorts ?? globalBrowser?.allowedControlPorts;
return {
enabled: agentBrowser?.enabled ?? globalBrowser?.enabled ?? false,
image: agentBrowser?.image ?? globalBrowser?.image ?? DEFAULT_SANDBOX_BROWSER_IMAGE,
@@ -105,18 +100,6 @@ export function resolveSandboxBrowserConfig(params: {
headless: agentBrowser?.headless ?? globalBrowser?.headless ?? false,
enableNoVnc: agentBrowser?.enableNoVnc ?? globalBrowser?.enableNoVnc ?? true,
allowHostControl: agentBrowser?.allowHostControl ?? globalBrowser?.allowHostControl ?? false,
allowedControlUrls:
Array.isArray(allowedControlUrls) && allowedControlUrls.length > 0
? allowedControlUrls
: undefined,
allowedControlHosts:
Array.isArray(allowedControlHosts) && allowedControlHosts.length > 0
? allowedControlHosts
: undefined,
allowedControlPorts:
Array.isArray(allowedControlPorts) && allowedControlPorts.length > 0
? allowedControlPorts
: undefined,
autoStart: agentBrowser?.autoStart ?? globalBrowser?.autoStart ?? true,
autoStartTimeoutMs:
agentBrowser?.autoStartTimeoutMs ??

3
src/agents/sandbox/context.ts
View File

@@ -87,9 +87,6 @@ export async function resolveSandboxContext(params: {
docker: cfg.docker,
tools: cfg.tools,
browserAllowHostControl: cfg.browser.allowHostControl,
browserAllowedControlUrls: cfg.browser.allowedControlUrls,
browserAllowedControlHosts: cfg.browser.allowedControlHosts,
browserAllowedControlPorts: cfg.browser.allowedControlPorts,
browser: browser ?? undefined,
};
}

8
src/agents/sandbox/types.ts
View File

@@ -37,9 +37,6 @@ export type SandboxBrowserConfig = {
headless: boolean;
enableNoVnc: boolean;
allowHostControl: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
autoStart: boolean;
autoStartTimeoutMs: number;
};
@@ -63,7 +60,7 @@ export type SandboxConfig = {
};
export type SandboxBrowserContext = {
controlUrl: string;
bridgeUrl: string;
noVncUrl?: string;
containerName: string;
};
@@ -79,9 +76,6 @@ export type SandboxContext = {
docker: SandboxDockerConfig;
tools: SandboxToolPolicy;
browserAllowHostControl: boolean;
browserAllowedControlUrls?: string[];
browserAllowedControlHosts?: string[];
browserAllowedControlPorts?: number[];
browser?: SandboxBrowserContext;
};

18
src/agents/system-prompt.ts
View File

@@ -165,12 +165,9 @@ export function buildAgentSystemPrompt(params: {
workspaceDir?: string;
workspaceAccess?: "none" | "ro" | "rw";
agentWorkspaceMount?: string;
browserControlUrl?: string;
browserBridgeUrl?: string;
browserNoVncUrl?: string;
hostBrowserAllowed?: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
elevated?: {
allowed: boolean;
defaultLevel: "on" | "off" | "ask" | "full";
@@ -419,9 +416,7 @@ export function buildAgentSystemPrompt(params: {
: ""
}`
: "",
params.sandboxInfo.browserControlUrl
? `Sandbox browser control URL: ${params.sandboxInfo.browserControlUrl}`
: "",
params.sandboxInfo.browserBridgeUrl ? "Sandbox browser: enabled." : "",
params.sandboxInfo.browserNoVncUrl
? `Sandbox browser observer (noVNC): ${params.sandboxInfo.browserNoVncUrl}`
: "",
@@ -430,15 +425,6 @@ export function buildAgentSystemPrompt(params: {
: params.sandboxInfo.hostBrowserAllowed === false
? "Host browser control: blocked."
: "",
params.sandboxInfo.allowedControlUrls?.length
? `Browser control URL allowlist: ${params.sandboxInfo.allowedControlUrls.join(", ")}`
: "",
params.sandboxInfo.allowedControlHosts?.length
? `Browser control host allowlist: ${params.sandboxInfo.allowedControlHosts.join(", ")}`
: "",
params.sandboxInfo.allowedControlPorts?.length
? `Browser control port allowlist: ${params.sandboxInfo.allowedControlPorts.join(", ")}`
: "",
params.sandboxInfo.elevated?.allowed
? "Elevated exec is available for this session."
: "",

3
src/agents/tools/browser-tool.schema.ts
View File

@@ -35,7 +35,7 @@ const BROWSER_TOOL_ACTIONS = [
"act",
] as const;
const BROWSER_TARGETS = ["sandbox", "host", "custom", "node"] as const;
const BROWSER_TARGETS = ["sandbox", "host", "node"] as const;
const BROWSER_SNAPSHOT_FORMATS = ["aria", "ai"] as const;
const BROWSER_SNAPSHOT_MODES = ["efficient"] as const;
@@ -86,7 +86,6 @@ export const BrowserToolSchema = Type.Object({
target: optionalStringEnum(BROWSER_TARGETS),
node: Type.Optional(Type.String()),
profile: Type.Optional(Type.String()),
controlUrl: Type.Optional(Type.String()),
targetUrl: Type.Optional(Type.String()),
targetId: Type.Optional(Type.String()),
limit: Type.Optional(Type.Number()),

36
src/agents/tools/browser-tool.test.ts
View File

@@ -28,23 +28,7 @@ vi.mock("../../browser/client.js", () => browserClientMocks);
const browserConfigMocks = vi.hoisted(() => ({
resolveBrowserConfig: vi.fn(() => ({
enabled: true,
controlUrl: "http://127.0.0.1:18791",
controlHost: "127.0.0.1",
controlPort: 18791,
cdpProtocol: "http",
cdpHost: "127.0.0.1",
cdpIsLoopback: true,
color: "#FF0000",
headless: true,
noSandbox: false,
attachOnly: false,
defaultProfile: "clawd",
profiles: {
clawd: {
cdpPort: 18792,
color: "#FF0000",
},
},
})),
}));
vi.mock("../../browser/config.js", () => browserConfigMocks);
@@ -99,7 +83,7 @@ describe("browser tool snapshot maxChars", () => {
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({
format: "ai",
maxChars: DEFAULT_AI_SNAPSHOT_MAX_CHARS,
@@ -117,7 +101,7 @@ describe("browser tool snapshot maxChars", () => {
});
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({
maxChars: override,
}),
@@ -141,7 +125,7 @@ describe("browser tool snapshot maxChars", () => {
const tool = createBrowserTool();
await tool.execute?.(null, { action: "profiles" });
expect(browserClientMocks.browserProfiles).toHaveBeenCalledWith("http://127.0.0.1:18791");
expect(browserClientMocks.browserProfiles).toHaveBeenCalledWith(undefined);
});
it("passes refs mode through to browser snapshot", async () => {
@@ -149,7 +133,7 @@ describe("browser tool snapshot maxChars", () => {
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai", refs: "aria" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({
format: "ai",
refs: "aria",
@@ -165,7 +149,7 @@ describe("browser tool snapshot maxChars", () => {
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({
mode: "efficient",
}),
@@ -185,11 +169,11 @@ describe("browser tool snapshot maxChars", () => {
});
it("defaults to host when using profile=chrome (even in sandboxed sessions)", async () => {
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
const tool = createBrowserTool({ sandboxBridgeUrl: "http://127.0.0.1:9999" });
await tool.execute?.(null, { action: "snapshot", profile: "chrome", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({
profile: "chrome",
}),
@@ -220,7 +204,7 @@ describe("browser tool snapshot maxChars", () => {
expect(browserClientMocks.browserStatus).not.toHaveBeenCalled();
});
it("keeps sandbox control url when node proxy is available", async () => {
it("keeps sandbox bridge url when node proxy is available", async () => {
nodesUtilsMocks.listNodes.mockResolvedValue([
{
nodeId: "node-1",
@@ -230,7 +214,7 @@ describe("browser tool snapshot maxChars", () => {
commands: ["browser.proxy"],
},
]);
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
const tool = createBrowserTool({ sandboxBridgeUrl: "http://127.0.0.1:9999" });
await tool.execute?.(null, { action: "status" });
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
@@ -254,7 +238,7 @@ describe("browser tool snapshot maxChars", () => {
await tool.execute?.(null, { action: "status", profile: "chrome" });
expect(browserClientMocks.browserStatus).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
undefined,
expect.objectContaining({ profile: "chrome" }),
);
expect(gatewayMocks.callGatewayTool).not.toHaveBeenCalled();

121
src/agents/tools/browser-tool.ts
View File

@@ -55,9 +55,8 @@ function isBrowserNode(node: NodeListNode) {
async function resolveBrowserNodeTarget(params: {
requestedNode?: string;
target?: "sandbox" | "host" | "custom" | "node";
controlUrl?: string;
defaultControlUrl?: string;
target?: "sandbox" | "host" | "node";
sandboxBridgeUrl?: string;
}): Promise<BrowserNodeTarget | null> {
const cfg = loadConfig();
const policy = cfg.gateway?.nodes?.browser;
@@ -68,10 +67,9 @@ async function resolveBrowserNodeTarget(params: {
}
return null;
}
if (params.defaultControlUrl?.trim() && params.target !== "node" && !params.requestedNode) {
if (params.sandboxBridgeUrl?.trim() && params.target !== "node" && !params.requestedNode) {
return null;
}
if (params.controlUrl?.trim()) return null;
if (params.target && params.target !== "node") return null;
if (mode === "manual" && params.target !== "node" && !params.requestedNode) {
return null;
@@ -187,70 +185,22 @@ function applyProxyPaths(result: unknown, mapping: Map<string, string>) {
}
function resolveBrowserBaseUrl(params: {
target?: "sandbox" | "host" | "custom";
controlUrl?: string;
defaultControlUrl?: string;
target?: "sandbox" | "host";
sandboxBridgeUrl?: string;
allowHostControl?: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
}) {
}): string | undefined {
const cfg = loadConfig();
const resolved = resolveBrowserConfig(cfg.browser);
const normalizedControlUrl = params.controlUrl?.trim() ?? "";
const normalizedDefault = params.defaultControlUrl?.trim() ?? "";
const target =
params.target ?? (normalizedControlUrl ? "custom" : normalizedDefault ? "sandbox" : "host");
const assertAllowedControlUrl = (url: string) => {
const allowedUrls = params.allowedControlUrls?.map((entry) => entry.trim().replace(/\/$/, ""));
const allowedHosts = params.allowedControlHosts?.map((entry) => entry.trim().toLowerCase());
const allowedPorts = params.allowedControlPorts;
if (!allowedUrls?.length && !allowedHosts?.length && !allowedPorts?.length) {
return;
}
let parsed: URL;
try {
parsed = new URL(url);
} catch {
throw new Error(`Invalid browser controlUrl: ${url}`);
}
const normalizedUrl = parsed.toString().replace(/\/$/, "");
if (allowedUrls?.length && !allowedUrls.includes(normalizedUrl)) {
throw new Error("Browser controlUrl is not in the allowed URL list.");
}
if (allowedHosts?.length && !allowedHosts.includes(parsed.hostname)) {
throw new Error("Browser controlUrl hostname is not in the allowed host list.");
}
if (allowedPorts?.length) {
const port =
parsed.port?.trim() !== "" ? Number(parsed.port) : parsed.protocol === "https:" ? 443 : 80;
if (!Number.isFinite(port) || !allowedPorts.includes(port)) {
throw new Error("Browser controlUrl port is not in the allowed port list.");
}
}
};
if (target !== "custom" && params.target && normalizedControlUrl) {
throw new Error('controlUrl is only supported with target="custom".');
}
if (target === "custom") {
if (!normalizedControlUrl) {
throw new Error("Custom browser target requires controlUrl.");
}
const normalized = normalizedControlUrl.replace(/\/$/, "");
assertAllowedControlUrl(normalized);
return normalized;
}
const resolved = resolveBrowserConfig(cfg.browser, cfg);
const normalizedSandbox = params.sandboxBridgeUrl?.trim() ?? "";
const target = params.target ?? (normalizedSandbox ? "sandbox" : "host");
if (target === "sandbox") {
if (!normalizedDefault) {
if (!normalizedSandbox) {
throw new Error(
'Sandbox browser is unavailable. Enable agents.defaults.sandbox.browser.enabled or use target="host" if allowed.',
);
}
return normalizedDefault.replace(/\/$/, "");
return normalizedSandbox.replace(/\/$/, "");
}
if (params.allowHostControl === false) {
@@ -261,27 +211,16 @@ function resolveBrowserBaseUrl(params: {
"Browser control is disabled. Set browser.enabled=true in ~/.clawdbot/clawdbot.json.",
);
}
const normalized = resolved.controlUrl.replace(/\/$/, "");
assertAllowedControlUrl(normalized);
return normalized;
return undefined;
}
export function createBrowserTool(opts?: {
defaultControlUrl?: string;
sandboxBridgeUrl?: string;
allowHostControl?: boolean;
allowedControlUrls?: string[];
allowedControlHosts?: string[];
allowedControlPorts?: number[];
}): AnyAgentTool {
const targetDefault = opts?.defaultControlUrl ? "sandbox" : "host";
const targetDefault = opts?.sandboxBridgeUrl ? "sandbox" : "host";
const hostHint =
opts?.allowHostControl === false ? "Host target blocked by policy." : "Host target allowed.";
const allowlistHint =
opts?.allowedControlUrls?.length ||
opts?.allowedControlHosts?.length ||
opts?.allowedControlPorts?.length
? "Custom targets are restricted by sandbox allowlists."
: "Custom targets are unrestricted.";
return {
label: "Browser",
name: "browser",
@@ -294,33 +233,22 @@ export function createBrowserTool(opts?: {
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc).",
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.',
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
`target selects browser location (sandbox|host|custom|node). Default: ${targetDefault}.`,
"controlUrl implies target=custom (remote control server).",
`target selects browser location (sandbox|host|node). Default: ${targetDefault}.`,
hostHint,
allowlistHint,
].join(" "),
parameters: BrowserToolSchema,
execute: async (_toolCallId, args) => {
const params = args as Record<string, unknown>;
const action = readStringParam(params, "action", { required: true });
const controlUrl = readStringParam(params, "controlUrl");
const profile = readStringParam(params, "profile");
const requestedNode = readStringParam(params, "node");
let target = readStringParam(params, "target") as
| "sandbox"
| "host"
| "custom"
| "node"
| undefined;
let target = readStringParam(params, "target") as "sandbox" | "host" | "node" | undefined;
if (controlUrl?.trim() && (target === "node" || requestedNode)) {
throw new Error('controlUrl is not supported with target="node".');
}
if (target === "custom" && requestedNode) {
throw new Error('node is not supported with target="custom".');
if (requestedNode && target && target !== "node") {
throw new Error('node is only supported with target="node".');
}
if (!target && !controlUrl?.trim() && !requestedNode && profile === "chrome") {
if (!target && !requestedNode && profile === "chrome") {
// Chrome extension relay takeover is a host Chrome feature; prefer host unless explicitly targeting a node.
target = "host";
}
@@ -328,21 +256,16 @@ export function createBrowserTool(opts?: {
const nodeTarget = await resolveBrowserNodeTarget({
requestedNode: requestedNode ?? undefined,
target,
controlUrl,
defaultControlUrl: opts?.defaultControlUrl,
sandboxBridgeUrl: opts?.sandboxBridgeUrl,
});
const resolvedTarget = target === "node" ? undefined : target;
const baseUrl = nodeTarget
? ""
? undefined
: resolveBrowserBaseUrl({
target: resolvedTarget,
controlUrl,
defaultControlUrl: opts?.defaultControlUrl,
sandboxBridgeUrl: opts?.sandboxBridgeUrl,
allowHostControl: opts?.allowHostControl,
allowedControlUrls: opts?.allowedControlUrls,
allowedControlHosts: opts?.allowedControlHosts,
allowedControlPorts: opts?.allowedControlPorts,
});
const proxyRequest = nodeTarget