fix: add browser snapshot default mode (#1336)

Co-authored-by: Seb Slight <sbarrios93@gmail.com>
This commit is contained in:
Peter Steinberger
2026-01-21 03:02:15 +00:00
parent 14d3d72bcc
commit a0cd295c0f
10 changed files with 149 additions and 9 deletions

View File

@@ -9,6 +9,7 @@ Docs: https://docs.clawd.bot
- Repo: remove the Peekaboo git submodule now that the SPM release is used. - Repo: remove the Peekaboo git submodule now that the SPM release is used.
- Update: sync plugin sources on channel switches and update npm-installed plugins during `clawdbot update`. - Update: sync plugin sources on channel switches and update npm-installed plugins during `clawdbot update`.
- Plugins: share npm plugin update logic between `clawdbot update` and `clawdbot plugins update`. - Plugins: share npm plugin update logic between `clawdbot update` and `clawdbot plugins update`.
- Browser: allow config defaults for efficient snapshots in the tool/CLI. (#1336) — thanks @sebslight.
- Channels: add the Nostr plugin channel with profile management + onboarding install defaults. (#1323) — thanks @joelklabo. - Channels: add the Nostr plugin channel with profile management + onboarding install defaults. (#1323) — thanks @joelklabo.
- Plugins: require manifest-embedded config schemas, validate configs without loading plugin code, and surface plugin config warnings. (#1272) — thanks @thewilloftheshadow. - Plugins: require manifest-embedded config schemas, validate configs without loading plugin code, and surface plugin config warnings. (#1272) — thanks @thewilloftheshadow.
- Plugins: move channel catalog metadata into plugin manifests; align Nextcloud Talk policy helpers with core patterns. (#1290) — thanks @NicholaiVogel. - Plugins: move channel catalog metadata into plugin manifests; align Nextcloud Talk policy helpers with core patterns. (#1290) — thanks @NicholaiVogel.

View File

@@ -2615,10 +2615,13 @@ Defaults:
// noSandbox: false, // noSandbox: false,
// executablePath: "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", // executablePath: "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
// attachOnly: false, // set true when tunneling a remote CDP to localhost // attachOnly: false, // set true when tunneling a remote CDP to localhost
// snapshotDefaults: { mode: "efficient" }, // tool/CLI default snapshot preset
} }
} }
``` ```
Note: `browser.snapshotDefaults` only affects Clawdbot's browser tool + CLI. Direct HTTP clients must pass `mode` explicitly.
### `ui` (Appearance) ### `ui` (Appearance)
Optional accent color used by the native apps for UI chrome (e.g. Talk Mode bubble tint). Optional accent color used by the native apps for UI chrome (e.g. Talk Mode bubble tint).

View File

@@ -500,6 +500,7 @@ Notes:
- `--format ai` (default when Playwright is installed): returns an AI snapshot with numeric refs (`aria-ref="<n>"`). - `--format ai` (default when Playwright is installed): returns an AI snapshot with numeric refs (`aria-ref="<n>"`).
- `--format aria`: returns the accessibility tree (no refs; inspection only). - `--format aria`: returns the accessibility tree (no refs; inspection only).
- `--efficient` (or `--mode efficient`): compact role snapshot preset (interactive + compact + depth + lower maxChars). - `--efficient` (or `--mode efficient`): compact role snapshot preset (interactive + compact + depth + lower maxChars).
- Config default (tool/CLI only): set `browser.snapshotDefaults.mode: "efficient"` to use efficient snapshots when the caller does not pass a mode (see [Gateway configuration](/gateway/configuration#browser-clawd-managed-browser)).
- Role snapshot options (`--interactive`, `--compact`, `--depth`, `--selector`) force a role-based snapshot with refs like `ref=e12`. - Role snapshot options (`--interactive`, `--compact`, `--depth`, `--selector`) force a role-based snapshot with refs like `ref=e12`.
- `--frame "<iframe selector>"` scopes role snapshots to an iframe (pairs with role refs like `e12`). - `--frame "<iframe selector>"` scopes role snapshots to an iframe (pairs with role refs like `e12`).
- `--interactive` outputs a flat, easy-to-pick list of interactive elements (best for driving actions). - `--interactive` outputs a flat, easy-to-pick list of interactive elements (best for driving actions).

View File

@@ -49,9 +49,10 @@ const browserConfigMocks = vi.hoisted(() => ({
})); }));
vi.mock("../../browser/config.js", () => browserConfigMocks); vi.mock("../../browser/config.js", () => browserConfigMocks);
vi.mock("../../config/config.js", () => ({ const configMocks = vi.hoisted(() => ({
loadConfig: vi.fn(() => ({ browser: {} })), loadConfig: vi.fn(() => ({ browser: {} })),
})); }));
vi.mock("../../config/config.js", () => configMocks);
const toolCommonMocks = vi.hoisted(() => ({ const toolCommonMocks = vi.hoisted(() => ({
imageResultFromFile: vi.fn(), imageResultFromFile: vi.fn(),
@@ -70,11 +71,12 @@ import { createBrowserTool } from "./browser-tool.js";
describe("browser tool snapshot maxChars", () => { describe("browser tool snapshot maxChars", () => {
afterEach(() => { afterEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
configMocks.loadConfig.mockReturnValue({ browser: {} });
}); });
it("applies the default ai snapshot limit", async () => { it("applies the default ai snapshot limit", async () => {
const tool = createBrowserTool(); const tool = createBrowserTool();
await tool.execute?.(null, { action: "snapshot", format: "ai" }); await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith( expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791", "http://127.0.0.1:18791",
@@ -90,7 +92,7 @@ describe("browser tool snapshot maxChars", () => {
const override = 2_000; const override = 2_000;
await tool.execute?.(null, { await tool.execute?.(null, {
action: "snapshot", action: "snapshot",
format: "ai", snapshotFormat: "ai",
maxChars: override, maxChars: override,
}); });
@@ -106,7 +108,7 @@ describe("browser tool snapshot maxChars", () => {
const tool = createBrowserTool(); const tool = createBrowserTool();
await tool.execute?.(null, { await tool.execute?.(null, {
action: "snapshot", action: "snapshot",
format: "ai", snapshotFormat: "ai",
maxChars: 0, maxChars: 0,
}); });
@@ -124,7 +126,7 @@ describe("browser tool snapshot maxChars", () => {
it("passes refs mode through to browser snapshot", async () => { it("passes refs mode through to browser snapshot", async () => {
const tool = createBrowserTool(); const tool = createBrowserTool();
await tool.execute?.(null, { action: "snapshot", format: "ai", refs: "aria" }); await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai", refs: "aria" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith( expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791", "http://127.0.0.1:18791",
@@ -135,9 +137,36 @@ describe("browser tool snapshot maxChars", () => {
); );
}); });
it("uses config snapshot defaults when mode is not provided", async () => {
configMocks.loadConfig.mockReturnValue({
browser: { snapshotDefaults: { mode: "efficient" } },
});
const tool = createBrowserTool();
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
expect.objectContaining({
mode: "efficient",
}),
);
});
it("does not apply config snapshot defaults to aria snapshots", async () => {
configMocks.loadConfig.mockReturnValue({
browser: { snapshotDefaults: { mode: "efficient" } },
});
const tool = createBrowserTool();
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "aria" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalled();
const [, opts] = browserClientMocks.browserSnapshot.mock.calls.at(-1) ?? [];
expect(opts?.mode).toBeUndefined();
});
it("defaults to host when using profile=chrome (even in sandboxed sessions)", async () => { it("defaults to host when using profile=chrome (even in sandboxed sessions)", async () => {
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" }); const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
await tool.execute?.(null, { action: "snapshot", profile: "chrome", format: "ai" }); await tool.execute?.(null, { action: "snapshot", profile: "chrome", snapshotFormat: "ai" });
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith( expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791", "http://127.0.0.1:18791",
@@ -151,6 +180,7 @@ describe("browser tool snapshot maxChars", () => {
describe("browser tool snapshot labels", () => { describe("browser tool snapshot labels", () => {
afterEach(() => { afterEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
configMocks.loadConfig.mockReturnValue({ browser: {} });
}); });
it("returns image + text when labels are requested", async () => { it("returns image + text when labels are requested", async () => {
@@ -175,7 +205,7 @@ describe("browser tool snapshot labels", () => {
const result = await tool.execute?.(null, { const result = await tool.execute?.(null, {
action: "snapshot", action: "snapshot",
format: "ai", snapshotFormat: "ai",
labels: true, labels: true,
}); });

View File

@@ -190,11 +190,17 @@ export function createBrowserTool(opts?: {
return jsonResult({ ok: true }); return jsonResult({ ok: true });
} }
case "snapshot": { case "snapshot": {
const snapshotDefaults = loadConfig().browser?.snapshotDefaults;
const format = const format =
params.snapshotFormat === "ai" || params.snapshotFormat === "aria" params.snapshotFormat === "ai" || params.snapshotFormat === "aria"
? (params.snapshotFormat as "ai" | "aria") ? (params.snapshotFormat as "ai" | "aria")
: "ai"; : "ai";
const mode = params.mode === "efficient" ? "efficient" : undefined; const mode =
params.mode === "efficient"
? "efficient"
: format === "ai" && snapshotDefaults?.mode === "efficient"
? "efficient"
: undefined;
const labels = typeof params.labels === "boolean" ? params.labels : undefined; const labels = typeof params.labels === "boolean" ? params.labels : undefined;
const refs = params.refs === "aria" || params.refs === "role" ? params.refs : undefined; const refs = params.refs === "aria" || params.refs === "role" ? params.refs : undefined;
const hasMaxChars = Object.hasOwn(params, "maxChars"); const hasMaxChars = Object.hasOwn(params, "maxChars");

View File

@@ -0,0 +1,78 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { Command } from "commander";
const clientMocks = vi.hoisted(() => ({
browserSnapshot: vi.fn(async () => ({
ok: true,
format: "ai",
targetId: "t1",
url: "https://example.com",
snapshot: "ok",
})),
resolveBrowserControlUrl: vi.fn(() => "http://127.0.0.1:18791"),
}));
vi.mock("../browser/client.js", () => clientMocks);
const configMocks = vi.hoisted(() => ({
loadConfig: vi.fn(() => ({ browser: {} })),
}));
vi.mock("../config/config.js", () => configMocks);
const runtime = {
log: vi.fn(),
error: vi.fn(),
exit: vi.fn(),
};
vi.mock("../runtime.js", () => ({
defaultRuntime: runtime,
}));
describe("browser cli snapshot defaults", () => {
afterEach(() => {
vi.clearAllMocks();
configMocks.loadConfig.mockReturnValue({ browser: {} });
});
it("uses config snapshot defaults when mode is not provided", async () => {
configMocks.loadConfig.mockReturnValue({
browser: { snapshotDefaults: { mode: "efficient" } },
});
const { registerBrowserInspectCommands } = await import("./browser-cli-inspect.js");
const program = new Command();
const browser = program.command("browser").option("--json", false);
registerBrowserInspectCommands(browser, () => ({}));
await program.parseAsync(["browser", "snapshot"], { from: "user" });
expect(clientMocks.browserSnapshot).toHaveBeenCalledWith(
"http://127.0.0.1:18791",
expect.objectContaining({
format: "ai",
mode: "efficient",
}),
);
});
it("does not apply config snapshot defaults to aria snapshots", async () => {
configMocks.loadConfig.mockReturnValue({
browser: { snapshotDefaults: { mode: "efficient" } },
});
clientMocks.browserSnapshot.mockResolvedValueOnce({
ok: true,
format: "aria",
targetId: "t1",
url: "https://example.com",
nodes: [],
});
const { registerBrowserInspectCommands } = await import("./browser-cli-inspect.js");
const program = new Command();
const browser = program.command("browser").option("--json", false);
registerBrowserInspectCommands(browser, () => ({}));
await program.parseAsync(["browser", "snapshot", "--format", "aria"], { from: "user" });
expect(clientMocks.browserSnapshot).toHaveBeenCalled();
const [, opts] = clientMocks.browserSnapshot.mock.calls.at(-1) ?? [];
expect(opts?.mode).toBeUndefined();
});
});

View File

@@ -2,6 +2,7 @@ import type { Command } from "commander";
import { browserSnapshot, resolveBrowserControlUrl } from "../browser/client.js"; import { browserSnapshot, resolveBrowserControlUrl } from "../browser/client.js";
import { browserScreenshotAction } from "../browser/client-actions.js"; import { browserScreenshotAction } from "../browser/client-actions.js";
import { loadConfig } from "../config/config.js";
import { danger } from "../globals.js"; import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js"; import { defaultRuntime } from "../runtime.js";
import type { BrowserParentOpts } from "./browser-cli-shared.js"; import type { BrowserParentOpts } from "./browser-cli-shared.js";
@@ -62,7 +63,11 @@ export function registerBrowserInspectCommands(
const baseUrl = resolveBrowserControlUrl(parent?.url); const baseUrl = resolveBrowserControlUrl(parent?.url);
const profile = parent?.browserProfile; const profile = parent?.browserProfile;
const format = opts.format === "aria" ? "aria" : "ai"; const format = opts.format === "aria" ? "aria" : "ai";
const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : undefined; const configMode =
format === "ai" && loadConfig().browser?.snapshotDefaults?.mode === "efficient"
? "efficient"
: undefined;
const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : configMode;
try { try {
const result = await browserSnapshot(baseUrl, { const result = await browserSnapshot(baseUrl, {
format, format,

View File

@@ -250,6 +250,8 @@ const FIELD_LABELS: Record<string, string> = {
"commands.useAccessGroups": "Use Access Groups", "commands.useAccessGroups": "Use Access Groups",
"ui.seamColor": "Accent Color", "ui.seamColor": "Accent Color",
"browser.controlUrl": "Browser Control URL", "browser.controlUrl": "Browser Control URL",
"browser.snapshotDefaults": "Browser Snapshot Defaults",
"browser.snapshotDefaults.mode": "Browser Snapshot Mode",
"browser.remoteCdpTimeoutMs": "Remote CDP Timeout (ms)", "browser.remoteCdpTimeoutMs": "Remote CDP Timeout (ms)",
"browser.remoteCdpHandshakeTimeoutMs": "Remote CDP Handshake Timeout (ms)", "browser.remoteCdpHandshakeTimeoutMs": "Remote CDP Handshake Timeout (ms)",
"session.dmScope": "DM Session Scope", "session.dmScope": "DM Session Scope",

View File

@@ -8,6 +8,10 @@ export type BrowserProfileConfig = {
/** Profile color (hex). Auto-assigned at creation. */ /** Profile color (hex). Auto-assigned at creation. */
color: string; color: string;
}; };
export type BrowserSnapshotDefaults = {
/** Default snapshot mode (applies when mode is not provided). */
mode?: "efficient";
};
export type BrowserConfig = { export type BrowserConfig = {
enabled?: boolean; enabled?: boolean;
/** Base URL of the clawd browser control server. Default: http://127.0.0.1:18791 */ /** Base URL of the clawd browser control server. Default: http://127.0.0.1:18791 */
@@ -39,4 +43,6 @@ export type BrowserConfig = {
defaultProfile?: string; defaultProfile?: string;
/** Named browser profiles with explicit CDP ports or URLs. */ /** Named browser profiles with explicit CDP ports or URLs. */
profiles?: Record<string, BrowserProfileConfig>; profiles?: Record<string, BrowserProfileConfig>;
/** Default snapshot options (applied by the browser tool/CLI when unset). */
snapshotDefaults?: BrowserSnapshotDefaults;
}; };

View File

@@ -6,6 +6,13 @@ import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-
import { ChannelsSchema } from "./zod-schema.providers.js"; import { ChannelsSchema } from "./zod-schema.providers.js";
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js"; import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
const BrowserSnapshotDefaultsSchema = z
.object({
mode: z.literal("efficient").optional(),
})
.strict()
.optional();
export const ClawdbotSchema = z export const ClawdbotSchema = z
.object({ .object({
meta: z meta: z
@@ -113,6 +120,7 @@ export const ClawdbotSchema = z
noSandbox: z.boolean().optional(), noSandbox: z.boolean().optional(),
attachOnly: z.boolean().optional(), attachOnly: z.boolean().optional(),
defaultProfile: z.string().optional(), defaultProfile: z.string().optional(),
snapshotDefaults: BrowserSnapshotDefaultsSchema,
profiles: z profiles: z
.record( .record(
z z