fix: add browser snapshot default mode (#1336)
Co-authored-by: Seb Slight <sbarrios93@gmail.com>
This commit is contained in:
@@ -9,6 +9,7 @@ Docs: https://docs.clawd.bot
|
|||||||
- Repo: remove the Peekaboo git submodule now that the SPM release is used.
|
- Repo: remove the Peekaboo git submodule now that the SPM release is used.
|
||||||
- Update: sync plugin sources on channel switches and update npm-installed plugins during `clawdbot update`.
|
- Update: sync plugin sources on channel switches and update npm-installed plugins during `clawdbot update`.
|
||||||
- Plugins: share npm plugin update logic between `clawdbot update` and `clawdbot plugins update`.
|
- Plugins: share npm plugin update logic between `clawdbot update` and `clawdbot plugins update`.
|
||||||
|
- Browser: allow config defaults for efficient snapshots in the tool/CLI. (#1336) — thanks @sebslight.
|
||||||
- Channels: add the Nostr plugin channel with profile management + onboarding install defaults. (#1323) — thanks @joelklabo.
|
- Channels: add the Nostr plugin channel with profile management + onboarding install defaults. (#1323) — thanks @joelklabo.
|
||||||
- Plugins: require manifest-embedded config schemas, validate configs without loading plugin code, and surface plugin config warnings. (#1272) — thanks @thewilloftheshadow.
|
- Plugins: require manifest-embedded config schemas, validate configs without loading plugin code, and surface plugin config warnings. (#1272) — thanks @thewilloftheshadow.
|
||||||
- Plugins: move channel catalog metadata into plugin manifests; align Nextcloud Talk policy helpers with core patterns. (#1290) — thanks @NicholaiVogel.
|
- Plugins: move channel catalog metadata into plugin manifests; align Nextcloud Talk policy helpers with core patterns. (#1290) — thanks @NicholaiVogel.
|
||||||
|
|||||||
@@ -2615,10 +2615,13 @@ Defaults:
|
|||||||
// noSandbox: false,
|
// noSandbox: false,
|
||||||
// executablePath: "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
// executablePath: "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
||||||
// attachOnly: false, // set true when tunneling a remote CDP to localhost
|
// attachOnly: false, // set true when tunneling a remote CDP to localhost
|
||||||
|
// snapshotDefaults: { mode: "efficient" }, // tool/CLI default snapshot preset
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Note: `browser.snapshotDefaults` only affects Clawdbot's browser tool + CLI. Direct HTTP clients must pass `mode` explicitly.
|
||||||
|
|
||||||
### `ui` (Appearance)
|
### `ui` (Appearance)
|
||||||
|
|
||||||
Optional accent color used by the native apps for UI chrome (e.g. Talk Mode bubble tint).
|
Optional accent color used by the native apps for UI chrome (e.g. Talk Mode bubble tint).
|
||||||
|
|||||||
@@ -500,6 +500,7 @@ Notes:
|
|||||||
- `--format ai` (default when Playwright is installed): returns an AI snapshot with numeric refs (`aria-ref="<n>"`).
|
- `--format ai` (default when Playwright is installed): returns an AI snapshot with numeric refs (`aria-ref="<n>"`).
|
||||||
- `--format aria`: returns the accessibility tree (no refs; inspection only).
|
- `--format aria`: returns the accessibility tree (no refs; inspection only).
|
||||||
- `--efficient` (or `--mode efficient`): compact role snapshot preset (interactive + compact + depth + lower maxChars).
|
- `--efficient` (or `--mode efficient`): compact role snapshot preset (interactive + compact + depth + lower maxChars).
|
||||||
|
- Config default (tool/CLI only): set `browser.snapshotDefaults.mode: "efficient"` to use efficient snapshots when the caller does not pass a mode (see [Gateway configuration](/gateway/configuration#browser-clawd-managed-browser)).
|
||||||
- Role snapshot options (`--interactive`, `--compact`, `--depth`, `--selector`) force a role-based snapshot with refs like `ref=e12`.
|
- Role snapshot options (`--interactive`, `--compact`, `--depth`, `--selector`) force a role-based snapshot with refs like `ref=e12`.
|
||||||
- `--frame "<iframe selector>"` scopes role snapshots to an iframe (pairs with role refs like `e12`).
|
- `--frame "<iframe selector>"` scopes role snapshots to an iframe (pairs with role refs like `e12`).
|
||||||
- `--interactive` outputs a flat, easy-to-pick list of interactive elements (best for driving actions).
|
- `--interactive` outputs a flat, easy-to-pick list of interactive elements (best for driving actions).
|
||||||
|
|||||||
@@ -49,9 +49,10 @@ const browserConfigMocks = vi.hoisted(() => ({
|
|||||||
}));
|
}));
|
||||||
vi.mock("../../browser/config.js", () => browserConfigMocks);
|
vi.mock("../../browser/config.js", () => browserConfigMocks);
|
||||||
|
|
||||||
vi.mock("../../config/config.js", () => ({
|
const configMocks = vi.hoisted(() => ({
|
||||||
loadConfig: vi.fn(() => ({ browser: {} })),
|
loadConfig: vi.fn(() => ({ browser: {} })),
|
||||||
}));
|
}));
|
||||||
|
vi.mock("../../config/config.js", () => configMocks);
|
||||||
|
|
||||||
const toolCommonMocks = vi.hoisted(() => ({
|
const toolCommonMocks = vi.hoisted(() => ({
|
||||||
imageResultFromFile: vi.fn(),
|
imageResultFromFile: vi.fn(),
|
||||||
@@ -70,11 +71,12 @@ import { createBrowserTool } from "./browser-tool.js";
|
|||||||
describe("browser tool snapshot maxChars", () => {
|
describe("browser tool snapshot maxChars", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
configMocks.loadConfig.mockReturnValue({ browser: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("applies the default ai snapshot limit", async () => {
|
it("applies the default ai snapshot limit", async () => {
|
||||||
const tool = createBrowserTool();
|
const tool = createBrowserTool();
|
||||||
await tool.execute?.(null, { action: "snapshot", format: "ai" });
|
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
|
||||||
|
|
||||||
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
||||||
"http://127.0.0.1:18791",
|
"http://127.0.0.1:18791",
|
||||||
@@ -90,7 +92,7 @@ describe("browser tool snapshot maxChars", () => {
|
|||||||
const override = 2_000;
|
const override = 2_000;
|
||||||
await tool.execute?.(null, {
|
await tool.execute?.(null, {
|
||||||
action: "snapshot",
|
action: "snapshot",
|
||||||
format: "ai",
|
snapshotFormat: "ai",
|
||||||
maxChars: override,
|
maxChars: override,
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -106,7 +108,7 @@ describe("browser tool snapshot maxChars", () => {
|
|||||||
const tool = createBrowserTool();
|
const tool = createBrowserTool();
|
||||||
await tool.execute?.(null, {
|
await tool.execute?.(null, {
|
||||||
action: "snapshot",
|
action: "snapshot",
|
||||||
format: "ai",
|
snapshotFormat: "ai",
|
||||||
maxChars: 0,
|
maxChars: 0,
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -124,7 +126,7 @@ describe("browser tool snapshot maxChars", () => {
|
|||||||
|
|
||||||
it("passes refs mode through to browser snapshot", async () => {
|
it("passes refs mode through to browser snapshot", async () => {
|
||||||
const tool = createBrowserTool();
|
const tool = createBrowserTool();
|
||||||
await tool.execute?.(null, { action: "snapshot", format: "ai", refs: "aria" });
|
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai", refs: "aria" });
|
||||||
|
|
||||||
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
||||||
"http://127.0.0.1:18791",
|
"http://127.0.0.1:18791",
|
||||||
@@ -135,9 +137,36 @@ describe("browser tool snapshot maxChars", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses config snapshot defaults when mode is not provided", async () => {
|
||||||
|
configMocks.loadConfig.mockReturnValue({
|
||||||
|
browser: { snapshotDefaults: { mode: "efficient" } },
|
||||||
|
});
|
||||||
|
const tool = createBrowserTool();
|
||||||
|
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "ai" });
|
||||||
|
|
||||||
|
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
||||||
|
"http://127.0.0.1:18791",
|
||||||
|
expect.objectContaining({
|
||||||
|
mode: "efficient",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not apply config snapshot defaults to aria snapshots", async () => {
|
||||||
|
configMocks.loadConfig.mockReturnValue({
|
||||||
|
browser: { snapshotDefaults: { mode: "efficient" } },
|
||||||
|
});
|
||||||
|
const tool = createBrowserTool();
|
||||||
|
await tool.execute?.(null, { action: "snapshot", snapshotFormat: "aria" });
|
||||||
|
|
||||||
|
expect(browserClientMocks.browserSnapshot).toHaveBeenCalled();
|
||||||
|
const [, opts] = browserClientMocks.browserSnapshot.mock.calls.at(-1) ?? [];
|
||||||
|
expect(opts?.mode).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
it("defaults to host when using profile=chrome (even in sandboxed sessions)", async () => {
|
it("defaults to host when using profile=chrome (even in sandboxed sessions)", async () => {
|
||||||
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
|
const tool = createBrowserTool({ defaultControlUrl: "http://127.0.0.1:9999" });
|
||||||
await tool.execute?.(null, { action: "snapshot", profile: "chrome", format: "ai" });
|
await tool.execute?.(null, { action: "snapshot", profile: "chrome", snapshotFormat: "ai" });
|
||||||
|
|
||||||
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
expect(browserClientMocks.browserSnapshot).toHaveBeenCalledWith(
|
||||||
"http://127.0.0.1:18791",
|
"http://127.0.0.1:18791",
|
||||||
@@ -151,6 +180,7 @@ describe("browser tool snapshot maxChars", () => {
|
|||||||
describe("browser tool snapshot labels", () => {
|
describe("browser tool snapshot labels", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
configMocks.loadConfig.mockReturnValue({ browser: {} });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns image + text when labels are requested", async () => {
|
it("returns image + text when labels are requested", async () => {
|
||||||
@@ -175,7 +205,7 @@ describe("browser tool snapshot labels", () => {
|
|||||||
|
|
||||||
const result = await tool.execute?.(null, {
|
const result = await tool.execute?.(null, {
|
||||||
action: "snapshot",
|
action: "snapshot",
|
||||||
format: "ai",
|
snapshotFormat: "ai",
|
||||||
labels: true,
|
labels: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -190,11 +190,17 @@ export function createBrowserTool(opts?: {
|
|||||||
return jsonResult({ ok: true });
|
return jsonResult({ ok: true });
|
||||||
}
|
}
|
||||||
case "snapshot": {
|
case "snapshot": {
|
||||||
|
const snapshotDefaults = loadConfig().browser?.snapshotDefaults;
|
||||||
const format =
|
const format =
|
||||||
params.snapshotFormat === "ai" || params.snapshotFormat === "aria"
|
params.snapshotFormat === "ai" || params.snapshotFormat === "aria"
|
||||||
? (params.snapshotFormat as "ai" | "aria")
|
? (params.snapshotFormat as "ai" | "aria")
|
||||||
: "ai";
|
: "ai";
|
||||||
const mode = params.mode === "efficient" ? "efficient" : undefined;
|
const mode =
|
||||||
|
params.mode === "efficient"
|
||||||
|
? "efficient"
|
||||||
|
: format === "ai" && snapshotDefaults?.mode === "efficient"
|
||||||
|
? "efficient"
|
||||||
|
: undefined;
|
||||||
const labels = typeof params.labels === "boolean" ? params.labels : undefined;
|
const labels = typeof params.labels === "boolean" ? params.labels : undefined;
|
||||||
const refs = params.refs === "aria" || params.refs === "role" ? params.refs : undefined;
|
const refs = params.refs === "aria" || params.refs === "role" ? params.refs : undefined;
|
||||||
const hasMaxChars = Object.hasOwn(params, "maxChars");
|
const hasMaxChars = Object.hasOwn(params, "maxChars");
|
||||||
|
|||||||
78
src/cli/browser-cli-inspect.test.ts
Normal file
78
src/cli/browser-cli-inspect.test.ts
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { Command } from "commander";
|
||||||
|
|
||||||
|
const clientMocks = vi.hoisted(() => ({
|
||||||
|
browserSnapshot: vi.fn(async () => ({
|
||||||
|
ok: true,
|
||||||
|
format: "ai",
|
||||||
|
targetId: "t1",
|
||||||
|
url: "https://example.com",
|
||||||
|
snapshot: "ok",
|
||||||
|
})),
|
||||||
|
resolveBrowserControlUrl: vi.fn(() => "http://127.0.0.1:18791"),
|
||||||
|
}));
|
||||||
|
vi.mock("../browser/client.js", () => clientMocks);
|
||||||
|
|
||||||
|
const configMocks = vi.hoisted(() => ({
|
||||||
|
loadConfig: vi.fn(() => ({ browser: {} })),
|
||||||
|
}));
|
||||||
|
vi.mock("../config/config.js", () => configMocks);
|
||||||
|
|
||||||
|
const runtime = {
|
||||||
|
log: vi.fn(),
|
||||||
|
error: vi.fn(),
|
||||||
|
exit: vi.fn(),
|
||||||
|
};
|
||||||
|
vi.mock("../runtime.js", () => ({
|
||||||
|
defaultRuntime: runtime,
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe("browser cli snapshot defaults", () => {
|
||||||
|
afterEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
configMocks.loadConfig.mockReturnValue({ browser: {} });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses config snapshot defaults when mode is not provided", async () => {
|
||||||
|
configMocks.loadConfig.mockReturnValue({
|
||||||
|
browser: { snapshotDefaults: { mode: "efficient" } },
|
||||||
|
});
|
||||||
|
const { registerBrowserInspectCommands } = await import("./browser-cli-inspect.js");
|
||||||
|
const program = new Command();
|
||||||
|
const browser = program.command("browser").option("--json", false);
|
||||||
|
registerBrowserInspectCommands(browser, () => ({}));
|
||||||
|
|
||||||
|
await program.parseAsync(["browser", "snapshot"], { from: "user" });
|
||||||
|
|
||||||
|
expect(clientMocks.browserSnapshot).toHaveBeenCalledWith(
|
||||||
|
"http://127.0.0.1:18791",
|
||||||
|
expect.objectContaining({
|
||||||
|
format: "ai",
|
||||||
|
mode: "efficient",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not apply config snapshot defaults to aria snapshots", async () => {
|
||||||
|
configMocks.loadConfig.mockReturnValue({
|
||||||
|
browser: { snapshotDefaults: { mode: "efficient" } },
|
||||||
|
});
|
||||||
|
clientMocks.browserSnapshot.mockResolvedValueOnce({
|
||||||
|
ok: true,
|
||||||
|
format: "aria",
|
||||||
|
targetId: "t1",
|
||||||
|
url: "https://example.com",
|
||||||
|
nodes: [],
|
||||||
|
});
|
||||||
|
const { registerBrowserInspectCommands } = await import("./browser-cli-inspect.js");
|
||||||
|
const program = new Command();
|
||||||
|
const browser = program.command("browser").option("--json", false);
|
||||||
|
registerBrowserInspectCommands(browser, () => ({}));
|
||||||
|
|
||||||
|
await program.parseAsync(["browser", "snapshot", "--format", "aria"], { from: "user" });
|
||||||
|
|
||||||
|
expect(clientMocks.browserSnapshot).toHaveBeenCalled();
|
||||||
|
const [, opts] = clientMocks.browserSnapshot.mock.calls.at(-1) ?? [];
|
||||||
|
expect(opts?.mode).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -2,6 +2,7 @@ import type { Command } from "commander";
|
|||||||
|
|
||||||
import { browserSnapshot, resolveBrowserControlUrl } from "../browser/client.js";
|
import { browserSnapshot, resolveBrowserControlUrl } from "../browser/client.js";
|
||||||
import { browserScreenshotAction } from "../browser/client-actions.js";
|
import { browserScreenshotAction } from "../browser/client-actions.js";
|
||||||
|
import { loadConfig } from "../config/config.js";
|
||||||
import { danger } from "../globals.js";
|
import { danger } from "../globals.js";
|
||||||
import { defaultRuntime } from "../runtime.js";
|
import { defaultRuntime } from "../runtime.js";
|
||||||
import type { BrowserParentOpts } from "./browser-cli-shared.js";
|
import type { BrowserParentOpts } from "./browser-cli-shared.js";
|
||||||
@@ -62,7 +63,11 @@ export function registerBrowserInspectCommands(
|
|||||||
const baseUrl = resolveBrowserControlUrl(parent?.url);
|
const baseUrl = resolveBrowserControlUrl(parent?.url);
|
||||||
const profile = parent?.browserProfile;
|
const profile = parent?.browserProfile;
|
||||||
const format = opts.format === "aria" ? "aria" : "ai";
|
const format = opts.format === "aria" ? "aria" : "ai";
|
||||||
const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : undefined;
|
const configMode =
|
||||||
|
format === "ai" && loadConfig().browser?.snapshotDefaults?.mode === "efficient"
|
||||||
|
? "efficient"
|
||||||
|
: undefined;
|
||||||
|
const mode = opts.efficient === true || opts.mode === "efficient" ? "efficient" : configMode;
|
||||||
try {
|
try {
|
||||||
const result = await browserSnapshot(baseUrl, {
|
const result = await browserSnapshot(baseUrl, {
|
||||||
format,
|
format,
|
||||||
|
|||||||
@@ -250,6 +250,8 @@ const FIELD_LABELS: Record<string, string> = {
|
|||||||
"commands.useAccessGroups": "Use Access Groups",
|
"commands.useAccessGroups": "Use Access Groups",
|
||||||
"ui.seamColor": "Accent Color",
|
"ui.seamColor": "Accent Color",
|
||||||
"browser.controlUrl": "Browser Control URL",
|
"browser.controlUrl": "Browser Control URL",
|
||||||
|
"browser.snapshotDefaults": "Browser Snapshot Defaults",
|
||||||
|
"browser.snapshotDefaults.mode": "Browser Snapshot Mode",
|
||||||
"browser.remoteCdpTimeoutMs": "Remote CDP Timeout (ms)",
|
"browser.remoteCdpTimeoutMs": "Remote CDP Timeout (ms)",
|
||||||
"browser.remoteCdpHandshakeTimeoutMs": "Remote CDP Handshake Timeout (ms)",
|
"browser.remoteCdpHandshakeTimeoutMs": "Remote CDP Handshake Timeout (ms)",
|
||||||
"session.dmScope": "DM Session Scope",
|
"session.dmScope": "DM Session Scope",
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ export type BrowserProfileConfig = {
|
|||||||
/** Profile color (hex). Auto-assigned at creation. */
|
/** Profile color (hex). Auto-assigned at creation. */
|
||||||
color: string;
|
color: string;
|
||||||
};
|
};
|
||||||
|
export type BrowserSnapshotDefaults = {
|
||||||
|
/** Default snapshot mode (applies when mode is not provided). */
|
||||||
|
mode?: "efficient";
|
||||||
|
};
|
||||||
export type BrowserConfig = {
|
export type BrowserConfig = {
|
||||||
enabled?: boolean;
|
enabled?: boolean;
|
||||||
/** Base URL of the clawd browser control server. Default: http://127.0.0.1:18791 */
|
/** Base URL of the clawd browser control server. Default: http://127.0.0.1:18791 */
|
||||||
@@ -39,4 +43,6 @@ export type BrowserConfig = {
|
|||||||
defaultProfile?: string;
|
defaultProfile?: string;
|
||||||
/** Named browser profiles with explicit CDP ports or URLs. */
|
/** Named browser profiles with explicit CDP ports or URLs. */
|
||||||
profiles?: Record<string, BrowserProfileConfig>;
|
profiles?: Record<string, BrowserProfileConfig>;
|
||||||
|
/** Default snapshot options (applied by the browser tool/CLI when unset). */
|
||||||
|
snapshotDefaults?: BrowserSnapshotDefaults;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -6,6 +6,13 @@ import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-
|
|||||||
import { ChannelsSchema } from "./zod-schema.providers.js";
|
import { ChannelsSchema } from "./zod-schema.providers.js";
|
||||||
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
|
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
|
||||||
|
|
||||||
|
const BrowserSnapshotDefaultsSchema = z
|
||||||
|
.object({
|
||||||
|
mode: z.literal("efficient").optional(),
|
||||||
|
})
|
||||||
|
.strict()
|
||||||
|
.optional();
|
||||||
|
|
||||||
export const ClawdbotSchema = z
|
export const ClawdbotSchema = z
|
||||||
.object({
|
.object({
|
||||||
meta: z
|
meta: z
|
||||||
@@ -113,6 +120,7 @@ export const ClawdbotSchema = z
|
|||||||
noSandbox: z.boolean().optional(),
|
noSandbox: z.boolean().optional(),
|
||||||
attachOnly: z.boolean().optional(),
|
attachOnly: z.boolean().optional(),
|
||||||
defaultProfile: z.string().optional(),
|
defaultProfile: z.string().optional(),
|
||||||
|
snapshotDefaults: BrowserSnapshotDefaultsSchema,
|
||||||
profiles: z
|
profiles: z
|
||||||
.record(
|
.record(
|
||||||
z
|
z
|
||||||
|
|||||||
Reference in New Issue
Block a user