diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f588aaf3..3d0808dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Browser: extension mode recovers when only one tab is attached (stale targetId fallback). - Browser: prefer stable Chrome for auto-detect, with Brave/Edge fallbacks and updated docs. (#983) — thanks @cpojer. - Browser: fix `tab not found` for extension relay snapshots/actions when Playwright blocks `newCDPSession` (use the single available Page). +- Browser: preserve auth/query tokens for remote CDP endpoints and pass Basic auth for CDP HTTP/WS. (#895) — thanks @mukhtharcm. - Telegram: add bidirectional reaction support with configurable notifications and agent guidance. (#964) — thanks @bohdanpodvirnyi. - Telegram: skip `message_thread_id=1` for General topic sends while keeping typing indicators. (#848) — thanks @azade-c. - Discord: allow allowlisted guilds without channel lists to receive messages when `groupPolicy="allowlist"`. — thanks @thewilloftheshadow. diff --git a/docs/tools/browser.md b/docs/tools/browser.md index f6cf54b21..2fab16c50 100644 --- a/docs/tools/browser.md +++ b/docs/tools/browser.md @@ -154,6 +154,14 @@ Example: Use `profiles..cdpUrl` for **remote CDP** if you want the Gateway to talk directly to a Chromium-based browser instance without a remote control server. +Remote CDP URLs can include auth: +- Query tokens (e.g., `https://provider.example?token=`) +- HTTP Basic auth (e.g., `https://user:pass@provider.example`) + +Clawdbot preserves the auth when calling `/json/*` endpoints and when connecting +to the CDP WebSocket. Prefer environment variables or secrets managers for +tokens instead of committing them to config files. + ### Running the control server on the browser machine Run a standalone browser control server (recommended when your Gateway is remote): diff --git a/src/browser/cdp.helpers.test.ts b/src/browser/cdp.helpers.test.ts new file mode 100644 index 000000000..0242d2ee6 --- /dev/null +++ b/src/browser/cdp.helpers.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { appendCdpPath, getHeadersWithAuth } from "./cdp.helpers.js"; + +describe("cdp.helpers", () => { + it("preserves query params when appending CDP paths", () => { + const url = appendCdpPath("https://example.com?token=abc", "/json/version"); + expect(url).toBe("https://example.com/json/version?token=abc"); + }); + + it("appends paths under a base prefix", () => { + const url = appendCdpPath("https://example.com/chrome/?token=abc", "json/list"); + expect(url).toBe("https://example.com/chrome/json/list?token=abc"); + }); + + it("adds basic auth headers when credentials are present", () => { + const headers = getHeadersWithAuth("https://user:pass@example.com"); + expect(headers.Authorization).toBe( + `Basic ${Buffer.from("user:pass").toString("base64")}`, + ); + }); + + it("keeps preexisting authorization headers", () => { + const headers = getHeadersWithAuth("https://user:pass@example.com", { + Authorization: "Bearer token", + }); + expect(headers.Authorization).toBe("Bearer token"); + }); +}); diff --git a/src/browser/cdp.helpers.ts b/src/browser/cdp.helpers.ts index 89f7fb563..c41f4c81d 100644 --- a/src/browser/cdp.helpers.ts +++ b/src/browser/cdp.helpers.ts @@ -28,6 +28,34 @@ export function isLoopbackHost(host: string) { ); } +export function getHeadersWithAuth( + url: string, + headers: Record = {}, +) { + try { + const parsed = new URL(url); + const hasAuthHeader = Object.keys(headers).some( + (key) => key.toLowerCase() === "authorization", + ); + if (hasAuthHeader) return headers; + if (parsed.username || parsed.password) { + const auth = Buffer.from(`${parsed.username}:${parsed.password}`).toString("base64"); + return { ...headers, Authorization: `Basic ${auth}` }; + } + } catch { + // ignore + } + return headers; +} + +export function appendCdpPath(cdpUrl: string, path: string): string { + const url = new URL(cdpUrl); + const basePath = url.pathname.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + url.pathname = `${basePath}${suffix}`; + return url.toString(); +} + function createCdpSender(ws: WebSocket) { let nextId = 1; const pending = new Map(); @@ -75,11 +103,19 @@ function createCdpSender(ws: WebSocket) { return { send, closeWithError }; } -export async function fetchJson(url: string, timeoutMs = 1500): Promise { +export async function fetchJson( + url: string, + timeoutMs = 1500, + init?: RequestInit, +): Promise { const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { - const res = await fetch(url, { signal: ctrl.signal }); + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return (await res.json()) as T; } finally { @@ -87,11 +123,35 @@ export async function fetchJson(url: string, timeoutMs = 1500): Promise { } } +export async function fetchOk( + url: string, + timeoutMs = 1500, + init?: RequestInit, +): Promise { + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + } finally { + clearTimeout(t); + } +} + export async function withCdpSocket( wsUrl: string, fn: (send: CdpSendFn) => Promise, + opts?: { headers?: Record }, ): Promise { - const ws = new WebSocket(wsUrl, { handshakeTimeout: 5000 }); + const headers = getHeadersWithAuth(wsUrl, opts?.headers ?? {}); + const ws = new WebSocket(wsUrl, { + handshakeTimeout: 5000, + ...(Object.keys(headers).length ? { headers } : {}), + }); const { send, closeWithError } = createCdpSender(ws); const openPromise = new Promise((resolve, reject) => { diff --git a/src/browser/cdp.test.ts b/src/browser/cdp.test.ts index a75cf7134..ba3fefdac 100644 --- a/src/browser/cdp.test.ts +++ b/src/browser/cdp.test.ts @@ -165,4 +165,12 @@ describe("cdp", () => { ); expect(normalized).toBe("ws://example.com:9222/devtools/browser/ABC"); }); + + it("propagates auth and query params onto normalized websocket URLs", () => { + const normalized = normalizeCdpWsUrl( + "ws://127.0.0.1:9222/devtools/browser/ABC", + "https://user:pass@example.com?token=abc", + ); + expect(normalized).toBe("wss://user:pass@example.com/devtools/browser/ABC?token=abc"); + }); }); diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index fce55048b..7b6ff30a0 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -1,33 +1,11 @@ -import { isLoopbackHost, withCdpSocket } from "./cdp.helpers.js"; +import { + appendCdpPath, + fetchJson, + isLoopbackHost, + withCdpSocket, +} from "./cdp.helpers.js"; -export function getHeadersWithAuth( - url: string, - headers: Record = {}, -) { - try { - const parsed = new URL(url); - if (parsed.username || parsed.password) { - const auth = Buffer.from( - `${parsed.username}:${parsed.password}`, - ).toString("base64"); - return { ...headers, Authorization: `Basic ${auth}` }; - } - } catch (_e) { - // ignore - } - return headers; -} - -export async function fetchJson(url: string): Promise { - const res = await fetch(url, { headers: getHeadersWithAuth(url) }); - if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); - return (await res.json()) as Promise; -} - -export async function fetchOk(url: string): Promise { - const res = await fetch(url, { headers: getHeadersWithAuth(url) }); - if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); -} +export { appendCdpPath, fetchJson, fetchOk, getHeadersWithAuth } from "./cdp.helpers.js"; export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string { const ws = new URL(wsUrl); @@ -38,6 +16,13 @@ export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string { if (cdpPort) ws.port = cdpPort; ws.protocol = cdp.protocol === "https:" ? "wss:" : "ws:"; } + if (!ws.username && !ws.password && (cdp.username || cdp.password)) { + ws.username = cdp.username; + ws.password = cdp.password; + } + for (const [key, value] of cdp.searchParams.entries()) { + if (!ws.searchParams.has(key)) ws.searchParams.append(key, value); + } return ws.toString(); } @@ -97,9 +82,9 @@ export async function createTargetViaCdp(opts: { cdpUrl: string; url: string; }): Promise<{ targetId: string }> { - const base = opts.cdpUrl.replace(/\/$/, ""); const version = await fetchJson<{ webSocketDebuggerUrl?: string }>( - `${base}/json/version`, + appendCdpPath(opts.cdpUrl, "/json/version"), + 1500, ); const wsUrlRaw = String(version?.webSocketDebuggerUrl ?? "").trim(); const wsUrl = wsUrlRaw ? normalizeCdpWsUrl(wsUrlRaw, opts.cdpUrl) : ""; diff --git a/src/browser/chrome.ts b/src/browser/chrome.ts index b7c6a7ec3..76e2ee00e 100644 --- a/src/browser/chrome.ts +++ b/src/browser/chrome.ts @@ -8,6 +8,7 @@ import { ensurePortAvailable } from "../infra/ports.js"; import { createSubsystemLogger } from "../logging.js"; import { CONFIG_DIR } from "../utils.js"; import { getHeadersWithAuth, normalizeCdpWsUrl } from "./cdp.js"; +import { appendCdpPath } from "./cdp.helpers.js"; import { type BrowserExecutable, resolveBrowserExecutableForPlatform, @@ -71,10 +72,10 @@ async function fetchChromeVersion(cdpUrl: string, timeoutMs = 500): Promise ctrl.abort(), timeoutMs); try { - const base = cdpUrl.replace(/\/$/, ""); - const res = await fetch(`${base}/json/version`, { + const versionUrl = appendCdpPath(cdpUrl, "/json/version"); + const res = await fetch(versionUrl, { signal: ctrl.signal, - headers: getHeadersWithAuth(`${base}/json/version`), + headers: getHeadersWithAuth(versionUrl), }); if (!res.ok) return null; const data = (await res.json()) as ChromeVersion; @@ -99,7 +100,11 @@ export async function getChromeWebSocketUrl( async function canOpenWebSocket(wsUrl: string, timeoutMs = 800): Promise { return await new Promise((resolve) => { - const ws = new WebSocket(wsUrl, { handshakeTimeout: timeoutMs }); + const headers = getHeadersWithAuth(wsUrl); + const ws = new WebSocket(wsUrl, { + handshakeTimeout: timeoutMs, + ...(Object.keys(headers).length ? { headers } : {}), + }); const timer = setTimeout( () => { try { diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index 7c29f63f1..119e1d315 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -8,7 +8,7 @@ import type { } from "playwright-core"; import { chromium } from "playwright-core"; import { formatErrorMessage } from "../infra/errors.js"; -import { isLoopbackHost } from "./cdp.helpers.js"; +import { getHeadersWithAuth } from "./cdp.helpers.js"; import { getChromeWebSocketUrl } from "./chrome.js"; export type BrowserConsoleMessage = { @@ -257,18 +257,6 @@ function observeBrowser(browser: Browser) { for (const context of browser.contexts()) observeContext(context); } -export async function getConnectedBrowser(cdpUrl: string): Promise { - const { browser } = await connectBrowser(cdpUrl); - return browser; -} - -function getWsEndpoint(cdpUrl: string): string { - const u = new URL(cdpUrl); - // Ensure we use wss/ws matching the protocol - u.protocol = u.protocol === "https:" ? "wss:" : "ws:"; - return u.toString(); -} - async function connectBrowser(cdpUrl: string): Promise { const normalized = normalizeCdpUrl(cdpUrl); if (cached?.cdpUrl === normalized) return cached; @@ -283,7 +271,8 @@ async function connectBrowser(cdpUrl: string): Promise { () => null, ); const endpoint = wsUrl ?? normalized; - const browser = await chromium.connectOverCDP(endpoint, { timeout }); + const headers = getHeadersWithAuth(endpoint); + const browser = await chromium.connectOverCDP(endpoint, { timeout, headers }); const connected: ConnectedBrowser = { browser, cdpUrl: normalized }; cached = connected; observeBrowser(browser); diff --git a/src/browser/server-context.ts b/src/browser/server-context.ts index 314b89b22..a425a49cf 100644 --- a/src/browser/server-context.ts +++ b/src/browser/server-context.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import { + appendCdpPath, createTargetViaCdp, getHeadersWithAuth, normalizeCdpWsUrl, @@ -14,7 +15,6 @@ import { } from "./chrome.js"; import type { ResolvedBrowserProfile } from "./config.js"; import { resolveProfile } from "./config.js"; -import { getConnectedBrowser } from "./pw-session.js"; import type { BrowserRouteContext, BrowserTab, @@ -110,40 +110,7 @@ function createProfileContext( profileState.running = running; }; - const listTabsViaPlaywright = async (): Promise => { - const browser = await getConnectedBrowser(profile.cdpUrl); - const contexts = browser.contexts(); - const pages = contexts.flatMap((c) => c.pages()); - // Note: Playwright pages don't sync title instantly, returning URL is safer for list - return pages.map((p) => ({ - targetId: p.url(), - title: "", - url: p.url(), - type: "page", - })); - }; - - const openTabViaPlaywright = async (url?: string): Promise => { - const browser = await getConnectedBrowser(profile.cdpUrl); - // Reuse context or create new - const context = browser.contexts()[0] || (await browser.newContext()); - const page = await context.newPage(); - if (url) await page.goto(url); - return { targetId: page.url(), title: "", url: page.url(), type: "page" }; - }; - - const closeTabViaPlaywright = async (targetId: string) => { - const browser = await getConnectedBrowser(profile.cdpUrl); - const pages = browser.contexts().flatMap((c) => c.pages()); - // Find page by URL match (simple strategy) - const page = pages.find((p) => p.url() === targetId); - if (page) await page.close(); - }; - const listTabs = async (): Promise => { - if (!profile.cdpIsLoopback) { - return await listTabsViaPlaywright(); - } const raw = await fetchJson< Array<{ id?: string; @@ -152,7 +119,7 @@ function createProfileContext( webSocketDebuggerUrl?: string; type?: string; }> - >(`${profile.cdpUrl.replace(/\/$/, "")}/json/list`); + >(appendCdpPath(profile.cdpUrl, "/json/list")); return raw .map((t) => ({ targetId: t.id ?? "", @@ -165,9 +132,6 @@ function createProfileContext( }; const openTab = async (url: string): Promise => { - if (!profile.cdpIsLoopback) { - return await openTabViaPlaywright(url); - } const createdViaCdp = await createTargetViaCdp({ cdpUrl: profile.cdpUrl, url, @@ -195,8 +159,13 @@ function createProfileContext( type?: string; }; - const base = profile.cdpUrl.replace(/\/$/, ""); - const endpoint = `${base}/json/new?${encoded}`; + const endpointUrl = new URL(appendCdpPath(profile.cdpUrl, "/json/new")); + const endpoint = endpointUrl.search + ? (() => { + endpointUrl.searchParams.set("url", url); + return endpointUrl.toString(); + })() + : `${endpointUrl.toString()}?${encoded}`; const created = await fetchJson(endpoint, 1500, { method: "PUT", }).catch(async (err) => { @@ -213,7 +182,7 @@ function createProfileContext( targetId: created.id, title: created.title ?? "", url: created.url ?? url, - wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, base), + wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, profile.cdpUrl), type: created.type, }; }; @@ -376,7 +345,6 @@ function createProfileContext( }; const focusTab = async (targetId: string): Promise => { - const base = profile.cdpUrl.replace(/\/$/, ""); const tabs = await listTabs(); const resolved = resolveTargetIdFromTabs(targetId, tabs); if (!resolved.ok) { @@ -385,16 +353,12 @@ function createProfileContext( } throw new Error("tab not found"); } - await fetchOk(`${base}/json/activate/${resolved.targetId}`); + await fetchOk(appendCdpPath(profile.cdpUrl, `/json/activate/${resolved.targetId}`)); const profileState = getProfileState(); profileState.lastTargetId = resolved.targetId; }; const closeTab = async (targetId: string): Promise => { - if (!profile.cdpIsLoopback) { - return await closeTabViaPlaywright(targetId); - } - const base = profile.cdpUrl.replace(/\/$/, ""); const tabs = await listTabs(); const resolved = resolveTargetIdFromTabs(targetId, tabs); if (!resolved.ok) { @@ -403,7 +367,7 @@ function createProfileContext( } throw new Error("tab not found"); } - await fetchOk(`${base}/json/close/${resolved.targetId}`); + await fetchOk(appendCdpPath(profile.cdpUrl, `/json/close/${resolved.targetId}`)); }; const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => { diff --git a/src/browser/server.agent-contract-form-layout-act-commands.test.ts b/src/browser/server.agent-contract-form-layout-act-commands.test.ts index 199673182..cbff6dca5 100644 --- a/src/browser/server.agent-contract-form-layout-act-commands.test.ts +++ b/src/browser/server.agent-contract-form-layout-act-commands.test.ts @@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.agent-contract-snapshot-endpoints.test.ts b/src/browser/server.agent-contract-snapshot-endpoints.test.ts index e6add0d32..65a86facb 100644 --- a/src/browser/server.agent-contract-snapshot-endpoints.test.ts +++ b/src/browser/server.agent-contract-snapshot-endpoints.test.ts @@ -130,6 +130,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.covers-additional-endpoint-branches.test.ts b/src/browser/server.covers-additional-endpoint-branches.test.ts index beee42dee..bc16040cf 100644 --- a/src/browser/server.covers-additional-endpoint-branches.test.ts +++ b/src/browser/server.covers-additional-endpoint-branches.test.ts @@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts index 055a18a99..4908a536c 100644 --- a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts +++ b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts @@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.serves-status-starts-browser-requested.test.ts b/src/browser/server.serves-status-starts-browser-requested.test.ts index 03d4f6d17..8cfcca40e 100644 --- a/src/browser/server.serves-status-starts-browser-requested.test.ts +++ b/src/browser/server.serves-status-starts-browser-requested.test.ts @@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts b/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts index aa695fa8c..d50313413 100644 --- a/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts +++ b/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts @@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({ normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks);