diff --git a/CHANGELOG.md b/CHANGELOG.md index 93064e66c..9601811b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Browser: extension mode recovers when only one tab is attached (stale targetId fallback). - Browser: prefer stable Chrome for auto-detect, with Brave/Edge fallbacks and updated docs. (#983) — thanks @cpojer. - Browser: fix `tab not found` for extension relay snapshots/actions when Playwright blocks `newCDPSession` (use the single available Page). +- Browser: preserve auth/query tokens for remote CDP endpoints and pass Basic auth for CDP HTTP/WS. (#895) — thanks @mukhtharcm. - Telegram: add bidirectional reaction support with configurable notifications and agent guidance. (#964) — thanks @bohdanpodvirnyi. - Telegram: allow custom commands in the bot menu (merged with native; conflicts ignored). (#860) — thanks @nachoiacovino. - Telegram: skip `message_thread_id=1` for General topic sends while keeping typing indicators. (#848) — thanks @azade-c. diff --git a/docs/tools/browser.md b/docs/tools/browser.md index f6cf54b21..2fab16c50 100644 --- a/docs/tools/browser.md +++ b/docs/tools/browser.md @@ -154,6 +154,14 @@ Example: Use `profiles..cdpUrl` for **remote CDP** if you want the Gateway to talk directly to a Chromium-based browser instance without a remote control server. +Remote CDP URLs can include auth: +- Query tokens (e.g., `https://provider.example?token=`) +- HTTP Basic auth (e.g., `https://user:pass@provider.example`) + +Clawdbot preserves the auth when calling `/json/*` endpoints and when connecting +to the CDP WebSocket. Prefer environment variables or secrets managers for +tokens instead of committing them to config files. + ### Running the control server on the browser machine Run a standalone browser control server (recommended when your Gateway is remote): diff --git a/src/browser/cdp.helpers.test.ts b/src/browser/cdp.helpers.test.ts new file mode 100644 index 000000000..0242d2ee6 --- /dev/null +++ b/src/browser/cdp.helpers.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; + +import { appendCdpPath, getHeadersWithAuth } from "./cdp.helpers.js"; + +describe("cdp.helpers", () => { + it("preserves query params when appending CDP paths", () => { + const url = appendCdpPath("https://example.com?token=abc", "/json/version"); + expect(url).toBe("https://example.com/json/version?token=abc"); + }); + + it("appends paths under a base prefix", () => { + const url = appendCdpPath("https://example.com/chrome/?token=abc", "json/list"); + expect(url).toBe("https://example.com/chrome/json/list?token=abc"); + }); + + it("adds basic auth headers when credentials are present", () => { + const headers = getHeadersWithAuth("https://user:pass@example.com"); + expect(headers.Authorization).toBe( + `Basic ${Buffer.from("user:pass").toString("base64")}`, + ); + }); + + it("keeps preexisting authorization headers", () => { + const headers = getHeadersWithAuth("https://user:pass@example.com", { + Authorization: "Bearer token", + }); + expect(headers.Authorization).toBe("Bearer token"); + }); +}); diff --git a/src/browser/cdp.helpers.ts b/src/browser/cdp.helpers.ts index 89f7fb563..c41f4c81d 100644 --- a/src/browser/cdp.helpers.ts +++ b/src/browser/cdp.helpers.ts @@ -28,6 +28,34 @@ export function isLoopbackHost(host: string) { ); } +export function getHeadersWithAuth( + url: string, + headers: Record = {}, +) { + try { + const parsed = new URL(url); + const hasAuthHeader = Object.keys(headers).some( + (key) => key.toLowerCase() === "authorization", + ); + if (hasAuthHeader) return headers; + if (parsed.username || parsed.password) { + const auth = Buffer.from(`${parsed.username}:${parsed.password}`).toString("base64"); + return { ...headers, Authorization: `Basic ${auth}` }; + } + } catch { + // ignore + } + return headers; +} + +export function appendCdpPath(cdpUrl: string, path: string): string { + const url = new URL(cdpUrl); + const basePath = url.pathname.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + url.pathname = `${basePath}${suffix}`; + return url.toString(); +} + function createCdpSender(ws: WebSocket) { let nextId = 1; const pending = new Map(); @@ -75,11 +103,19 @@ function createCdpSender(ws: WebSocket) { return { send, closeWithError }; } -export async function fetchJson(url: string, timeoutMs = 1500): Promise { +export async function fetchJson( + url: string, + timeoutMs = 1500, + init?: RequestInit, +): Promise { const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { - const res = await fetch(url, { signal: ctrl.signal }); + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return (await res.json()) as T; } finally { @@ -87,11 +123,35 @@ export async function fetchJson(url: string, timeoutMs = 1500): Promise { } } +export async function fetchOk( + url: string, + timeoutMs = 1500, + init?: RequestInit, +): Promise { + const ctrl = new AbortController(); + const t = setTimeout(() => ctrl.abort(), timeoutMs); + try { + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + } finally { + clearTimeout(t); + } +} + export async function withCdpSocket( wsUrl: string, fn: (send: CdpSendFn) => Promise, + opts?: { headers?: Record }, ): Promise { - const ws = new WebSocket(wsUrl, { handshakeTimeout: 5000 }); + const headers = getHeadersWithAuth(wsUrl, opts?.headers ?? {}); + const ws = new WebSocket(wsUrl, { + handshakeTimeout: 5000, + ...(Object.keys(headers).length ? { headers } : {}), + }); const { send, closeWithError } = createCdpSender(ws); const openPromise = new Promise((resolve, reject) => { diff --git a/src/browser/cdp.test.ts b/src/browser/cdp.test.ts index a75cf7134..ba3fefdac 100644 --- a/src/browser/cdp.test.ts +++ b/src/browser/cdp.test.ts @@ -165,4 +165,12 @@ describe("cdp", () => { ); expect(normalized).toBe("ws://example.com:9222/devtools/browser/ABC"); }); + + it("propagates auth and query params onto normalized websocket URLs", () => { + const normalized = normalizeCdpWsUrl( + "ws://127.0.0.1:9222/devtools/browser/ABC", + "https://user:pass@example.com?token=abc", + ); + expect(normalized).toBe("wss://user:pass@example.com/devtools/browser/ABC?token=abc"); + }); }); diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index b3d2f2046..7b6ff30a0 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -1,4 +1,11 @@ -import { fetchJson, isLoopbackHost, withCdpSocket } from "./cdp.helpers.js"; +import { + appendCdpPath, + fetchJson, + isLoopbackHost, + withCdpSocket, +} from "./cdp.helpers.js"; + +export { appendCdpPath, fetchJson, fetchOk, getHeadersWithAuth } from "./cdp.helpers.js"; export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string { const ws = new URL(wsUrl); @@ -9,6 +16,13 @@ export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string { if (cdpPort) ws.port = cdpPort; ws.protocol = cdp.protocol === "https:" ? "wss:" : "ws:"; } + if (!ws.username && !ws.password && (cdp.username || cdp.password)) { + ws.username = cdp.username; + ws.password = cdp.password; + } + for (const [key, value] of cdp.searchParams.entries()) { + if (!ws.searchParams.has(key)) ws.searchParams.append(key, value); + } return ws.toString(); } @@ -68,8 +82,10 @@ export async function createTargetViaCdp(opts: { cdpUrl: string; url: string; }): Promise<{ targetId: string }> { - const base = opts.cdpUrl.replace(/\/$/, ""); - const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`${base}/json/version`, 1500); + const version = await fetchJson<{ webSocketDebuggerUrl?: string }>( + appendCdpPath(opts.cdpUrl, "/json/version"), + 1500, + ); const wsUrlRaw = String(version?.webSocketDebuggerUrl ?? "").trim(); const wsUrl = wsUrlRaw ? normalizeCdpWsUrl(wsUrlRaw, opts.cdpUrl) : ""; if (!wsUrl) throw new Error("CDP /json/version missing webSocketDebuggerUrl"); diff --git a/src/browser/chrome.ts b/src/browser/chrome.ts index 6667d18c9..76e2ee00e 100644 --- a/src/browser/chrome.ts +++ b/src/browser/chrome.ts @@ -7,7 +7,8 @@ import WebSocket from "ws"; import { ensurePortAvailable } from "../infra/ports.js"; import { createSubsystemLogger } from "../logging.js"; import { CONFIG_DIR } from "../utils.js"; -import { normalizeCdpWsUrl } from "./cdp.js"; +import { getHeadersWithAuth, normalizeCdpWsUrl } from "./cdp.js"; +import { appendCdpPath } from "./cdp.helpers.js"; import { type BrowserExecutable, resolveBrowserExecutableForPlatform, @@ -71,9 +72,10 @@ async function fetchChromeVersion(cdpUrl: string, timeoutMs = 500): Promise ctrl.abort(), timeoutMs); try { - const base = cdpUrl.replace(/\/$/, ""); - const res = await fetch(`${base}/json/version`, { + const versionUrl = appendCdpPath(cdpUrl, "/json/version"); + const res = await fetch(versionUrl, { signal: ctrl.signal, + headers: getHeadersWithAuth(versionUrl), }); if (!res.ok) return null; const data = (await res.json()) as ChromeVersion; @@ -98,7 +100,11 @@ export async function getChromeWebSocketUrl( async function canOpenWebSocket(wsUrl: string, timeoutMs = 800): Promise { return await new Promise((resolve) => { - const ws = new WebSocket(wsUrl, { handshakeTimeout: timeoutMs }); + const headers = getHeadersWithAuth(wsUrl); + const ws = new WebSocket(wsUrl, { + handshakeTimeout: timeoutMs, + ...(Object.keys(headers).length ? { headers } : {}), + }); const timer = setTimeout( () => { try { diff --git a/src/browser/pw-session.ts b/src/browser/pw-session.ts index 671c6b01f..119e1d315 100644 --- a/src/browser/pw-session.ts +++ b/src/browser/pw-session.ts @@ -8,6 +8,7 @@ import type { } from "playwright-core"; import { chromium } from "playwright-core"; import { formatErrorMessage } from "../infra/errors.js"; +import { getHeadersWithAuth } from "./cdp.helpers.js"; import { getChromeWebSocketUrl } from "./chrome.js"; export type BrowserConsoleMessage = { @@ -266,9 +267,12 @@ async function connectBrowser(cdpUrl: string): Promise { for (let attempt = 0; attempt < 3; attempt += 1) { try { const timeout = 5000 + attempt * 2000; - const wsUrl = await getChromeWebSocketUrl(normalized, timeout).catch(() => null); + const wsUrl = await getChromeWebSocketUrl(normalized, timeout).catch( + () => null, + ); const endpoint = wsUrl ?? normalized; - const browser = await chromium.connectOverCDP(endpoint, { timeout }); + const headers = getHeadersWithAuth(endpoint); + const browser = await chromium.connectOverCDP(endpoint, { timeout, headers }); const connected: ConnectedBrowser = { browser, cdpUrl: normalized }; cached = connected; observeBrowser(browser); diff --git a/src/browser/server-context.ts b/src/browser/server-context.ts index 5ed460d10..a425a49cf 100644 --- a/src/browser/server-context.ts +++ b/src/browser/server-context.ts @@ -1,6 +1,11 @@ import fs from "node:fs"; -import { createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js"; +import { + appendCdpPath, + createTargetViaCdp, + getHeadersWithAuth, + normalizeCdpWsUrl, +} from "./cdp.js"; import { isChromeCdpReady, isChromeReachable, @@ -50,7 +55,11 @@ async function fetchJson(url: string, timeoutMs = 1500, init?: RequestInit): const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { - const res = await fetch(url, { ...init, signal: ctrl.signal }); + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return (await res.json()) as T; } finally { @@ -62,7 +71,11 @@ async function fetchOk(url: string, timeoutMs = 1500, init?: RequestInit): Promi const ctrl = new AbortController(); const t = setTimeout(() => ctrl.abort(), timeoutMs); try { - const res = await fetch(url, { ...init, signal: ctrl.signal }); + const headers = getHeadersWithAuth( + url, + (init?.headers as Record) || {}, + ); + const res = await fetch(url, { ...init, headers, signal: ctrl.signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); } finally { clearTimeout(t); @@ -106,7 +119,7 @@ function createProfileContext( webSocketDebuggerUrl?: string; type?: string; }> - >(`${profile.cdpUrl.replace(/\/$/, "")}/json/list`); + >(appendCdpPath(profile.cdpUrl, "/json/list")); return raw .map((t) => ({ targetId: t.id ?? "", @@ -146,8 +159,13 @@ function createProfileContext( type?: string; }; - const base = profile.cdpUrl.replace(/\/$/, ""); - const endpoint = `${base}/json/new?${encoded}`; + const endpointUrl = new URL(appendCdpPath(profile.cdpUrl, "/json/new")); + const endpoint = endpointUrl.search + ? (() => { + endpointUrl.searchParams.set("url", url); + return endpointUrl.toString(); + })() + : `${endpointUrl.toString()}?${encoded}`; const created = await fetchJson(endpoint, 1500, { method: "PUT", }).catch(async (err) => { @@ -164,7 +182,7 @@ function createProfileContext( targetId: created.id, title: created.title ?? "", url: created.url ?? url, - wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, base), + wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, profile.cdpUrl), type: created.type, }; }; @@ -327,7 +345,6 @@ function createProfileContext( }; const focusTab = async (targetId: string): Promise => { - const base = profile.cdpUrl.replace(/\/$/, ""); const tabs = await listTabs(); const resolved = resolveTargetIdFromTabs(targetId, tabs); if (!resolved.ok) { @@ -336,13 +353,12 @@ function createProfileContext( } throw new Error("tab not found"); } - await fetchOk(`${base}/json/activate/${resolved.targetId}`); + await fetchOk(appendCdpPath(profile.cdpUrl, `/json/activate/${resolved.targetId}`)); const profileState = getProfileState(); profileState.lastTargetId = resolved.targetId; }; const closeTab = async (targetId: string): Promise => { - const base = profile.cdpUrl.replace(/\/$/, ""); const tabs = await listTabs(); const resolved = resolveTargetIdFromTabs(targetId, tabs); if (!resolved.ok) { @@ -351,7 +367,7 @@ function createProfileContext( } throw new Error("tab not found"); } - await fetchOk(`${base}/json/close/${resolved.targetId}`); + await fetchOk(appendCdpPath(profile.cdpUrl, `/json/close/${resolved.targetId}`)); }; const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => { diff --git a/src/browser/server.agent-contract-form-layout-act-commands.test.ts b/src/browser/server.agent-contract-form-layout-act-commands.test.ts index 20ad4ba73..cbff6dca5 100644 --- a/src/browser/server.agent-contract-form-layout-act-commands.test.ts +++ b/src/browser/server.agent-contract-form-layout-act-commands.test.ts @@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.agent-contract-snapshot-endpoints.test.ts b/src/browser/server.agent-contract-snapshot-endpoints.test.ts index 622b6e06a..65a86facb 100644 --- a/src/browser/server.agent-contract-snapshot-endpoints.test.ts +++ b/src/browser/server.agent-contract-snapshot-endpoints.test.ts @@ -129,6 +129,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.covers-additional-endpoint-branches.test.ts b/src/browser/server.covers-additional-endpoint-branches.test.ts index d7771e5e0..bc16040cf 100644 --- a/src/browser/server.covers-additional-endpoint-branches.test.ts +++ b/src/browser/server.covers-additional-endpoint-branches.test.ts @@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts index be1800e7f..4908a536c 100644 --- a/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts +++ b/src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts @@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.serves-status-starts-browser-requested.test.ts b/src/browser/server.serves-status-starts-browser-requested.test.ts index c99698ab7..8cfcca40e 100644 --- a/src/browser/server.serves-status-starts-browser-requested.test.ts +++ b/src/browser/server.serves-status-starts-browser-requested.test.ts @@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks); diff --git a/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts b/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts index a5050298a..d50313413 100644 --- a/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts +++ b/src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts @@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({ createTargetViaCdp: cdpMocks.createTargetViaCdp, normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl), snapshotAria: cdpMocks.snapshotAria, + getHeadersWithAuth: vi.fn(() => ({})), + appendCdpPath: vi.fn((cdpUrl: string, path: string) => { + const base = cdpUrl.replace(/\/$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; + }), })); vi.mock("./pw-ai.js", () => pwMocks);