Merge pull request #895 from mukhtharcm/feat/chrome-browser-improvements

feat(browser): add support for authenticated remote browser profiles

CHANGELOG.md

@@ -37,6 +37,7 @@
 - Browser: extension mode recovers when only one tab is attached (stale targetId fallback).
 - Browser: prefer stable Chrome for auto-detect, with Brave/Edge fallbacks and updated docs. (#983) — thanks @cpojer.
 - Browser: fix `tab not found` for extension relay snapshots/actions when Playwright blocks `newCDPSession` (use the single available Page).
+- Browser: preserve auth/query tokens for remote CDP endpoints and pass Basic auth for CDP HTTP/WS. (#895) — thanks @mukhtharcm.
 - Telegram: add bidirectional reaction support with configurable notifications and agent guidance. (#964) — thanks @bohdanpodvirnyi.
 - Telegram: allow custom commands in the bot menu (merged with native; conflicts ignored). (#860) — thanks @nachoiacovino.
 - Telegram: skip `message_thread_id=1` for General topic sends while keeping typing indicators. (#848) — thanks @azade-c.

docs/tools/browser.md

@@ -154,6 +154,14 @@ Example:
 Use `profiles.<name>.cdpUrl` for **remote CDP** if you want the Gateway to talk
 directly to a Chromium-based browser instance without a remote control server.
 
+Remote CDP URLs can include auth:
+- Query tokens (e.g., `https://provider.example?token=<token>`)
+- HTTP Basic auth (e.g., `https://user:pass@provider.example`)
+
+Clawdbot preserves the auth when calling `/json/*` endpoints and when connecting
+to the CDP WebSocket. Prefer environment variables or secrets managers for
+tokens instead of committing them to config files.
+
 ### Running the control server on the browser machine
 
 Run a standalone browser control server (recommended when your Gateway is remote):

src/browser/cdp.helpers.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+
+import { appendCdpPath, getHeadersWithAuth } from "./cdp.helpers.js";
+
+describe("cdp.helpers", () => {
+  it("preserves query params when appending CDP paths", () => {
+    const url = appendCdpPath("https://example.com?token=abc", "/json/version");
+    expect(url).toBe("https://example.com/json/version?token=abc");
+  });
+
+  it("appends paths under a base prefix", () => {
+    const url = appendCdpPath("https://example.com/chrome/?token=abc", "json/list");
+    expect(url).toBe("https://example.com/chrome/json/list?token=abc");
+  });
+
+  it("adds basic auth headers when credentials are present", () => {
+    const headers = getHeadersWithAuth("https://user:pass@example.com");
+    expect(headers.Authorization).toBe(
+      `Basic ${Buffer.from("user:pass").toString("base64")}`,
+    );
+  });
+
+  it("keeps preexisting authorization headers", () => {
+    const headers = getHeadersWithAuth("https://user:pass@example.com", {
+      Authorization: "Bearer token",
+    });
+    expect(headers.Authorization).toBe("Bearer token");
+  });
+});

src/browser/cdp.helpers.ts

@@ -28,6 +28,34 @@ export function isLoopbackHost(host: string) {
   );
 }
 
+export function getHeadersWithAuth(
+  url: string,
+  headers: Record<string, string> = {},
+) {
+  try {
+    const parsed = new URL(url);
+    const hasAuthHeader = Object.keys(headers).some(
+      (key) => key.toLowerCase() === "authorization",
+    );
+    if (hasAuthHeader) return headers;
+    if (parsed.username || parsed.password) {
+      const auth = Buffer.from(`${parsed.username}:${parsed.password}`).toString("base64");
+      return { ...headers, Authorization: `Basic ${auth}` };
+    }
+  } catch {
+    // ignore
+  }
+  return headers;
+}
+
+export function appendCdpPath(cdpUrl: string, path: string): string {
+  const url = new URL(cdpUrl);
+  const basePath = url.pathname.replace(/\/$/, "");
+  const suffix = path.startsWith("/") ? path : `/${path}`;
+  url.pathname = `${basePath}${suffix}`;
+  return url.toString();
+}
+
 function createCdpSender(ws: WebSocket) {
   let nextId = 1;
   const pending = new Map<number, Pending>();
@@ -75,11 +103,19 @@ function createCdpSender(ws: WebSocket) {
   return { send, closeWithError };
 }
 
-export async function fetchJson<T>(url: string, timeoutMs = 1500): Promise<T> {
+export async function fetchJson<T>(
+  url: string,
+  timeoutMs = 1500,
+  init?: RequestInit,
+): Promise<T> {
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const res = await fetch(url, { signal: ctrl.signal });
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
     return (await res.json()) as T;
   } finally {
@@ -87,11 +123,35 @@ export async function fetchJson<T>(url: string, timeoutMs = 1500): Promise<T> {
   }
 }
 
+export async function fetchOk(
+  url: string,
+  timeoutMs = 1500,
+  init?: RequestInit,
+): Promise<void> {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  } finally {
+    clearTimeout(t);
+  }
+}
+
 export async function withCdpSocket<T>(
   wsUrl: string,
   fn: (send: CdpSendFn) => Promise<T>,
+  opts?: { headers?: Record<string, string> },
 ): Promise<T> {
-  const ws = new WebSocket(wsUrl, { handshakeTimeout: 5000 });
+  const headers = getHeadersWithAuth(wsUrl, opts?.headers ?? {});
+  const ws = new WebSocket(wsUrl, {
+    handshakeTimeout: 5000,
+    ...(Object.keys(headers).length ? { headers } : {}),
+  });
   const { send, closeWithError } = createCdpSender(ws);
 
   const openPromise = new Promise<void>((resolve, reject) => {

src/browser/cdp.test.ts

@@ -165,4 +165,12 @@ describe("cdp", () => {
     );
     expect(normalized).toBe("ws://example.com:9222/devtools/browser/ABC");
   });
+
+  it("propagates auth and query params onto normalized websocket URLs", () => {
+    const normalized = normalizeCdpWsUrl(
+      "ws://127.0.0.1:9222/devtools/browser/ABC",
+      "https://user:pass@example.com?token=abc",
+    );
+    expect(normalized).toBe("wss://user:pass@example.com/devtools/browser/ABC?token=abc");
+  });
 });

src/browser/cdp.ts

@@ -1,4 +1,11 @@
-import { fetchJson, isLoopbackHost, withCdpSocket } from "./cdp.helpers.js";
+import {
+  appendCdpPath,
+  fetchJson,
+  isLoopbackHost,
+  withCdpSocket,
+} from "./cdp.helpers.js";
+
+export { appendCdpPath, fetchJson, fetchOk, getHeadersWithAuth } from "./cdp.helpers.js";
 
 export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string {
   const ws = new URL(wsUrl);
@@ -9,6 +16,13 @@ export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string {
     if (cdpPort) ws.port = cdpPort;
     ws.protocol = cdp.protocol === "https:" ? "wss:" : "ws:";
   }
+  if (!ws.username && !ws.password && (cdp.username || cdp.password)) {
+    ws.username = cdp.username;
+    ws.password = cdp.password;
+  }
+  for (const [key, value] of cdp.searchParams.entries()) {
+    if (!ws.searchParams.has(key)) ws.searchParams.append(key, value);
+  }
   return ws.toString();
 }
 
@@ -68,8 +82,10 @@ export async function createTargetViaCdp(opts: {
   cdpUrl: string;
   url: string;
 }): Promise<{ targetId: string }> {
-  const base = opts.cdpUrl.replace(/\/$/, "");
-  const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(`${base}/json/version`, 1500);
+  const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
+    appendCdpPath(opts.cdpUrl, "/json/version"),
+    1500,
+  );
   const wsUrlRaw = String(version?.webSocketDebuggerUrl ?? "").trim();
   const wsUrl = wsUrlRaw ? normalizeCdpWsUrl(wsUrlRaw, opts.cdpUrl) : "";
   if (!wsUrl) throw new Error("CDP /json/version missing webSocketDebuggerUrl");

src/browser/chrome.ts

@@ -7,7 +7,8 @@ import WebSocket from "ws";
 import { ensurePortAvailable } from "../infra/ports.js";
 import { createSubsystemLogger } from "../logging.js";
 import { CONFIG_DIR } from "../utils.js";
-import { normalizeCdpWsUrl } from "./cdp.js";
+import { getHeadersWithAuth, normalizeCdpWsUrl } from "./cdp.js";
+import { appendCdpPath } from "./cdp.helpers.js";
 import {
   type BrowserExecutable,
   resolveBrowserExecutableForPlatform,
@@ -71,9 +72,10 @@ async function fetchChromeVersion(cdpUrl: string, timeoutMs = 500): Promise<Chro
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const base = cdpUrl.replace(/\/$/, "");
-    const res = await fetch(`${base}/json/version`, {
+    const versionUrl = appendCdpPath(cdpUrl, "/json/version");
+    const res = await fetch(versionUrl, {
       signal: ctrl.signal,
+      headers: getHeadersWithAuth(versionUrl),
     });
     if (!res.ok) return null;
     const data = (await res.json()) as ChromeVersion;
@@ -98,7 +100,11 @@ export async function getChromeWebSocketUrl(
 
 async function canOpenWebSocket(wsUrl: string, timeoutMs = 800): Promise<boolean> {
   return await new Promise<boolean>((resolve) => {
-    const ws = new WebSocket(wsUrl, { handshakeTimeout: timeoutMs });
+    const headers = getHeadersWithAuth(wsUrl);
+    const ws = new WebSocket(wsUrl, {
+      handshakeTimeout: timeoutMs,
+      ...(Object.keys(headers).length ? { headers } : {}),
+    });
     const timer = setTimeout(
       () => {
         try {

src/browser/pw-session.ts

@@ -8,6 +8,7 @@ import type {
 } from "playwright-core";
 import { chromium } from "playwright-core";
 import { formatErrorMessage } from "../infra/errors.js";
+import { getHeadersWithAuth } from "./cdp.helpers.js";
 import { getChromeWebSocketUrl } from "./chrome.js";
 
 export type BrowserConsoleMessage = {
@@ -266,9 +267,12 @@ async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
     for (let attempt = 0; attempt < 3; attempt += 1) {
       try {
         const timeout = 5000 + attempt * 2000;
-        const wsUrl = await getChromeWebSocketUrl(normalized, timeout).catch(() => null);
+        const wsUrl = await getChromeWebSocketUrl(normalized, timeout).catch(
+          () => null,
+        );
         const endpoint = wsUrl ?? normalized;
-        const browser = await chromium.connectOverCDP(endpoint, { timeout });
+        const headers = getHeadersWithAuth(endpoint);
+        const browser = await chromium.connectOverCDP(endpoint, { timeout, headers });
         const connected: ConnectedBrowser = { browser, cdpUrl: normalized };
         cached = connected;
         observeBrowser(browser);

src/browser/server-context.ts

@@ -1,6 +1,11 @@
 import fs from "node:fs";
 
-import { createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js";
+import {
+  appendCdpPath,
+  createTargetViaCdp,
+  getHeadersWithAuth,
+  normalizeCdpWsUrl,
+} from "./cdp.js";
 import {
   isChromeCdpReady,
   isChromeReachable,
@@ -50,7 +55,11 @@ async function fetchJson<T>(url: string, timeoutMs = 1500, init?: RequestInit):
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const res = await fetch(url, { ...init, signal: ctrl.signal });
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
     return (await res.json()) as T;
   } finally {
@@ -62,7 +71,11 @@ async function fetchOk(url: string, timeoutMs = 1500, init?: RequestInit): Promi
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const res = await fetch(url, { ...init, signal: ctrl.signal });
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
   } finally {
     clearTimeout(t);
@@ -106,7 +119,7 @@ function createProfileContext(
         webSocketDebuggerUrl?: string;
         type?: string;
       }>
-    >(`${profile.cdpUrl.replace(/\/$/, "")}/json/list`);
+    >(appendCdpPath(profile.cdpUrl, "/json/list"));
     return raw
       .map((t) => ({
         targetId: t.id ?? "",
@@ -146,8 +159,13 @@ function createProfileContext(
       type?: string;
     };
 
-    const base = profile.cdpUrl.replace(/\/$/, "");
-    const endpoint = `${base}/json/new?${encoded}`;
+    const endpointUrl = new URL(appendCdpPath(profile.cdpUrl, "/json/new"));
+    const endpoint = endpointUrl.search
+      ? (() => {
+          endpointUrl.searchParams.set("url", url);
+          return endpointUrl.toString();
+        })()
+      : `${endpointUrl.toString()}?${encoded}`;
     const created = await fetchJson<CdpTarget>(endpoint, 1500, {
       method: "PUT",
     }).catch(async (err) => {
@@ -164,7 +182,7 @@ function createProfileContext(
       targetId: created.id,
       title: created.title ?? "",
       url: created.url ?? url,
-      wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, base),
+      wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, profile.cdpUrl),
       type: created.type,
     };
   };
@@ -327,7 +345,6 @@ function createProfileContext(
   };
 
   const focusTab = async (targetId: string): Promise<void> => {
-    const base = profile.cdpUrl.replace(/\/$/, "");
     const tabs = await listTabs();
     const resolved = resolveTargetIdFromTabs(targetId, tabs);
     if (!resolved.ok) {
@@ -336,13 +353,12 @@ function createProfileContext(
       }
       throw new Error("tab not found");
     }
-    await fetchOk(`${base}/json/activate/${resolved.targetId}`);
+    await fetchOk(appendCdpPath(profile.cdpUrl, `/json/activate/${resolved.targetId}`));
     const profileState = getProfileState();
     profileState.lastTargetId = resolved.targetId;
   };
 
   const closeTab = async (targetId: string): Promise<void> => {
-    const base = profile.cdpUrl.replace(/\/$/, "");
     const tabs = await listTabs();
     const resolved = resolveTargetIdFromTabs(targetId, tabs);
     if (!resolved.ok) {
@@ -351,7 +367,7 @@ function createProfileContext(
       }
       throw new Error("tab not found");
     }
-    await fetchOk(`${base}/json/close/${resolved.targetId}`);
+    await fetchOk(appendCdpPath(profile.cdpUrl, `/json/close/${resolved.targetId}`));
   };
 
   const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => {

src/browser/server.agent-contract-form-layout-act-commands.test.ts

@@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.agent-contract-snapshot-endpoints.test.ts

@@ -129,6 +129,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.covers-additional-endpoint-branches.test.ts

@@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts

@@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.serves-status-starts-browser-requested.test.ts

@@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts

@@ -128,6 +128,12 @@ vi.mock("./cdp.js", () => ({
   createTargetViaCdp: cdpMocks.createTargetViaCdp,
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
+  getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);