fix: support authenticated remote CDP URLs (#895) (thanks @mukhtharcm)

CHANGELOG.md

@@ -36,6 +36,7 @@
 - Browser: extension mode recovers when only one tab is attached (stale targetId fallback).
 - Browser: prefer stable Chrome for auto-detect, with Brave/Edge fallbacks and updated docs. (#983) — thanks @cpojer.
 - Browser: fix `tab not found` for extension relay snapshots/actions when Playwright blocks `newCDPSession` (use the single available Page).
+- Browser: preserve auth/query tokens for remote CDP endpoints and pass Basic auth for CDP HTTP/WS. (#895) — thanks @mukhtharcm.
 - Telegram: add bidirectional reaction support with configurable notifications and agent guidance. (#964) — thanks @bohdanpodvirnyi.
 - Telegram: skip `message_thread_id=1` for General topic sends while keeping typing indicators. (#848) — thanks @azade-c.
 - Discord: allow allowlisted guilds without channel lists to receive messages when `groupPolicy="allowlist"`. — thanks @thewilloftheshadow.

docs/tools/browser.md

@@ -154,6 +154,14 @@ Example:
 Use `profiles.<name>.cdpUrl` for **remote CDP** if you want the Gateway to talk
 directly to a Chromium-based browser instance without a remote control server.
 
+Remote CDP URLs can include auth:
+- Query tokens (e.g., `https://provider.example?token=<token>`)
+- HTTP Basic auth (e.g., `https://user:pass@provider.example`)
+
+Clawdbot preserves the auth when calling `/json/*` endpoints and when connecting
+to the CDP WebSocket. Prefer environment variables or secrets managers for
+tokens instead of committing them to config files.
+
 ### Running the control server on the browser machine
 
 Run a standalone browser control server (recommended when your Gateway is remote):

src/browser/cdp.helpers.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+
+import { appendCdpPath, getHeadersWithAuth } from "./cdp.helpers.js";
+
+describe("cdp.helpers", () => {
+  it("preserves query params when appending CDP paths", () => {
+    const url = appendCdpPath("https://example.com?token=abc", "/json/version");
+    expect(url).toBe("https://example.com/json/version?token=abc");
+  });
+
+  it("appends paths under a base prefix", () => {
+    const url = appendCdpPath("https://example.com/chrome/?token=abc", "json/list");
+    expect(url).toBe("https://example.com/chrome/json/list?token=abc");
+  });
+
+  it("adds basic auth headers when credentials are present", () => {
+    const headers = getHeadersWithAuth("https://user:pass@example.com");
+    expect(headers.Authorization).toBe(
+      `Basic ${Buffer.from("user:pass").toString("base64")}`,
+    );
+  });
+
+  it("keeps preexisting authorization headers", () => {
+    const headers = getHeadersWithAuth("https://user:pass@example.com", {
+      Authorization: "Bearer token",
+    });
+    expect(headers.Authorization).toBe("Bearer token");
+  });
+});

src/browser/cdp.helpers.ts

@@ -28,6 +28,34 @@ export function isLoopbackHost(host: string) {
   );
 }
 
+export function getHeadersWithAuth(
+  url: string,
+  headers: Record<string, string> = {},
+) {
+  try {
+    const parsed = new URL(url);
+    const hasAuthHeader = Object.keys(headers).some(
+      (key) => key.toLowerCase() === "authorization",
+    );
+    if (hasAuthHeader) return headers;
+    if (parsed.username || parsed.password) {
+      const auth = Buffer.from(`${parsed.username}:${parsed.password}`).toString("base64");
+      return { ...headers, Authorization: `Basic ${auth}` };
+    }
+  } catch {
+    // ignore
+  }
+  return headers;
+}
+
+export function appendCdpPath(cdpUrl: string, path: string): string {
+  const url = new URL(cdpUrl);
+  const basePath = url.pathname.replace(/\/$/, "");
+  const suffix = path.startsWith("/") ? path : `/${path}`;
+  url.pathname = `${basePath}${suffix}`;
+  return url.toString();
+}
+
 function createCdpSender(ws: WebSocket) {
   let nextId = 1;
   const pending = new Map<number, Pending>();
@@ -75,11 +103,19 @@ function createCdpSender(ws: WebSocket) {
   return { send, closeWithError };
 }
 
-export async function fetchJson<T>(url: string, timeoutMs = 1500): Promise<T> {
+export async function fetchJson<T>(
+  url: string,
+  timeoutMs = 1500,
+  init?: RequestInit,
+): Promise<T> {
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const res = await fetch(url, { signal: ctrl.signal });
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
     return (await res.json()) as T;
   } finally {
@@ -87,11 +123,35 @@ export async function fetchJson<T>(url: string, timeoutMs = 1500): Promise<T> {
   }
 }
 
+export async function fetchOk(
+  url: string,
+  timeoutMs = 1500,
+  init?: RequestInit,
+): Promise<void> {
+  const ctrl = new AbortController();
+  const t = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const headers = getHeadersWithAuth(
+      url,
+      (init?.headers as Record<string, string>) || {},
+    );
+    const res = await fetch(url, { ...init, headers, signal: ctrl.signal });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  } finally {
+    clearTimeout(t);
+  }
+}
+
 export async function withCdpSocket<T>(
   wsUrl: string,
   fn: (send: CdpSendFn) => Promise<T>,
+  opts?: { headers?: Record<string, string> },
 ): Promise<T> {
-  const ws = new WebSocket(wsUrl, { handshakeTimeout: 5000 });
+  const headers = getHeadersWithAuth(wsUrl, opts?.headers ?? {});
+  const ws = new WebSocket(wsUrl, {
+    handshakeTimeout: 5000,
+    ...(Object.keys(headers).length ? { headers } : {}),
+  });
   const { send, closeWithError } = createCdpSender(ws);
 
   const openPromise = new Promise<void>((resolve, reject) => {

src/browser/cdp.test.ts

@@ -165,4 +165,12 @@ describe("cdp", () => {
     );
     expect(normalized).toBe("ws://example.com:9222/devtools/browser/ABC");
   });
+
+  it("propagates auth and query params onto normalized websocket URLs", () => {
+    const normalized = normalizeCdpWsUrl(
+      "ws://127.0.0.1:9222/devtools/browser/ABC",
+      "https://user:pass@example.com?token=abc",
+    );
+    expect(normalized).toBe("wss://user:pass@example.com/devtools/browser/ABC?token=abc");
+  });
 });

src/browser/cdp.ts

@@ -1,33 +1,11 @@
-import { isLoopbackHost, withCdpSocket } from "./cdp.helpers.js";
+import {
+  appendCdpPath,
+  fetchJson,
+  isLoopbackHost,
+  withCdpSocket,
+} from "./cdp.helpers.js";
 
-export function getHeadersWithAuth(
+export { appendCdpPath, fetchJson, fetchOk, getHeadersWithAuth } from "./cdp.helpers.js";
-  url: string,
-  headers: Record<string, string> = {},
-) {
-  try {
-    const parsed = new URL(url);
-    if (parsed.username || parsed.password) {
-      const auth = Buffer.from(
-        `${parsed.username}:${parsed.password}`,
-      ).toString("base64");
-      return { ...headers, Authorization: `Basic ${auth}` };
-    }
-  } catch (_e) {
-    // ignore
-  }
-  return headers;
-}
-
-export async function fetchJson<T>(url: string): Promise<T> {
-  const res = await fetch(url, { headers: getHeadersWithAuth(url) });
-  if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
-  return (await res.json()) as Promise<T>;
-}
-
-export async function fetchOk(url: string): Promise<void> {
-  const res = await fetch(url, { headers: getHeadersWithAuth(url) });
-  if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
-}
 
 export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string {
   const ws = new URL(wsUrl);
@@ -38,6 +16,13 @@ export function normalizeCdpWsUrl(wsUrl: string, cdpUrl: string): string {
     if (cdpPort) ws.port = cdpPort;
     ws.protocol = cdp.protocol === "https:" ? "wss:" : "ws:";
   }
+  if (!ws.username && !ws.password && (cdp.username || cdp.password)) {
+    ws.username = cdp.username;
+    ws.password = cdp.password;
+  }
+  for (const [key, value] of cdp.searchParams.entries()) {
+    if (!ws.searchParams.has(key)) ws.searchParams.append(key, value);
+  }
   return ws.toString();
 }
 
@@ -97,9 +82,9 @@ export async function createTargetViaCdp(opts: {
   cdpUrl: string;
   url: string;
 }): Promise<{ targetId: string }> {
-  const base = opts.cdpUrl.replace(/\/$/, "");
   const version = await fetchJson<{ webSocketDebuggerUrl?: string }>(
-    `${base}/json/version`,
+    appendCdpPath(opts.cdpUrl, "/json/version"),
+    1500,
   );
   const wsUrlRaw = String(version?.webSocketDebuggerUrl ?? "").trim();
   const wsUrl = wsUrlRaw ? normalizeCdpWsUrl(wsUrlRaw, opts.cdpUrl) : "";

src/browser/chrome.ts

@@ -8,6 +8,7 @@ import { ensurePortAvailable } from "../infra/ports.js";
 import { createSubsystemLogger } from "../logging.js";
 import { CONFIG_DIR } from "../utils.js";
 import { getHeadersWithAuth, normalizeCdpWsUrl } from "./cdp.js";
+import { appendCdpPath } from "./cdp.helpers.js";
 import {
   type BrowserExecutable,
   resolveBrowserExecutableForPlatform,
@@ -71,10 +72,10 @@ async function fetchChromeVersion(cdpUrl: string, timeoutMs = 500): Promise<Chro
   const ctrl = new AbortController();
   const t = setTimeout(() => ctrl.abort(), timeoutMs);
   try {
-    const base = cdpUrl.replace(/\/$/, "");
+    const versionUrl = appendCdpPath(cdpUrl, "/json/version");
-    const res = await fetch(`${base}/json/version`, {
+    const res = await fetch(versionUrl, {
       signal: ctrl.signal,
-      headers: getHeadersWithAuth(`${base}/json/version`),
+      headers: getHeadersWithAuth(versionUrl),
     });
     if (!res.ok) return null;
     const data = (await res.json()) as ChromeVersion;
@@ -99,7 +100,11 @@ export async function getChromeWebSocketUrl(
 
 async function canOpenWebSocket(wsUrl: string, timeoutMs = 800): Promise<boolean> {
   return await new Promise<boolean>((resolve) => {
-    const ws = new WebSocket(wsUrl, { handshakeTimeout: timeoutMs });
+    const headers = getHeadersWithAuth(wsUrl);
+    const ws = new WebSocket(wsUrl, {
+      handshakeTimeout: timeoutMs,
+      ...(Object.keys(headers).length ? { headers } : {}),
+    });
     const timer = setTimeout(
       () => {
         try {

src/browser/pw-session.ts

@@ -8,7 +8,7 @@ import type {
 } from "playwright-core";
 import { chromium } from "playwright-core";
 import { formatErrorMessage } from "../infra/errors.js";
-import { isLoopbackHost } from "./cdp.helpers.js";
+import { getHeadersWithAuth } from "./cdp.helpers.js";
 import { getChromeWebSocketUrl } from "./chrome.js";
 
 export type BrowserConsoleMessage = {
@@ -257,18 +257,6 @@ function observeBrowser(browser: Browser) {
   for (const context of browser.contexts()) observeContext(context);
 }
 
-export async function getConnectedBrowser(cdpUrl: string): Promise<Browser> {
-  const { browser } = await connectBrowser(cdpUrl);
-  return browser;
-}
-
-function getWsEndpoint(cdpUrl: string): string {
-  const u = new URL(cdpUrl);
-  // Ensure we use wss/ws matching the protocol
-  u.protocol = u.protocol === "https:" ? "wss:" : "ws:";
-  return u.toString();
-}
-
 async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
   const normalized = normalizeCdpUrl(cdpUrl);
   if (cached?.cdpUrl === normalized) return cached;
@@ -283,7 +271,8 @@ async function connectBrowser(cdpUrl: string): Promise<ConnectedBrowser> {
           () => null,
         );
         const endpoint = wsUrl ?? normalized;
-        const browser = await chromium.connectOverCDP(endpoint, { timeout });
+        const headers = getHeadersWithAuth(endpoint);
+        const browser = await chromium.connectOverCDP(endpoint, { timeout, headers });
         const connected: ConnectedBrowser = { browser, cdpUrl: normalized };
         cached = connected;
         observeBrowser(browser);

src/browser/server-context.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 
 import {
+  appendCdpPath,
   createTargetViaCdp,
   getHeadersWithAuth,
   normalizeCdpWsUrl,
@@ -14,7 +15,6 @@ import {
 } from "./chrome.js";
 import type { ResolvedBrowserProfile } from "./config.js";
 import { resolveProfile } from "./config.js";
-import { getConnectedBrowser } from "./pw-session.js";
 import type {
   BrowserRouteContext,
   BrowserTab,
@@ -110,40 +110,7 @@ function createProfileContext(
     profileState.running = running;
   };
 
-  const listTabsViaPlaywright = async (): Promise<BrowserTab[]> => {
-    const browser = await getConnectedBrowser(profile.cdpUrl);
-    const contexts = browser.contexts();
-    const pages = contexts.flatMap((c) => c.pages());
-    // Note: Playwright pages don't sync title instantly, returning URL is safer for list
-    return pages.map((p) => ({
-      targetId: p.url(),
-      title: "",
-      url: p.url(),
-      type: "page",
-    }));
-  };
-
-  const openTabViaPlaywright = async (url?: string): Promise<BrowserTab> => {
-    const browser = await getConnectedBrowser(profile.cdpUrl);
-    // Reuse context or create new
-    const context = browser.contexts()[0] || (await browser.newContext());
-    const page = await context.newPage();
-    if (url) await page.goto(url);
-    return { targetId: page.url(), title: "", url: page.url(), type: "page" };
-  };
-
-  const closeTabViaPlaywright = async (targetId: string) => {
-    const browser = await getConnectedBrowser(profile.cdpUrl);
-    const pages = browser.contexts().flatMap((c) => c.pages());
-    // Find page by URL match (simple strategy)
-    const page = pages.find((p) => p.url() === targetId);
-    if (page) await page.close();
-  };
-
   const listTabs = async (): Promise<BrowserTab[]> => {
-    if (!profile.cdpIsLoopback) {
-      return await listTabsViaPlaywright();
-    }
     const raw = await fetchJson<
       Array<{
         id?: string;
@@ -152,7 +119,7 @@ function createProfileContext(
         webSocketDebuggerUrl?: string;
         type?: string;
       }>
-    >(`${profile.cdpUrl.replace(/\/$/, "")}/json/list`);
+    >(appendCdpPath(profile.cdpUrl, "/json/list"));
     return raw
       .map((t) => ({
         targetId: t.id ?? "",
@@ -165,9 +132,6 @@ function createProfileContext(
   };
 
   const openTab = async (url: string): Promise<BrowserTab> => {
-    if (!profile.cdpIsLoopback) {
-      return await openTabViaPlaywright(url);
-    }
     const createdViaCdp = await createTargetViaCdp({
       cdpUrl: profile.cdpUrl,
       url,
@@ -195,8 +159,13 @@ function createProfileContext(
       type?: string;
     };
 
-    const base = profile.cdpUrl.replace(/\/$/, "");
+    const endpointUrl = new URL(appendCdpPath(profile.cdpUrl, "/json/new"));
-    const endpoint = `${base}/json/new?${encoded}`;
+    const endpoint = endpointUrl.search
+      ? (() => {
+          endpointUrl.searchParams.set("url", url);
+          return endpointUrl.toString();
+        })()
+      : `${endpointUrl.toString()}?${encoded}`;
     const created = await fetchJson<CdpTarget>(endpoint, 1500, {
       method: "PUT",
     }).catch(async (err) => {
@@ -213,7 +182,7 @@ function createProfileContext(
       targetId: created.id,
       title: created.title ?? "",
       url: created.url ?? url,
-      wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, base),
+      wsUrl: normalizeWsUrl(created.webSocketDebuggerUrl, profile.cdpUrl),
       type: created.type,
     };
   };
@@ -376,7 +345,6 @@ function createProfileContext(
   };
 
   const focusTab = async (targetId: string): Promise<void> => {
-    const base = profile.cdpUrl.replace(/\/$/, "");
     const tabs = await listTabs();
     const resolved = resolveTargetIdFromTabs(targetId, tabs);
     if (!resolved.ok) {
@@ -385,16 +353,12 @@ function createProfileContext(
       }
       throw new Error("tab not found");
     }
-    await fetchOk(`${base}/json/activate/${resolved.targetId}`);
+    await fetchOk(appendCdpPath(profile.cdpUrl, `/json/activate/${resolved.targetId}`));
     const profileState = getProfileState();
     profileState.lastTargetId = resolved.targetId;
   };
 
   const closeTab = async (targetId: string): Promise<void> => {
-    if (!profile.cdpIsLoopback) {
-      return await closeTabViaPlaywright(targetId);
-    }
-    const base = profile.cdpUrl.replace(/\/$/, "");
     const tabs = await listTabs();
     const resolved = resolveTargetIdFromTabs(targetId, tabs);
     if (!resolved.ok) {
@@ -403,7 +367,7 @@ function createProfileContext(
       }
       throw new Error("tab not found");
     }
-    await fetchOk(`${base}/json/close/${resolved.targetId}`);
+    await fetchOk(appendCdpPath(profile.cdpUrl, `/json/close/${resolved.targetId}`));
   };
 
   const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => {

src/browser/server.agent-contract-form-layout-act-commands.test.ts

@@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.agent-contract-snapshot-endpoints.test.ts

@@ -130,6 +130,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.covers-additional-endpoint-branches.test.ts

@@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts

@@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.serves-status-starts-browser-requested.test.ts

@@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);

src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts

@@ -129,6 +129,11 @@ vi.mock("./cdp.js", () => ({
   normalizeCdpWsUrl: vi.fn((wsUrl: string) => wsUrl),
   snapshotAria: cdpMocks.snapshotAria,
   getHeadersWithAuth: vi.fn(() => ({})),
+  appendCdpPath: vi.fn((cdpUrl: string, path: string) => {
+    const base = cdpUrl.replace(/\/$/, "");
+    const suffix = path.startsWith("/") ? path : `/${path}`;
+    return `${base}${suffix}`;
+  }),
 }));
 
 vi.mock("./pw-ai.js", () => pwMocks);