feat(browser): expand browser control surface

src/browser/pw-tools-core.ts

@@ -1,11 +1,17 @@
+import type { CDPSession, Page } from "playwright-core";
+import { devices as playwrightDevices } from "playwright-core";
 import type { BrowserFormField } from "./client-actions-core.js";
 import {
   buildRoleSnapshotFromAriaSnapshot,
+  getRoleSnapshotStats,
   parseRoleRef,
   type RoleSnapshotOptions,
 } from "./pw-role-snapshot.js";
 import {
   type BrowserConsoleMessage,
+  type BrowserNetworkRequest,
+  type BrowserPageError,
+  ensureContextState,
   ensurePageState,
   getPageForTargetId,
   refLocator,
@@ -23,6 +29,42 @@ function requireRef(value: unknown): string {
   return ref;
 }
 
+function toAIFriendlyError(error: unknown, selector: string): Error {
+  const message = error instanceof Error ? error.message : String(error);
+
+  if (message.includes("strict mode violation")) {
+    const countMatch = message.match(/resolved to (\d+) elements/);
+    const count = countMatch ? countMatch[1] : "multiple";
+    return new Error(
+      `Selector "${selector}" matched ${count} elements. ` +
+        `Run a new snapshot to get updated refs, or use a different ref.`,
+    );
+  }
+
+  if (
+    (message.includes("Timeout") || message.includes("waiting for")) &&
+    (message.includes("to be visible") || message.includes("not visible"))
+  ) {
+    return new Error(
+      `Element "${selector}" not found or not visible. ` +
+        `Run a new snapshot to see current page elements.`,
+    );
+  }
+
+  if (
+    message.includes("intercepts pointer events") ||
+    message.includes("not visible") ||
+    message.includes("not receive pointer events")
+  ) {
+    return new Error(
+      `Element "${selector}" is not interactable (hidden or covered). ` +
+        `Try scrolling it into view, closing overlays, or re-snapshotting.`,
+    );
+  }
+
+  return error instanceof Error ? error : new Error(message);
+}
+
 export async function snapshotAiViaPlaywright(opts: {
   cdpUrl: string;
   targetId?: string;
@@ -66,17 +108,28 @@ export async function snapshotRoleViaPlaywright(opts: {
   cdpUrl: string;
   targetId?: string;
   selector?: string;
+  frameSelector?: string;
   options?: RoleSnapshotOptions;
-}): Promise<{ snapshot: string }> {
+}): Promise<{
+  snapshot: string;
+  refs: Record<string, { role: string; name?: string; nth?: number }>;
+  stats: { lines: number; chars: number; refs: number; interactive: number };
+}> {
   const page = await getPageForTargetId({
     cdpUrl: opts.cdpUrl,
     targetId: opts.targetId,
   });
   const state = ensurePageState(page);
 
-  const locator = opts.selector?.trim()
-    ? page.locator(opts.selector.trim())
-    : page.locator(":root");
+  const frameSelector = opts.frameSelector?.trim() || "";
+  const selector = opts.selector?.trim() || "";
+  const locator = frameSelector
+    ? selector
+      ? page.frameLocator(frameSelector).locator(selector)
+      : page.frameLocator(frameSelector).locator(":root")
+    : selector
+      ? page.locator(selector)
+      : page.locator(":root");
 
   const ariaSnapshot = await locator.ariaSnapshot();
   const built = buildRoleSnapshotFromAriaSnapshot(
@@ -84,7 +137,95 @@ export async function snapshotRoleViaPlaywright(opts: {
     opts.options,
   );
   state.roleRefs = built.refs;
-  return { snapshot: built.snapshot };
+  state.roleRefsFrameSelector = frameSelector || undefined;
+  return {
+    snapshot: built.snapshot,
+    refs: built.refs,
+    stats: getRoleSnapshotStats(built.snapshot, built.refs),
+  };
+}
+
+export async function getPageErrorsViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  clear?: boolean;
+}): Promise<{ errors: BrowserPageError[] }> {
+  const page = await getPageForTargetId(opts);
+  const state = ensurePageState(page);
+  const errors = [...state.errors];
+  if (opts.clear) state.errors = [];
+  return { errors };
+}
+
+export async function getNetworkRequestsViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  filter?: string;
+  clear?: boolean;
+}): Promise<{ requests: BrowserNetworkRequest[] }> {
+  const page = await getPageForTargetId(opts);
+  const state = ensurePageState(page);
+  const raw = [...state.requests];
+  const filter = typeof opts.filter === "string" ? opts.filter.trim() : "";
+  const requests = filter ? raw.filter((r) => r.url.includes(filter)) : raw;
+  if (opts.clear) {
+    state.requests = [];
+    state.requestIds = new WeakMap();
+  }
+  return { requests };
+}
+
+export async function highlightViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  ref: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const ref = requireRef(opts.ref);
+  try {
+    await refLocator(page, ref).highlight();
+  } catch (err) {
+    throw toAIFriendlyError(err, ref);
+  }
+}
+
+export async function traceStartViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  screenshots?: boolean;
+  snapshots?: boolean;
+  sources?: boolean;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  const context = page.context();
+  const ctxState = ensureContextState(context);
+  if (ctxState.traceActive) {
+    throw new Error(
+      "Trace already running. Stop the current trace before starting a new one.",
+    );
+  }
+  await context.tracing.start({
+    screenshots: opts.screenshots ?? true,
+    snapshots: opts.snapshots ?? true,
+    sources: opts.sources ?? false,
+  });
+  ctxState.traceActive = true;
+}
+
+export async function traceStopViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  path: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  const context = page.context();
+  const ctxState = ensureContextState(context);
+  if (!ctxState.traceActive) {
+    throw new Error("No active trace. Start a trace before stopping it.");
+  }
+  await context.tracing.stop({ path: opts.path });
+  ctxState.traceActive = false;
 }
 
 export async function clickViaPlaywright(opts: {
@@ -101,23 +242,28 @@ export async function clickViaPlaywright(opts: {
     targetId: opts.targetId,
   });
   ensurePageState(page);
-  const locator = refLocator(page, requireRef(opts.ref));
+  const ref = requireRef(opts.ref);
+  const locator = refLocator(page, ref);
   const timeout = Math.max(
     500,
     Math.min(60_000, Math.floor(opts.timeoutMs ?? 8000)),
   );
-  if (opts.doubleClick) {
-    await locator.dblclick({
-      timeout,
-      button: opts.button,
-      modifiers: opts.modifiers,
-    });
-  } else {
-    await locator.click({
-      timeout,
-      button: opts.button,
-      modifiers: opts.modifiers,
-    });
+  try {
+    if (opts.doubleClick) {
+      await locator.dblclick({
+        timeout,
+        button: opts.button,
+        modifiers: opts.modifiers,
+      });
+    } else {
+      await locator.click({
+        timeout,
+        button: opts.button,
+        modifiers: opts.modifiers,
+      });
+    }
+  } catch (err) {
+    throw toAIFriendlyError(err, ref);
   }
 }
 
@@ -130,9 +276,13 @@ export async function hoverViaPlaywright(opts: {
   const ref = requireRef(opts.ref);
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
-  await refLocator(page, ref).hover({
-    timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
-  });
+  try {
+    await refLocator(page, ref).hover({
+      timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
+    });
+  } catch (err) {
+    throw toAIFriendlyError(err, ref);
+  }
 }
 
 export async function dragViaPlaywright(opts: {
@@ -147,9 +297,13 @@ export async function dragViaPlaywright(opts: {
   if (!startRef || !endRef) throw new Error("startRef and endRef are required");
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
-  await refLocator(page, startRef).dragTo(refLocator(page, endRef), {
-    timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
-  });
+  try {
+    await refLocator(page, startRef).dragTo(refLocator(page, endRef), {
+      timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
+    });
+  } catch (err) {
+    throw toAIFriendlyError(err, `${startRef} -> ${endRef}`);
+  }
 }
 
 export async function selectOptionViaPlaywright(opts: {
@@ -163,9 +317,13 @@ export async function selectOptionViaPlaywright(opts: {
   if (!opts.values?.length) throw new Error("values are required");
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
-  await refLocator(page, ref).selectOption(opts.values, {
-    timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
-  });
+  try {
+    await refLocator(page, ref).selectOption(opts.values, {
+      timeout: Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000)),
+    });
+  } catch (err) {
+    throw toAIFriendlyError(err, ref);
+  }
 }
 
 export async function pressKeyViaPlaywright(opts: {
@@ -183,6 +341,330 @@ export async function pressKeyViaPlaywright(opts: {
   });
 }
 
+export async function cookiesGetViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+}): Promise<{ cookies: unknown[] }> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const cookies = await page.context().cookies();
+  return { cookies };
+}
+
+export async function cookiesSetViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  cookie: {
+    name: string;
+    value: string;
+    url?: string;
+    domain?: string;
+    path?: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: "Lax" | "None" | "Strict";
+  };
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const cookie = opts.cookie;
+  if (!cookie.name || cookie.value === undefined) {
+    throw new Error("cookie name and value are required");
+  }
+  const hasUrl = typeof cookie.url === "string" && cookie.url.trim();
+  const hasDomainPath =
+    typeof cookie.domain === "string" &&
+    cookie.domain.trim() &&
+    typeof cookie.path === "string" &&
+    cookie.path.trim();
+  if (!hasUrl && !hasDomainPath) {
+    throw new Error("cookie requires url, or domain+path");
+  }
+  await page.context().addCookies([cookie]);
+}
+
+export async function cookiesClearViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  await page.context().clearCookies();
+}
+
+type StorageKind = "local" | "session";
+
+export async function storageGetViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  kind: StorageKind;
+  key?: string;
+}): Promise<{ values: Record<string, string> }> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const kind = opts.kind;
+  const key = typeof opts.key === "string" ? opts.key : undefined;
+  const values = await page.evaluate(
+    ({ kind: kind2, key: key2 }) => {
+      const store =
+        kind2 === "session" ? window.sessionStorage : window.localStorage;
+      if (key2) {
+        const value = store.getItem(key2);
+        return value === null ? {} : { [key2]: value };
+      }
+      const out: Record<string, string> = {};
+      for (let i = 0; i < store.length; i += 1) {
+        const k = store.key(i);
+        if (!k) continue;
+        const v = store.getItem(k);
+        if (v !== null) out[k] = v;
+      }
+      return out;
+    },
+    { kind, key },
+  );
+  return { values: values ?? {} };
+}
+
+export async function storageSetViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  kind: StorageKind;
+  key: string;
+  value: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const key = String(opts.key ?? "");
+  if (!key) throw new Error("key is required");
+  await page.evaluate(
+    ({ kind, key: k, value }) => {
+      const store =
+        kind === "session" ? window.sessionStorage : window.localStorage;
+      store.setItem(k, value);
+    },
+    { kind: opts.kind, key, value: String(opts.value ?? "") },
+  );
+}
+
+export async function storageClearViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  kind: StorageKind;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  await page.evaluate(
+    ({ kind }) => {
+      const store =
+        kind === "session" ? window.sessionStorage : window.localStorage;
+      store.clear();
+    },
+    { kind: opts.kind },
+  );
+}
+
+export async function setOfflineViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  offline: boolean;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  await page.context().setOffline(Boolean(opts.offline));
+}
+
+export async function setExtraHTTPHeadersViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  headers: Record<string, string>;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  await page.context().setExtraHTTPHeaders(opts.headers);
+}
+
+export async function setHttpCredentialsViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  username?: string;
+  password?: string;
+  clear?: boolean;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  if (opts.clear) {
+    await page.context().setHTTPCredentials(null);
+    return;
+  }
+  const username = String(opts.username ?? "");
+  const password = String(opts.password ?? "");
+  if (!username) throw new Error("username is required (or set clear=true)");
+  await page.context().setHTTPCredentials({ username, password });
+}
+
+export async function setGeolocationViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  latitude?: number;
+  longitude?: number;
+  accuracy?: number;
+  origin?: string;
+  clear?: boolean;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const context = page.context();
+  if (opts.clear) {
+    await context.setGeolocation(null);
+    await context.clearPermissions().catch(() => {});
+    return;
+  }
+  if (typeof opts.latitude !== "number" || typeof opts.longitude !== "number") {
+    throw new Error("latitude and longitude are required (or set clear=true)");
+  }
+  await context.setGeolocation({
+    latitude: opts.latitude,
+    longitude: opts.longitude,
+    accuracy: typeof opts.accuracy === "number" ? opts.accuracy : undefined,
+  });
+  const origin =
+    opts.origin?.trim() ||
+    (() => {
+      try {
+        return new URL(page.url()).origin;
+      } catch {
+        return "";
+      }
+    })();
+  if (origin) {
+    await context.grantPermissions(["geolocation"], { origin }).catch(() => {});
+  }
+}
+
+export async function emulateMediaViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  colorScheme: "dark" | "light" | "no-preference" | null;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  await page.emulateMedia({ colorScheme: opts.colorScheme });
+}
+
+async function withCdpSession<T>(
+  page: Page,
+  fn: (session: CDPSession) => Promise<T>,
+): Promise<T> {
+  const session = await page.context().newCDPSession(page);
+  try {
+    return await fn(session);
+  } finally {
+    await session.detach().catch(() => {});
+  }
+}
+
+export async function setLocaleViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  locale: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const locale = String(opts.locale ?? "").trim();
+  if (!locale) throw new Error("locale is required");
+  await withCdpSession(page, async (session) => {
+    try {
+      await session.send("Emulation.setLocaleOverride", { locale });
+    } catch (err) {
+      if (
+        String(err).includes("Another locale override is already in effect")
+      ) {
+        return;
+      }
+      throw err;
+    }
+  });
+}
+
+export async function setTimezoneViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  timezoneId: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const timezoneId = String(opts.timezoneId ?? "").trim();
+  if (!timezoneId) throw new Error("timezoneId is required");
+  await withCdpSession(page, async (session) => {
+    try {
+      await session.send("Emulation.setTimezoneOverride", { timezoneId });
+    } catch (err) {
+      const msg = String(err);
+      if (msg.includes("Timezone override is already in effect")) return;
+      if (msg.includes("Invalid timezone"))
+        throw new Error(`Invalid timezone ID: ${timezoneId}`);
+      throw err;
+    }
+  });
+}
+
+export async function setDeviceViaPlaywright(opts: {
+  cdpUrl: string;
+  targetId?: string;
+  name: string;
+}): Promise<void> {
+  const page = await getPageForTargetId(opts);
+  ensurePageState(page);
+  const name = String(opts.name ?? "").trim();
+  if (!name) throw new Error("device name is required");
+  const descriptor = (playwrightDevices as Record<string, unknown>)[name] as
+    | {
+        userAgent?: string;
+        viewport?: { width: number; height: number };
+        deviceScaleFactor?: number;
+        isMobile?: boolean;
+        hasTouch?: boolean;
+        locale?: string;
+      }
+    | undefined;
+  if (!descriptor) {
+    throw new Error(`Unknown device "${name}".`);
+  }
+
+  if (descriptor.viewport) {
+    await page.setViewportSize({
+      width: descriptor.viewport.width,
+      height: descriptor.viewport.height,
+    });
+  }
+
+  await withCdpSession(page, async (session) => {
+    if (descriptor.userAgent || descriptor.locale) {
+      await session.send("Emulation.setUserAgentOverride", {
+        userAgent: descriptor.userAgent ?? "",
+        acceptLanguage: descriptor.locale ?? undefined,
+      });
+    }
+    if (descriptor.viewport) {
+      await session.send("Emulation.setDeviceMetricsOverride", {
+        mobile: Boolean(descriptor.isMobile),
+        width: descriptor.viewport.width,
+        height: descriptor.viewport.height,
+        deviceScaleFactor: descriptor.deviceScaleFactor ?? 1,
+        screenWidth: descriptor.viewport.width,
+        screenHeight: descriptor.viewport.height,
+      });
+    }
+    if (descriptor.hasTouch) {
+      await session.send("Emulation.setTouchEmulationEnabled", {
+        enabled: true,
+      });
+    }
+  });
+}
+
 export async function typeViaPlaywright(opts: {
   cdpUrl: string;
   targetId?: string;
@@ -195,16 +677,21 @@ export async function typeViaPlaywright(opts: {
   const text = String(opts.text ?? "");
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
-  const locator = refLocator(page, requireRef(opts.ref));
+  const ref = requireRef(opts.ref);
+  const locator = refLocator(page, ref);
   const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000));
-  if (opts.slowly) {
-    await locator.click({ timeout });
-    await locator.type(text, { timeout, delay: 75 });
-  } else {
-    await locator.fill(text, { timeout });
-  }
-  if (opts.submit) {
-    await locator.press("Enter", { timeout });
+  try {
+    if (opts.slowly) {
+      await locator.click({ timeout });
+      await locator.type(text, { timeout, delay: 75 });
+    } else {
+      await locator.fill(text, { timeout });
+    }
+    if (opts.submit) {
+      await locator.press("Enter", { timeout });
+    }
+  } catch (err) {
+    throw toAIFriendlyError(err, ref);
   }
 }
 
@@ -212,9 +699,11 @@ export async function fillFormViaPlaywright(opts: {
   cdpUrl: string;
   targetId?: string;
   fields: BrowserFormField[];
+  timeoutMs?: number;
 }): Promise<void> {
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
+  const timeout = Math.max(500, Math.min(60_000, opts.timeoutMs ?? 8000));
   for (const field of opts.fields) {
     const ref = field.ref.trim();
     const type = field.type.trim();
@@ -233,10 +722,18 @@ export async function fillFormViaPlaywright(opts: {
         rawValue === 1 ||
         rawValue === "1" ||
         rawValue === "true";
-      await locator.setChecked(checked);
+      try {
+        await locator.setChecked(checked, { timeout });
+      } catch (err) {
+        throw toAIFriendlyError(err, ref);
+      }
       continue;
     }
-    await locator.fill(value);
+    try {
+      await locator.fill(value, { timeout });
+    } catch (err) {
+      throw toAIFriendlyError(err, ref);
+    }
   }
 }
 
@@ -359,7 +856,11 @@ export async function setInputFilesViaPlaywright(opts: {
     ? refLocator(page, inputRef)
     : page.locator(element).first();
 
-  await locator.setInputFiles(opts.paths);
+  try {
+    await locator.setInputFiles(opts.paths);
+  } catch (err) {
+    throw toAIFriendlyError(err, inputRef || element);
+  }
   try {
     const handle = await locator.elementHandle();
     if (handle) {
@@ -421,30 +922,54 @@ export async function waitForViaPlaywright(opts: {
   timeMs?: number;
   text?: string;
   textGone?: string;
+  selector?: string;
+  url?: string;
+  loadState?: "load" | "domcontentloaded" | "networkidle";
+  fn?: string;
   timeoutMs?: number;
 }): Promise<void> {
   const page = await getPageForTargetId(opts);
   ensurePageState(page);
+  const timeout = Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000));
+
   if (typeof opts.timeMs === "number" && Number.isFinite(opts.timeMs)) {
     await page.waitForTimeout(Math.max(0, opts.timeMs));
   }
   if (opts.text) {
-    await page
-      .getByText(opts.text)
-      .first()
-      .waitFor({
-        state: "visible",
-        timeout: Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)),
-      });
+    await page.getByText(opts.text).first().waitFor({
+      state: "visible",
+      timeout,
+    });
   }
   if (opts.textGone) {
-    await page
-      .getByText(opts.textGone)
-      .first()
-      .waitFor({
-        state: "hidden",
-        timeout: Math.max(500, Math.min(120_000, opts.timeoutMs ?? 20_000)),
-      });
+    await page.getByText(opts.textGone).first().waitFor({
+      state: "hidden",
+      timeout,
+    });
+  }
+  if (opts.selector) {
+    const selector = String(opts.selector).trim();
+    if (selector) {
+      await page
+        .locator(selector)
+        .first()
+        .waitFor({ state: "visible", timeout });
+    }
+  }
+  if (opts.url) {
+    const url = String(opts.url).trim();
+    if (url) {
+      await page.waitForURL(url, { timeout });
+    }
+  }
+  if (opts.loadState) {
+    await page.waitForLoadState(opts.loadState, { timeout });
+  }
+  if (opts.fn) {
+    const fn = String(opts.fn).trim();
+    if (fn) {
+      await page.waitForFunction(fn, { timeout });
+    }
   }
 }