From 710c6812839869ed8236e6a469ee7f0587da27ea Mon Sep 17 00:00:00 2001
From: John Rood <62669593+John-Rood@users.noreply.github.com>
Date: Tue, 20 Jan 2026 00:44:15 -0500
Subject: [PATCH] fix(browser): register refs from AI snapshot for act commands

When using the default AI snapshot format without explicit options like
interactive/compact/labels, refs were not being registered because
snapshotAiViaPlaywright returns raw text without ref registration.

This caused 'Unknown ref' errors when subsequently using act commands
with refs like e12 that appeared in the snapshot text.

The fix extracts refs from the AI snapshot using buildRoleSnapshotFromAiSnapshot
and registers them via rememberRoleRefsForTarget so act commands can resolve them.

Fixes #1268
---
 src/browser/routes/agent.snapshot.ts | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/browser/routes/agent.snapshot.ts b/src/browser/routes/agent.snapshot.ts
index fdeb7f69e..e66647f3f 100644
--- a/src/browser/routes/agent.snapshot.ts
+++ b/src/browser/routes/agent.snapshot.ts
@@ -9,6 +9,8 @@ import {
   DEFAULT_AI_SNAPSHOT_EFFICIENT_MAX_CHARS,
   DEFAULT_AI_SNAPSHOT_MAX_CHARS,
 } from "../constants.js";
+import { buildRoleSnapshotFromAiSnapshot } from "../pw-role-snapshot.js";
+import { rememberRoleRefsForTarget } from "../pw-session.js";
 import {
   DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
   DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
@@ -214,6 +216,20 @@ export function registerBrowserAgentSnapshotRoutes(app: express.Express, ctx: Br
                 targetId: tab.targetId,
                 ...(typeof resolvedMaxChars === "number" ? { maxChars: resolvedMaxChars } : {}),
               })
+              .then((result) => {
+                // Extract and register refs from AI snapshot so act commands can resolve them.
+                // snapshotAiViaPlaywright returns raw text without ref registration.
+                const parsed = buildRoleSnapshotFromAiSnapshot(result.snapshot);
+                if (Object.keys(parsed.refs).length > 0) {
+                  rememberRoleRefsForTarget({
+                    cdpUrl: profileCtx.profile.cdpUrl,
+                    targetId: tab.targetId,
+                    refs: parsed.refs,
+                    mode: "aria",
+                  });
+                }
+                return { ...result, refs: parsed.refs };
+              })
               .catch(async (err) => {
                 // Public-API fallback when Playwright's private _snapshotForAI is missing.
                 if (String(err).toLowerCase().includes("_snapshotforai")) {