fix: improve browser upload triggering

CHANGELOG.md

@@ -19,6 +19,7 @@
 
 ### Fixes
 - Docs/agent tools: clarify that browser `wait` should be avoided by default and used only in exceptional cases.
+- Browser tools: `upload` can auto-click a ref after arming and now emits input/change events after `setFiles` so sites like X pick up attachments.
 - macOS: Voice Wake now fully tears down the Speech pipeline when disabled (cancel pending restarts, drop stale callbacks) to avoid high CPU in the background.
 - macOS menu: add a Talk Mode action alongside the Open Dashboard/Chat/Canvas entries.
 - macOS Debug: hide “Restart Gateway” when the app won’t start a local gateway (remote mode / attach-only).

docs/browser.md

@@ -190,6 +190,7 @@ Actions:
 
 Notes:
 - `upload` and `dialog` are **arming** calls; run them before the click/press that triggers the chooser/dialog.
+- `upload` can take a `ref` to auto-click after arming (useful for single-step file uploads).
 - The arm default timeout is **2 minutes** (clamped to max 2 minutes); pass `timeoutMs` if you need shorter.
 - `snapshot` defaults to `ai`; `aria` returns an accessibility tree for debugging.
 - `click`/`type` require `ref` from `snapshot --format ai`; use `evaluate` for rare CSS selector one-offs.

docs/tools.md

@@ -52,6 +52,7 @@ Notes:
 - `snapshot` defaults to `ai`; use `aria` for the accessibility tree.
 - `act` requires `ref` from `snapshot --format ai`; use `evaluate` for rare CSS selector needs.
 - Avoid `act` → `wait` by default; use it only in exceptional cases (no reliable UI state to wait on).
+- `upload` can optionally pass a `ref` to auto-click after arming.
 
 ### `clawdis_canvas`
 Drive the node Canvas (present, eval, snapshot, A2UI).

src/agents/clawdis-tools.ts

@@ -483,6 +483,7 @@ const BrowserToolSchema = Type.Union([
     action: Type.Literal("upload"),
     controlUrl: Type.Optional(Type.String()),
     paths: Type.Array(Type.String()),
+    ref: Type.Optional(Type.String()),
     targetId: Type.Optional(Type.String()),
     timeoutMs: Type.Optional(Type.Number()),
   }),
@@ -625,6 +626,7 @@ function createBrowserTool(): AnyAgentTool {
             ? params.paths.map((p) => String(p))
             : [];
           if (paths.length === 0) throw new Error("paths required");
+          const ref = readStringParam(params, "ref");
           const targetId =
             typeof params.targetId === "string"
               ? params.targetId.trim()
@@ -637,6 +639,7 @@ function createBrowserTool(): AnyAgentTool {
           return jsonResult(
             await browserArmFileChooser(baseUrl, {
               paths,
+              ref,
               targetId,
               timeoutMs,
             }),

src/browser/client-actions-core.ts

@@ -93,6 +93,7 @@ export async function browserArmFileChooser(
   baseUrl: string,
   opts: {
     paths: string[];
+    ref?: string;
     targetId?: string;
     timeoutMs?: number;
   },
@@ -104,6 +105,7 @@ export async function browserArmFileChooser(
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
         paths: opts.paths,
+        ref: opts.ref,
         targetId: opts.targetId,
         timeoutMs: opts.timeoutMs,
       }),

src/browser/pw-tools-core.ts

@@ -283,6 +283,20 @@ export async function armFileUploadViaPlaywright(opts: {
         return;
       }
       await fileChooser.setFiles(opts.paths);
+      try {
+        const input =
+          typeof fileChooser.element === "function"
+            ? await fileChooser.element()
+            : null;
+        if (input) {
+          await input.evaluate((el) => {
+            el.dispatchEvent(new Event("input", { bubbles: true }));
+            el.dispatchEvent(new Event("change", { bubbles: true }));
+          });
+        }
+      } catch {
+        // Best-effort for sites that don't react to setFiles alone.
+      }
     })
     .catch(() => {
       // Ignore timeouts; the chooser may never appear.

src/browser/routes/agent.ts

@@ -338,6 +338,7 @@ export function registerBrowserAgentRoutes(
   app.post("/hooks/file-chooser", async (req, res) => {
     const body = readBody(req);
     const targetId = toStringOrEmpty(body.targetId) || undefined;
+    const ref = toStringOrEmpty(body.ref) || undefined;
     const paths = toStringArray(body.paths) ?? [];
     const timeoutMs = toNumber(body.timeoutMs);
     if (!paths.length) return jsonError(res, 400, "paths are required");
@@ -351,6 +352,13 @@ export function registerBrowserAgentRoutes(
         paths,
         timeoutMs: timeoutMs ?? undefined,
       });
+      if (ref) {
+        await pw.clickViaPlaywright({
+          cdpPort: ctx.state().cdpPort,
+          targetId: tab.targetId,
+          ref,
+        });
+      }
       res.json({ ok: true });
     } catch (err) {
       handleRouteError(ctx, res, err);

src/browser/server.test.ts

@@ -475,6 +475,24 @@ describe("browser control server", () => {
       timeoutMs: 1234,
     });
 
+    const uploadWithRef = await realFetch(`${base}/hooks/file-chooser`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ paths: ["/tmp/b.txt"], ref: "e12" }),
+    }).then((r) => r.json());
+    expect(uploadWithRef).toMatchObject({ ok: true });
+    expect(pwMocks.armFileUploadViaPlaywright).toHaveBeenCalledWith({
+      cdpPort: testPort + 1,
+      targetId: "abcd1234",
+      paths: ["/tmp/b.txt"],
+      timeoutMs: undefined,
+    });
+    expect(pwMocks.clickViaPlaywright).toHaveBeenCalledWith({
+      cdpPort: testPort + 1,
+      targetId: "abcd1234",
+      ref: "e12",
+    });
+
     const dialog = await realFetch(`${base}/hooks/dialog`, {
       method: "POST",
       headers: { "Content-Type": "application/json" },

src/cli/browser-cli-actions-input.ts

@@ -301,6 +301,7 @@ export function registerBrowserActionInputCommands(
     .command("upload")
     .description("Arm file upload for the next file chooser")
     .argument("<paths...>", "File paths to upload")
+    .option("--ref <ref>", "Ref id from ai snapshot to click after arming")
     .option("--target-id <id>", "CDP target id (or unique prefix)")
     .option(
       "--timeout-ms <ms>",
@@ -313,6 +314,7 @@ export function registerBrowserActionInputCommands(
       try {
         const result = await browserArmFileChooser(baseUrl, {
           paths,
+          ref: opts.ref?.trim() || undefined,
           targetId: opts.targetId?.trim() || undefined,
           timeoutMs: Number.isFinite(opts.timeoutMs)
             ? opts.timeoutMs