From 0a2dcd844b06bf02de035693bd5d257e9b0a2a99 Mon Sep 17 00:00:00 2001
From: Peter Steinberger <steipete@gmail.com>
Date: Mon, 12 Jan 2026 18:17:39 +0000
Subject: [PATCH] fix(image): support data URLs

---
 src/agents/tools/image-tool.test.ts | 18 +++++++++
 src/agents/tools/image-tool.ts      | 61 +++++++++++++++++++++++------
 2 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/src/agents/tools/image-tool.test.ts b/src/agents/tools/image-tool.test.ts
index 31c3869af..20e212996 100644
--- a/src/agents/tools/image-tool.test.ts
+++ b/src/agents/tools/image-tool.test.ts
@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
 import type { ClawdbotConfig } from "../../config/config.js";
 import {
+  __testing,
   createImageTool,
   resolveImageModelConfigForTool,
 } from "./image-tool.js";
@@ -132,3 +133,20 @@ describe("image tool implicit imageModel config", () => {
     ).rejects.toThrow(/escapes sandbox root/i);
   });
 });
+
+describe("image tool data URL support", () => {
+  it("decodes base64 image data URLs", () => {
+    const pngB64 =
+      "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=";
+    const out = __testing.decodeDataUrl(`data:image/png;base64,${pngB64}`);
+    expect(out.kind).toBe("image");
+    expect(out.mimeType).toBe("image/png");
+    expect(out.buffer.length).toBeGreaterThan(0);
+  });
+
+  it("rejects non-image data URLs", () => {
+    expect(() =>
+      __testing.decodeDataUrl("data:text/plain;base64,SGVsbG8="),
+    ).toThrow(/Unsupported data URL type/i);
+  });
+});
diff --git a/src/agents/tools/image-tool.ts b/src/agents/tools/image-tool.ts
index 7b6b6628d..a3e7f3787 100644
--- a/src/agents/tools/image-tool.ts
+++ b/src/agents/tools/image-tool.ts
@@ -31,6 +31,30 @@ const DEFAULT_PROMPT = "Describe the image.";
 
 type ImageModelConfig = { primary?: string; fallbacks?: string[] };
 
+function decodeDataUrl(dataUrl: string): {
+  buffer: Buffer;
+  mimeType: string;
+  kind: "image";
+} {
+  const trimmed = dataUrl.trim();
+  const match = /^data:([^;,]+);base64,([a-z0-9+/=\r\n]+)$/i.exec(trimmed);
+  if (!match) throw new Error("Invalid data URL (expected base64 data: URL).");
+  const mimeType = (match[1] ?? "").trim().toLowerCase();
+  if (!mimeType.startsWith("image/")) {
+    throw new Error(`Unsupported data URL type: ${mimeType || "unknown"}`);
+  }
+  const b64 = (match[2] ?? "").trim();
+  const buffer = Buffer.from(b64, "base64");
+  if (buffer.length === 0) {
+    throw new Error("Invalid data URL: empty payload.");
+  }
+  return { buffer, mimeType, kind: "image" };
+}
+
+export const __testing = {
+  decodeDataUrl,
+} as const;
+
 function coerceImageModelConfig(cfg?: ClawdbotConfig): ImageModelConfig {
   const imageModel = cfg?.agents?.defaults?.imageModel as
     | { primary?: string; fallbacks?: string[] }
@@ -349,18 +373,31 @@ export function createImageTool(options?: {
         throw new Error("Sandboxed image tool does not allow remote URLs.");
       }
 
-      const resolvedImage = sandboxRoot
-        ? (
-            await assertSandboxPath({
-              filePath: imageRaw,
-              cwd: sandboxRoot,
-              root: sandboxRoot,
-            })
-          ).resolved
-        : imageRaw.startsWith("~")
-          ? resolveUserPath(imageRaw)
-          : imageRaw;
-      const media = await loadWebMedia(resolvedImage, maxBytes);
+      const isDataUrl = /^data:/i.test(imageRaw);
+      const resolvedImage = (() => {
+        if (sandboxRoot) return imageRaw;
+        if (imageRaw.startsWith("~")) return resolveUserPath(imageRaw);
+        return imageRaw;
+      })();
+      const resolvedPath = isDataUrl
+        ? null
+        : sandboxRoot
+          ? (
+              await assertSandboxPath({
+                filePath: resolvedImage.startsWith("file://")
+                  ? resolvedImage.slice("file://".length)
+                  : resolvedImage,
+                cwd: sandboxRoot,
+                root: sandboxRoot,
+              })
+            ).resolved
+          : resolvedImage.startsWith("file://")
+            ? resolvedImage.slice("file://".length)
+            : resolvedImage;
+
+      const media = isDataUrl
+        ? decodeDataUrl(resolvedImage)
+        : await loadWebMedia(resolvedPath ?? resolvedImage, maxBytes);
       if (media.kind !== "image") {
         throw new Error(`Unsupported media type: ${media.kind}`);
       }