Agent: avoid silent failures on oversized images

src/agents/pi-tools.ts

@@ -1,6 +1,7 @@
 import type { AgentTool, AgentToolResult } from "@mariozechner/pi-ai";
-import { codingTools, readTool } from "@mariozechner/pi-coding-agent";
+import { bashTool, codingTools, readTool } from "@mariozechner/pi-coding-agent";
 import type { TSchema } from "@sinclair/typebox";
+import sharp from "sharp";
 
 import { detectMime } from "../media/mime.js";
 
@@ -10,6 +11,8 @@ type ToolContentBlock = AgentToolResult<unknown>["content"][number];
 type ImageContentBlock = Extract<ToolContentBlock, { type: "image" }>;
 type TextContentBlock = Extract<ToolContentBlock, { type: "text" }>;
 
+const MAX_IMAGE_DIMENSION_PX = 2000;
+
 function sniffMimeFromBase64(base64: string): string | undefined {
   const trimmed = base64.trim();
   if (!trimmed) return undefined;
@@ -94,6 +97,122 @@ function normalizeReadImageResult(
 
 type AnyAgentTool = AgentTool<TSchema, unknown>;
 
+function isImageBlock(block: unknown): block is ImageContentBlock {
+  if (!block || typeof block !== "object") return false;
+  const rec = block as Record<string, unknown>;
+  return (
+    rec.type === "image" &&
+    typeof rec.data === "string" &&
+    typeof rec.mimeType === "string"
+  );
+}
+
+function isTextBlock(block: unknown): block is TextContentBlock {
+  if (!block || typeof block !== "object") return false;
+  const rec = block as Record<string, unknown>;
+  return rec.type === "text" && typeof rec.text === "string";
+}
+
+async function resizeImageBase64IfNeeded(params: {
+  base64: string;
+  mimeType: string;
+  maxDimensionPx: number;
+}): Promise<{ base64: string; mimeType: string; resized: boolean }> {
+  const buf = Buffer.from(params.base64, "base64");
+  const img = sharp(buf, { failOnError: false });
+  const meta = await img.metadata();
+
+  const width = meta.width;
+  const height = meta.height;
+  if (
+    typeof width !== "number" ||
+    typeof height !== "number" ||
+    (width <= params.maxDimensionPx && height <= params.maxDimensionPx)
+  ) {
+    return { base64: params.base64, mimeType: params.mimeType, resized: false };
+  }
+
+  const resized = img.resize({
+    width: params.maxDimensionPx,
+    height: params.maxDimensionPx,
+    fit: "inside",
+    withoutEnlargement: true,
+  });
+
+  const mime = params.mimeType.toLowerCase();
+  let out: Buffer;
+  if (mime === "image/jpeg" || mime === "image/jpg") {
+    out = await resized.jpeg({ quality: 85 }).toBuffer();
+  } else if (mime === "image/webp") {
+    out = await resized.webp({ quality: 85 }).toBuffer();
+  } else if (mime === "image/png") {
+    out = await resized.png().toBuffer();
+  } else {
+    out = await resized.png().toBuffer();
+  }
+
+  const sniffed = detectMime({ buffer: out.slice(0, 256) });
+  const nextMime = sniffed?.startsWith("image/") ? sniffed : params.mimeType;
+
+  return { base64: out.toString("base64"), mimeType: nextMime, resized: true };
+}
+
+export async function sanitizeContentBlocksImages(
+  blocks: ToolContentBlock[],
+  label: string,
+  opts: { maxDimensionPx?: number } = {},
+): Promise<ToolContentBlock[]> {
+  const maxDimensionPx = Math.max(
+    opts.maxDimensionPx ?? MAX_IMAGE_DIMENSION_PX,
+    1,
+  );
+  const out: ToolContentBlock[] = [];
+
+  for (const block of blocks) {
+    if (!isImageBlock(block)) {
+      out.push(block);
+      continue;
+    }
+
+    const data = block.data.trim();
+    if (!data) {
+      out.push({
+        type: "text",
+        text: `[${label}] omitted empty image payload`,
+      } satisfies TextContentBlock);
+      continue;
+    }
+
+    try {
+      const resized = await resizeImageBase64IfNeeded({
+        base64: data,
+        mimeType: block.mimeType,
+        maxDimensionPx,
+      });
+      out.push({ ...block, data: resized.base64, mimeType: resized.mimeType });
+    } catch (err) {
+      out.push({
+        type: "text",
+        text: `[${label}] omitted image payload: ${String(err)}`,
+      } satisfies TextContentBlock);
+    }
+  }
+
+  return out;
+}
+
+export async function sanitizeToolResultImages(
+  result: AgentToolResult<unknown>,
+  label: string,
+  opts: { maxDimensionPx?: number } = {},
+): Promise<AgentToolResult<unknown>> {
+  const content = Array.isArray(result.content) ? result.content : [];
+  if (!content.some((b) => isImageBlock(b) || isTextBlock(b))) return result;
+
+  const next = await sanitizeContentBlocksImages(content, label, opts);
+  return { ...result, content: next };
+}
+
 function createClawdisReadTool(base: AnyAgentTool): AnyAgentTool {
   return {
     ...base,
@@ -109,7 +228,22 @@ function createClawdisReadTool(base: AnyAgentTool): AnyAgentTool {
           : undefined;
       const filePath =
         typeof record?.path === "string" ? String(record.path) : "<unknown>";
-      return normalizeReadImageResult(result, filePath);
+      const normalized = normalizeReadImageResult(result, filePath);
+      return sanitizeToolResultImages(normalized, `read:${filePath}`);
+    },
+  };
+}
+
+function createClawdisBashTool(base: AnyAgentTool): AnyAgentTool {
+  return {
+    ...base,
+    execute: async (toolCallId, params, signal) => {
+      const result = (await base.execute(
+        toolCallId,
+        params,
+        signal,
+      )) as AgentToolResult<unknown>;
+      return sanitizeToolResultImages(result, "bash");
     },
   };
 }
@@ -118,6 +252,8 @@ export function createClawdisCodingTools(): AnyAgentTool[] {
   return (codingTools as unknown as AnyAgentTool[]).map((tool) =>
     tool.name === readTool.name
       ? createClawdisReadTool(tool)
-      : (tool as AnyAgentTool),
+      : tool.name === bashTool.name
+        ? createClawdisBashTool(tool)
+        : (tool as AnyAgentTool),
   );
 }