From 8c0e290db18da2f698b0d491ea452dddefd69887 Mon Sep 17 00:00:00 2001
From: Tyler Yust <TYTYYUST@YAHOO.COM>
Date: Sat, 17 Jan 2026 03:24:34 -0800
Subject: [PATCH] fix: enhance image reference detection and optimize image
 processing

- Added support for detecting file URLs in prompts using fileURLToPath for accurate path resolution.
- Updated image loading logic to default to JPEG format for optimized image processing.
- Improved error handling in image optimization to continue processing on failures.
---
 src/agents/pi-embedded-runner/run/images.ts | 20 ++++++++++--
 src/web/media.ts                            | 36 ++++++++++++---------
 2 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/src/agents/pi-embedded-runner/run/images.ts b/src/agents/pi-embedded-runner/run/images.ts
index dad5805bf..fbfe9ee21 100644
--- a/src/agents/pi-embedded-runner/run/images.ts
+++ b/src/agents/pi-embedded-runner/run/images.ts
@@ -1,5 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
+import { fileURLToPath } from "node:url";
 
 import type { ImageContent } from "@mariozechner/pi-ai";
 
@@ -119,6 +120,18 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] {
     refs.push({ raw, type: "url", resolved: raw });
   }
 
+  // Pattern for file:// URLs - treat as paths since loadWebMedia handles them
+  const fileUrlPattern =
+    /file:\/\/[^\s<>"'`\]]+\.(?:png|jpe?g|gif|webp|bmp|tiff?|heic|heif)/gi;
+  while ((match = fileUrlPattern.exec(prompt)) !== null) {
+    const raw = match[0];
+    if (seen.has(raw.toLowerCase())) continue;
+    seen.add(raw.toLowerCase());
+    // Use fileURLToPath for proper handling (e.g., file://localhost/path)
+    const resolved = fileURLToPath(raw);
+    refs.push({ raw, type: "path", resolved });
+  }
+
   // Pattern for file paths (absolute, relative, or home)
   // Matches:
   // - /absolute/path/to/file.ext (including paths with special chars like Messages/Attachments)
@@ -127,8 +140,8 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] {
   // - ~/home/path.ext
   const pathPattern = /(?:^|\s|["'`(])((\.\.?\/|[~/])[^\s"'`()[\]]*\.(?:png|jpe?g|gif|webp|bmp|tiff?|heic|heif))/gi;
   while ((match = pathPattern.exec(prompt)) !== null) {
-    const raw = match[1] || match[0];
-    addPathRef(raw);
+    // Use capture group 1 (the path without delimiter prefix); skip if undefined
+    if (match[1]) addPathRef(match[1]);
   }
 
   return refs;
@@ -205,7 +218,8 @@ export async function loadImageFromRef(
     }
 
     // EXIF orientation is already normalized by loadWebMedia -> resizeToJpeg
-    const mimeType = media.contentType ?? "image/png";
+    // Default to JPEG since optimization converts images to JPEG format
+    const mimeType = media.contentType ?? "image/jpeg";
     const data = media.buffer.toString("base64");
 
     return { type: "image", data, mimeType };
diff --git a/src/web/media.ts b/src/web/media.ts
index 1d9bdceab..4768772d1 100644
--- a/src/web/media.ts
+++ b/src/web/media.ts
@@ -161,23 +161,27 @@ export async function optimizeImageToJpeg(
 
   for (const side of sides) {
     for (const quality of qualities) {
-      const out = await resizeToJpeg({
-        buffer,
-        maxSide: side,
-        quality,
-        withoutEnlargement: true,
-      });
-      const size = out.length;
-      if (!smallest || size < smallest.size) {
-        smallest = { buffer: out, size, resizeSide: side, quality };
-      }
-      if (size <= maxBytes) {
-        return {
-          buffer: out,
-          optimizedSize: size,
-          resizeSide: side,
+      try {
+        const out = await resizeToJpeg({
+          buffer,
+          maxSide: side,
           quality,
-        };
+          withoutEnlargement: true,
+        });
+        const size = out.length;
+        if (!smallest || size < smallest.size) {
+          smallest = { buffer: out, size, resizeSide: side, quality };
+        }
+        if (size <= maxBytes) {
+          return {
+            buffer: out,
+            optimizedSize: size,
+            resizeSide: side,
+            quality,
+          };
+        }
+      } catch {
+        // Continue trying other size/quality combinations
       }
     }
   }