feat: native image injection for vision-capable models

- Auto-detect and load images referenced in user prompts
- Inject history images at their original message positions
- Fix EXIF orientation - rotate before resizing in resizeToJpeg
- Sandbox security: validate paths, block remote URLs when sandbox enabled
- Prevent duplicate history image injection across turns
- Handle string-based user message content (convert to array)
- Add bounds check for message index in history processing
- Fix regex to properly match relative paths (./  ../)
- Add multi-image support for iMessage attachments
- Pass MAX_IMAGE_BYTES limit to image loading

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Tyler Yust
2026-01-17 03:10:10 -08:00
committed by Peter Steinberger
parent f7123ec30a
commit 8d74578ceb
9 changed files with 892 additions and 16 deletions

View File

@@ -307,9 +307,16 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
const mentionRegexes = buildMentionRegexes(cfg, route.agentId);
const messageText = (message.text ?? "").trim();
const attachments = includeAttachments ? (message.attachments ?? []) : [];
const firstAttachment = attachments?.find((entry) => entry?.original_path && !entry?.missing);
// Filter to valid attachments with paths
const validAttachments = attachments.filter(
(entry) => entry?.original_path && !entry?.missing,
);
const firstAttachment = validAttachments[0];
const mediaPath = firstAttachment?.original_path ?? undefined;
const mediaType = firstAttachment?.mime_type ?? undefined;
// Build arrays for all attachments (for multi-image support)
const mediaPaths = validAttachments.map((a) => a.original_path).filter(Boolean) as string[];
const mediaTypes = validAttachments.map((a) => a.mime_type ?? undefined);
const kind = mediaKindFromMime(mediaType ?? undefined);
const placeholder = kind ? `<media:${kind}>` : attachments?.length ? "<media:attachment>" : "";
const bodyText = messageText || placeholder;
@@ -445,6 +452,9 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
MediaPath: mediaPath,
MediaType: mediaType,
MediaUrl: mediaPath,
MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaTypes: mediaTypes.length > 0 ? mediaTypes : undefined,
MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined,
MediaRemoteHost: remoteHost,
WasMentioned: effectiveWasMentioned,
CommandAuthorized: commandAuthorized,