feat: native image injection for vision-capable models

- Auto-detect and load images referenced in user prompts
- Inject history images at their original message positions
- Fix EXIF orientation - rotate before resizing in resizeToJpeg
- Sandbox security: validate paths, block remote URLs when sandbox enabled
- Prevent duplicate history image injection across turns
- Handle string-based user message content (convert to array)
- Add bounds check for message index in history processing
- Fix regex to properly match relative paths (./  ../)
- Add multi-image support for iMessage attachments
- Pass MAX_IMAGE_BYTES limit to image loading

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Tyler Yust
2026-01-17 03:10:10 -08:00
committed by Peter Steinberger
parent f7123ec30a
commit 8d74578ceb
9 changed files with 892 additions and 16 deletions

View File

@@ -41,12 +41,15 @@ export function createClawdbotTools(options?: {
replyToMode?: "off" | "first" | "all";
/** Mutable ref to track if a reply was sent (for "first" mode). */
hasRepliedRef?: { value: boolean };
/** If true, the model has native vision capability */
modelHasVision?: boolean;
}): AnyAgentTool[] {
const imageTool = options?.agentDir?.trim()
? createImageTool({
config: options?.config,
agentDir: options.agentDir,
sandboxRoot: options?.sandboxRoot,
modelHasVision: options?.modelHasVision,
})
: null;
const webSearchTool = createWebSearchTool({