fix: resolve camera tool handling

This commit is contained in:
Peter Steinberger
2026-01-02 17:44:25 +00:00
parent 8e48cffe3b
commit 43f6b9ef32
7 changed files with 105 additions and 37 deletions

View File

@@ -1,6 +1,8 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const callGateway = vi.fn();
const { callGateway } = vi.hoisted(() => ({
callGateway: vi.fn(),
}));
vi.mock("../gateway/call.js", () => ({ callGateway }));
vi.mock("../media/image-ops.js", () => ({

View File

@@ -865,20 +865,6 @@ function createCanvasTool(): AnyAgentTool {
Number.isFinite(params.quality)
? params.quality
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const deviceId =
typeof params.deviceId === "string" && params.deviceId.trim()
? params.deviceId.trim()
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const raw = (await invoke("canvas.snapshot", {
format,
maxWidth,
@@ -889,8 +875,7 @@ function createCanvasTool(): AnyAgentTool {
ext: payload.format === "jpeg" ? "jpg" : payload.format,
});
await writeBase64ToFile(filePath, payload.base64);
const mimeType =
imageMimeFromFormat(payload.format) ?? "image/png";
const mimeType = imageMimeFromFormat(payload.format) ?? "image/png";
return await imageResult({
label: "canvas:snapshot",
path: filePath,
@@ -1139,6 +1124,15 @@ function createNodesTool(): AnyAgentTool {
Number.isFinite(params.quality)
? params.quality
: undefined;
const delayMs =
typeof params.delayMs === "number" &&
Number.isFinite(params.delayMs)
? params.delayMs
: undefined;
const deviceId =
typeof params.deviceId === "string" && params.deviceId.trim()
? params.deviceId.trim()
: undefined;
const content: AgentToolResult<unknown>["content"] = [];
const details: Array<Record<string, unknown>> = [];
@@ -1158,10 +1152,23 @@ function createNodesTool(): AnyAgentTool {
idempotencyKey: crypto.randomUUID(),
})) as { payload?: unknown };
const payload = parseCameraSnapPayload(raw?.payload);
const normalizedFormat = payload.format.toLowerCase();
if (
normalizedFormat !== "jpg" &&
normalizedFormat !== "jpeg" &&
normalizedFormat !== "png"
) {
throw new Error(
`unsupported camera.snap format: ${payload.format}`,
);
}
const isJpeg =
normalizedFormat === "jpg" || normalizedFormat === "jpeg";
const filePath = cameraTempPath({
kind: "snap",
facing,
ext: payload.format === "jpeg" ? "jpg" : payload.format,
ext: isJpeg ? "jpg" : "png",
});
await writeBase64ToFile(filePath, payload.base64);
content.push({ type: "text", text: `MEDIA:${filePath}` });
@@ -1169,7 +1176,8 @@ function createNodesTool(): AnyAgentTool {
type: "image",
data: payload.base64,
mimeType:
imageMimeFromFormat(payload.format) ?? "image/png",
imageMimeFromFormat(payload.format) ??
(isJpeg ? "image/jpeg" : "image/png"),
});
details.push({
facing,

View File

@@ -32,4 +32,32 @@ describe("tool image sanitizing", () => {
expect(size).toBeLessThanOrEqual(5 * 1024 * 1024);
expect(image.mimeType).toBe("image/jpeg");
}, 20_000);
it("corrects mismatched jpeg mimeType", async () => {
const jpeg = await sharp({
create: {
width: 10,
height: 10,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.jpeg()
.toBuffer();
const blocks = [
{
type: "image" as const,
data: jpeg.toString("base64"),
mimeType: "image/png",
},
];
const out = await sanitizeContentBlocksImages(blocks, "test");
const image = out.find((b) => b.type === "image");
if (!image || image.type !== "image") {
throw new Error("expected image block");
}
expect(image.mimeType).toBe("image/jpeg");
});
});

View File

@@ -31,6 +31,15 @@ function isTextBlock(block: unknown): block is TextContentBlock {
return rec.type === "text" && typeof rec.text === "string";
}
function inferMimeTypeFromBase64(base64: string): string | undefined {
const trimmed = base64.trim();
if (!trimmed) return undefined;
if (trimmed.startsWith("/9j/")) return "image/jpeg";
if (trimmed.startsWith("iVBOR")) return "image/png";
if (trimmed.startsWith("R0lGOD")) return "image/gif";
return undefined;
}
async function resizeImageBase64IfNeeded(params: {
base64: string;
mimeType: string;
@@ -127,13 +136,19 @@ export async function sanitizeContentBlocksImages(
}
try {
const inferredMimeType = inferMimeTypeFromBase64(data);
const mimeType = inferredMimeType ?? block.mimeType;
const resized = await resizeImageBase64IfNeeded({
base64: data,
mimeType: block.mimeType,
mimeType,
maxDimensionPx,
maxBytes,
});
out.push({ ...block, data: resized.base64, mimeType: resized.mimeType });
out.push({
...block,
data: resized.base64,
mimeType: resized.resized ? resized.mimeType : mimeType,
});
} catch (err) {
out.push({
type: "text",