fix: resolve camera tool handling
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const callGateway = vi.fn();
|
||||
const { callGateway } = vi.hoisted(() => ({
|
||||
callGateway: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../gateway/call.js", () => ({ callGateway }));
|
||||
vi.mock("../media/image-ops.js", () => ({
|
||||
|
||||
@@ -865,20 +865,6 @@ function createCanvasTool(): AnyAgentTool {
|
||||
Number.isFinite(params.quality)
|
||||
? params.quality
|
||||
: undefined;
|
||||
const delayMs =
|
||||
typeof params.delayMs === "number" &&
|
||||
Number.isFinite(params.delayMs)
|
||||
? params.delayMs
|
||||
: undefined;
|
||||
const deviceId =
|
||||
typeof params.deviceId === "string" && params.deviceId.trim()
|
||||
? params.deviceId.trim()
|
||||
: undefined;
|
||||
const delayMs =
|
||||
typeof params.delayMs === "number" &&
|
||||
Number.isFinite(params.delayMs)
|
||||
? params.delayMs
|
||||
: undefined;
|
||||
const raw = (await invoke("canvas.snapshot", {
|
||||
format,
|
||||
maxWidth,
|
||||
@@ -889,8 +875,7 @@ function createCanvasTool(): AnyAgentTool {
|
||||
ext: payload.format === "jpeg" ? "jpg" : payload.format,
|
||||
});
|
||||
await writeBase64ToFile(filePath, payload.base64);
|
||||
const mimeType =
|
||||
imageMimeFromFormat(payload.format) ?? "image/png";
|
||||
const mimeType = imageMimeFromFormat(payload.format) ?? "image/png";
|
||||
return await imageResult({
|
||||
label: "canvas:snapshot",
|
||||
path: filePath,
|
||||
@@ -1139,6 +1124,15 @@ function createNodesTool(): AnyAgentTool {
|
||||
Number.isFinite(params.quality)
|
||||
? params.quality
|
||||
: undefined;
|
||||
const delayMs =
|
||||
typeof params.delayMs === "number" &&
|
||||
Number.isFinite(params.delayMs)
|
||||
? params.delayMs
|
||||
: undefined;
|
||||
const deviceId =
|
||||
typeof params.deviceId === "string" && params.deviceId.trim()
|
||||
? params.deviceId.trim()
|
||||
: undefined;
|
||||
|
||||
const content: AgentToolResult<unknown>["content"] = [];
|
||||
const details: Array<Record<string, unknown>> = [];
|
||||
@@ -1158,10 +1152,23 @@ function createNodesTool(): AnyAgentTool {
|
||||
idempotencyKey: crypto.randomUUID(),
|
||||
})) as { payload?: unknown };
|
||||
const payload = parseCameraSnapPayload(raw?.payload);
|
||||
const normalizedFormat = payload.format.toLowerCase();
|
||||
if (
|
||||
normalizedFormat !== "jpg" &&
|
||||
normalizedFormat !== "jpeg" &&
|
||||
normalizedFormat !== "png"
|
||||
) {
|
||||
throw new Error(
|
||||
`unsupported camera.snap format: ${payload.format}`,
|
||||
);
|
||||
}
|
||||
|
||||
const isJpeg =
|
||||
normalizedFormat === "jpg" || normalizedFormat === "jpeg";
|
||||
const filePath = cameraTempPath({
|
||||
kind: "snap",
|
||||
facing,
|
||||
ext: payload.format === "jpeg" ? "jpg" : payload.format,
|
||||
ext: isJpeg ? "jpg" : "png",
|
||||
});
|
||||
await writeBase64ToFile(filePath, payload.base64);
|
||||
content.push({ type: "text", text: `MEDIA:${filePath}` });
|
||||
@@ -1169,7 +1176,8 @@ function createNodesTool(): AnyAgentTool {
|
||||
type: "image",
|
||||
data: payload.base64,
|
||||
mimeType:
|
||||
imageMimeFromFormat(payload.format) ?? "image/png",
|
||||
imageMimeFromFormat(payload.format) ??
|
||||
(isJpeg ? "image/jpeg" : "image/png"),
|
||||
});
|
||||
details.push({
|
||||
facing,
|
||||
|
||||
@@ -32,4 +32,32 @@ describe("tool image sanitizing", () => {
|
||||
expect(size).toBeLessThanOrEqual(5 * 1024 * 1024);
|
||||
expect(image.mimeType).toBe("image/jpeg");
|
||||
}, 20_000);
|
||||
|
||||
it("corrects mismatched jpeg mimeType", async () => {
|
||||
const jpeg = await sharp({
|
||||
create: {
|
||||
width: 10,
|
||||
height: 10,
|
||||
channels: 3,
|
||||
background: { r: 255, g: 0, b: 0 },
|
||||
},
|
||||
})
|
||||
.jpeg()
|
||||
.toBuffer();
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: "image" as const,
|
||||
data: jpeg.toString("base64"),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
];
|
||||
|
||||
const out = await sanitizeContentBlocksImages(blocks, "test");
|
||||
const image = out.find((b) => b.type === "image");
|
||||
if (!image || image.type !== "image") {
|
||||
throw new Error("expected image block");
|
||||
}
|
||||
expect(image.mimeType).toBe("image/jpeg");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -31,6 +31,15 @@ function isTextBlock(block: unknown): block is TextContentBlock {
|
||||
return rec.type === "text" && typeof rec.text === "string";
|
||||
}
|
||||
|
||||
function inferMimeTypeFromBase64(base64: string): string | undefined {
|
||||
const trimmed = base64.trim();
|
||||
if (!trimmed) return undefined;
|
||||
if (trimmed.startsWith("/9j/")) return "image/jpeg";
|
||||
if (trimmed.startsWith("iVBOR")) return "image/png";
|
||||
if (trimmed.startsWith("R0lGOD")) return "image/gif";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function resizeImageBase64IfNeeded(params: {
|
||||
base64: string;
|
||||
mimeType: string;
|
||||
@@ -127,13 +136,19 @@ export async function sanitizeContentBlocksImages(
|
||||
}
|
||||
|
||||
try {
|
||||
const inferredMimeType = inferMimeTypeFromBase64(data);
|
||||
const mimeType = inferredMimeType ?? block.mimeType;
|
||||
const resized = await resizeImageBase64IfNeeded({
|
||||
base64: data,
|
||||
mimeType: block.mimeType,
|
||||
mimeType,
|
||||
maxDimensionPx,
|
||||
maxBytes,
|
||||
});
|
||||
out.push({ ...block, data: resized.base64, mimeType: resized.mimeType });
|
||||
out.push({
|
||||
...block,
|
||||
data: resized.base64,
|
||||
mimeType: resized.resized ? resized.mimeType : mimeType,
|
||||
});
|
||||
} catch (err) {
|
||||
out.push({
|
||||
type: "text",
|
||||
|
||||
Reference in New Issue
Block a user