fix(image): route MiniMax vision to VLM

This commit is contained in:
Peter Steinberger
2026-01-12 20:38:11 +00:00
parent b73042500e
commit 36a02b3e67
4 changed files with 338 additions and 22 deletions

View File

@@ -21,6 +21,8 @@ async function writeAuthProfiles(agentDir: string, profiles: unknown) {
}
describe("image tool implicit imageModel config", () => {
const priorFetch = global.fetch;
beforeEach(() => {
vi.stubEnv("OPENAI_API_KEY", "");
vi.stubEnv("ANTHROPIC_API_KEY", "");
@@ -30,6 +32,8 @@ describe("image tool implicit imageModel config", () => {
afterEach(() => {
vi.unstubAllEnvs();
// @ts-expect-error global fetch cleanup
global.fetch = priorFetch;
});
it("stays disabled without auth when no pairing is possible", async () => {
@@ -132,6 +136,60 @@ describe("image tool implicit imageModel config", () => {
tool.execute("t2", { image: "../escape.png" }),
).rejects.toThrow(/escapes sandbox root/i);
});
it("rewrites inbound absolute paths into sandbox media/inbound", async () => {
const stateDir = await fs.mkdtemp(
path.join(os.tmpdir(), "clawdbot-image-sandbox-"),
);
const agentDir = path.join(stateDir, "agent");
const sandboxRoot = path.join(stateDir, "sandbox");
await fs.mkdir(agentDir, { recursive: true });
await fs.mkdir(path.join(sandboxRoot, "media", "inbound"), {
recursive: true,
});
const pngB64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=";
await fs.writeFile(
path.join(sandboxRoot, "media", "inbound", "photo.png"),
Buffer.from(pngB64, "base64"),
);
const fetch = vi.fn().mockResolvedValue({
ok: true,
status: 200,
statusText: "OK",
headers: new Headers(),
json: async () => ({
content: "ok",
base_resp: { status_code: 0, status_msg: "" },
}),
});
// @ts-expect-error partial global
global.fetch = fetch;
vi.stubEnv("MINIMAX_API_KEY", "minimax-test");
const cfg: ClawdbotConfig = {
agents: {
defaults: {
model: { primary: "minimax/MiniMax-M2.1" },
imageModel: { primary: "minimax/MiniMax-VL-01" },
},
},
};
const tool = createImageTool({ config: cfg, agentDir, sandboxRoot });
expect(tool).not.toBeNull();
if (!tool) throw new Error("expected image tool");
const res = await tool.execute("t1", {
prompt: "Describe the image.",
image: "/Users/steipete/.clawdbot/media/inbound/photo.png",
});
expect(fetch).toHaveBeenCalledTimes(1);
expect((res.details as { rewrittenFrom?: string }).rewrittenFrom).toContain(
"photo.png",
);
});
});
describe("image tool data URL support", () => {
@@ -151,6 +209,99 @@ describe("image tool data URL support", () => {
});
});
describe("image tool MiniMax VLM routing", () => {
const pngB64 =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=";
const priorFetch = global.fetch;
beforeEach(() => {
vi.stubEnv("MINIMAX_API_KEY", "");
});
afterEach(() => {
vi.unstubAllEnvs();
// @ts-expect-error global fetch cleanup
global.fetch = priorFetch;
});
it("calls /v1/coding_plan/vlm for minimax image models", async () => {
const fetch = vi.fn().mockResolvedValue({
ok: true,
status: 200,
statusText: "OK",
headers: new Headers(),
json: async () => ({
content: "ok",
base_resp: { status_code: 0, status_msg: "" },
}),
});
// @ts-expect-error partial global
global.fetch = fetch;
const agentDir = await fs.mkdtemp(
path.join(os.tmpdir(), "clawdbot-minimax-vlm-"),
);
vi.stubEnv("MINIMAX_API_KEY", "minimax-test");
const cfg: ClawdbotConfig = {
agents: { defaults: { model: { primary: "minimax/MiniMax-M2.1" } } },
};
const tool = createImageTool({ config: cfg, agentDir });
expect(tool).not.toBeNull();
if (!tool) throw new Error("expected image tool");
const res = await tool.execute("t1", {
prompt: "Describe the image.",
image: `data:image/png;base64,${pngB64}`,
});
expect(fetch).toHaveBeenCalledTimes(1);
const [url, init] = fetch.mock.calls[0];
expect(String(url)).toBe("https://api.minimax.io/v1/coding_plan/vlm");
expect(init?.method).toBe("POST");
expect(
String((init?.headers as Record<string, string>)?.Authorization),
).toBe("Bearer minimax-test");
expect(String(init?.body)).toContain('"prompt":"Describe the image."');
expect(String(init?.body)).toContain('"image_url":"data:image/png;base64,');
const text = res.content?.find((b) => b.type === "text")?.text ?? "";
expect(text).toBe("ok");
});
it("surfaces MiniMax API errors from /v1/coding_plan/vlm", async () => {
const fetch = vi.fn().mockResolvedValue({
ok: true,
status: 200,
statusText: "OK",
headers: new Headers(),
json: async () => ({
content: "",
base_resp: { status_code: 1004, status_msg: "bad key" },
}),
});
// @ts-expect-error partial global
global.fetch = fetch;
const agentDir = await fs.mkdtemp(
path.join(os.tmpdir(), "clawdbot-minimax-vlm-"),
);
vi.stubEnv("MINIMAX_API_KEY", "minimax-test");
const cfg: ClawdbotConfig = {
agents: { defaults: { model: { primary: "minimax/MiniMax-M2.1" } } },
};
const tool = createImageTool({ config: cfg, agentDir });
expect(tool).not.toBeNull();
if (!tool) throw new Error("expected image tool");
await expect(
tool.execute("t1", {
prompt: "Describe the image.",
image: `data:image/png;base64,${pngB64}`,
}),
).rejects.toThrow(/MiniMax VLM API error/i);
});
});
describe("image tool response validation", () => {
it("rejects image-model responses with no final text", () => {
expect(() =>