import { describe, expect, it, vi } from "vitest"; import type { ClawdbotConfig } from "../config/config.js"; import { runWithModelFallback } from "./model-fallback.js"; function makeCfg(overrides: Partial = {}): ClawdbotConfig { return { agents: { defaults: { model: { primary: "openai/gpt-4.1-mini", fallbacks: ["anthropic/claude-haiku-3-5"], }, }, }, ...overrides, } as ClawdbotConfig; } describe("runWithModelFallback", () => { it("does not fall back on non-auth errors", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce(new Error("bad request")) .mockResolvedValueOnce("ok"); await expect( runWithModelFallback({ cfg, provider: "openai", model: "gpt-4.1-mini", run, }), ).rejects.toThrow("bad request"); expect(run).toHaveBeenCalledTimes(1); }); it("falls back on auth errors", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce(Object.assign(new Error("nope"), { status: 401 })) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "openai", model: "gpt-4.1-mini", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(run.mock.calls[1]?.[0]).toBe("anthropic"); expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5"); }); it("falls back on 402 payment required", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce( Object.assign(new Error("payment required"), { status: 402 }), ) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "openai", model: "gpt-4.1-mini", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(run.mock.calls[1]?.[0]).toBe("anthropic"); expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5"); }); it("falls back on billing errors", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce( new Error( "LLM request rejected: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", ), ) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "openai", model: "gpt-4.1-mini", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(run.mock.calls[1]?.[0]).toBe("anthropic"); expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5"); }); it("falls back on credential validation errors", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce( new Error('No credentials found for profile "anthropic:claude-cli".'), ) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "anthropic", model: "claude-opus-4", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(run.mock.calls[1]?.[0]).toBe("anthropic"); expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5"); }); it("falls back on missing API key errors", async () => { const cfg = makeCfg(); const run = vi .fn() .mockRejectedValueOnce(new Error("No API key found for profile openai.")) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "openai", model: "gpt-4.1-mini", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(run.mock.calls[1]?.[0]).toBe("anthropic"); expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5"); }); it("appends the configured primary as a last fallback", async () => { const cfg = makeCfg({ agents: { defaults: { model: { primary: "openai/gpt-4.1-mini", fallbacks: [], }, }, }, }); const run = vi .fn() .mockRejectedValueOnce( Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }), ) .mockResolvedValueOnce("ok"); const result = await runWithModelFallback({ cfg, provider: "openrouter", model: "meta-llama/llama-3.3-70b:free", run, }); expect(result.result).toBe("ok"); expect(run).toHaveBeenCalledTimes(2); expect(result.provider).toBe("openai"); expect(result.model).toBe("gpt-4.1-mini"); }); });