136 lines
3.5 KiB
TypeScript
136 lines
3.5 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { ClawdbotConfig } from "../config/config.js";
|
|
import { runWithModelFallback } from "./model-fallback.js";
|
|
|
|
function makeCfg(overrides: Partial<ClawdbotConfig> = {}): ClawdbotConfig {
|
|
return {
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
fallbacks: ["anthropic/claude-haiku-3-5"],
|
|
},
|
|
},
|
|
},
|
|
...overrides,
|
|
} as ClawdbotConfig;
|
|
}
|
|
|
|
describe("runWithModelFallback", () => {
|
|
it("does not fall back on non-auth errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(new Error("bad request"))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
await expect(
|
|
runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
}),
|
|
).rejects.toThrow("bad request");
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("falls back on auth errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("nope"), { status: 401 }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on 402 payment required", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
Object.assign(new Error("payment required"), { status: 402 }),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on billing errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
new Error(
|
|
"LLM request rejected: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.",
|
|
),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("appends the configured primary as a last fallback", async () => {
|
|
const cfg = makeCfg({
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
fallbacks: [],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openrouter",
|
|
model: "meta-llama/llama-3.3-70b:free",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(result.provider).toBe("openai");
|
|
expect(result.model).toBe("gpt-4.1-mini");
|
|
});
|
|
});
|