419 lines
11 KiB
TypeScript
419 lines
11 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
|
|
import type { ClawdbotConfig } from "../config/config.js";
|
|
import { runWithModelFallback } from "./model-fallback.js";
|
|
|
|
function makeCfg(overrides: Partial<ClawdbotConfig> = {}): ClawdbotConfig {
|
|
return {
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
fallbacks: ["anthropic/claude-haiku-3-5"],
|
|
},
|
|
},
|
|
},
|
|
...overrides,
|
|
} as ClawdbotConfig;
|
|
}
|
|
|
|
describe("runWithModelFallback", () => {
|
|
it("does not fall back on non-auth errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi.fn().mockRejectedValueOnce(new Error("bad request")).mockResolvedValueOnce("ok");
|
|
|
|
await expect(
|
|
runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
}),
|
|
).rejects.toThrow("bad request");
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("falls back on auth errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("nope"), { status: 401 }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on 402 payment required", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("payment required"), { status: 402 }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on billing errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
new Error(
|
|
"LLM request rejected: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.",
|
|
),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on credential validation errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(new Error('No credentials found for profile "anthropic:default".'))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "anthropic",
|
|
model: "claude-opus-4",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("does not append configured primary when fallbacksOverride is set", async () => {
|
|
const cfg = makeCfg({
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const run = vi
|
|
.fn()
|
|
.mockImplementation(() => Promise.reject(Object.assign(new Error("nope"), { status: 401 })));
|
|
|
|
await expect(
|
|
runWithModelFallback({
|
|
cfg,
|
|
provider: "anthropic",
|
|
model: "claude-opus-4-5",
|
|
fallbacksOverride: ["anthropic/claude-haiku-3-5"],
|
|
run,
|
|
}),
|
|
).rejects.toThrow("All models failed");
|
|
|
|
expect(run.mock.calls).toEqual([
|
|
["anthropic", "claude-opus-4-5"],
|
|
["anthropic", "claude-haiku-3-5"],
|
|
]);
|
|
});
|
|
|
|
it("uses fallbacksOverride instead of agents.defaults.model.fallbacks", async () => {
|
|
const cfg = {
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
fallbacks: ["openai/gpt-5.2"],
|
|
},
|
|
},
|
|
},
|
|
} as ClawdbotConfig;
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
const res = await runWithModelFallback({
|
|
cfg,
|
|
provider: "anthropic",
|
|
model: "claude-opus-4-5",
|
|
fallbacksOverride: ["openai/gpt-4.1"],
|
|
run: async (provider, model) => {
|
|
calls.push({ provider, model });
|
|
if (provider === "anthropic") {
|
|
throw Object.assign(new Error("nope"), { status: 401 });
|
|
}
|
|
if (provider === "openai" && model === "gpt-4.1") {
|
|
return "ok";
|
|
}
|
|
throw new Error(`unexpected candidate: ${provider}/${model}`);
|
|
},
|
|
});
|
|
|
|
expect(res.result).toBe("ok");
|
|
expect(calls).toEqual([
|
|
{ provider: "anthropic", model: "claude-opus-4-5" },
|
|
{ provider: "openai", model: "gpt-4.1" },
|
|
]);
|
|
});
|
|
|
|
it("treats an empty fallbacksOverride as disabling global fallbacks", async () => {
|
|
const cfg = {
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
fallbacks: ["openai/gpt-5.2"],
|
|
},
|
|
},
|
|
},
|
|
} as ClawdbotConfig;
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
await expect(
|
|
runWithModelFallback({
|
|
cfg,
|
|
provider: "anthropic",
|
|
model: "claude-opus-4-5",
|
|
fallbacksOverride: [],
|
|
run: async (provider, model) => {
|
|
calls.push({ provider, model });
|
|
throw new Error("primary failed");
|
|
},
|
|
}),
|
|
).rejects.toThrow("primary failed");
|
|
|
|
expect(calls).toEqual([{ provider: "anthropic", model: "claude-opus-4-5" }]);
|
|
});
|
|
|
|
it("defaults provider/model when missing (regression #946)", async () => {
|
|
const cfg = makeCfg({
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
fallbacks: [],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: undefined as unknown as string,
|
|
model: undefined as unknown as string,
|
|
run: async (provider, model) => {
|
|
calls.push({ provider, model });
|
|
return "ok";
|
|
},
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(calls).toEqual([{ provider: "openai", model: "gpt-4.1-mini" }]);
|
|
});
|
|
|
|
it("falls back on missing API key errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(new Error("No API key found for profile openai."))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on lowercase credential errors", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(new Error("no api key found for profile openai"))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on timeout abort errors", async () => {
|
|
const cfg = makeCfg();
|
|
const timeoutCause = Object.assign(new Error("request timed out"), { name: "TimeoutError" });
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
Object.assign(new Error("aborted"), { name: "AbortError", cause: timeoutCause }),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on abort errors with timeout reasons", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
Object.assign(new Error("aborted"), { name: "AbortError", reason: "deadline exceeded" }),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back when message says aborted but error is a timeout", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("request aborted"), { code: "ETIMEDOUT" }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("falls back on provider abort errors with request-aborted messages", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(
|
|
Object.assign(new Error("Request was aborted"), { name: "AbortError" }),
|
|
)
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
});
|
|
|
|
it("does not fall back on user aborts", async () => {
|
|
const cfg = makeCfg();
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("aborted"), { name: "AbortError" }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
await expect(
|
|
runWithModelFallback({
|
|
cfg,
|
|
provider: "openai",
|
|
model: "gpt-4.1-mini",
|
|
run,
|
|
}),
|
|
).rejects.toThrow("aborted");
|
|
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it("appends the configured primary as a last fallback", async () => {
|
|
const cfg = makeCfg({
|
|
agents: {
|
|
defaults: {
|
|
model: {
|
|
primary: "openai/gpt-4.1-mini",
|
|
fallbacks: [],
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const run = vi
|
|
.fn()
|
|
.mockRejectedValueOnce(Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }))
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
const result = await runWithModelFallback({
|
|
cfg,
|
|
provider: "openrouter",
|
|
model: "meta-llama/llama-3.3-70b:free",
|
|
run,
|
|
});
|
|
|
|
expect(result.result).toBe("ok");
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
expect(result.provider).toBe("openai");
|
|
expect(result.model).toBe("gpt-4.1-mini");
|
|
});
|
|
});
|