fix(agents): fail over on billing/credits errors
This commit is contained in:
@@ -18,6 +18,7 @@
|
|||||||
- Models/Auth: add MiniMax Anthropic-compatible API onboarding (minimax-api). (#590) — thanks @mneves75
|
- Models/Auth: add MiniMax Anthropic-compatible API onboarding (minimax-api). (#590) — thanks @mneves75
|
||||||
- Models: centralize model override validation + hooks Gmail warnings in doctor. (#602) — thanks @steipete
|
- Models: centralize model override validation + hooks Gmail warnings in doctor. (#602) — thanks @steipete
|
||||||
- Agents: avoid base-to-string error stringification in model fallback. (#604) — thanks @steipete
|
- Agents: avoid base-to-string error stringification in model fallback. (#604) — thanks @steipete
|
||||||
|
- Agents: treat billing/insufficient-credits errors as failover-worthy so model fallbacks kick in. (#486) — thanks @steipete
|
||||||
- Commands: harden slash command registry and list text-only commands in `/commands`.
|
- Commands: harden slash command registry and list text-only commands in `/commands`.
|
||||||
- Models/Auth: show per-agent auth candidates in `/model status`, and add `clawdbot models auth order {get,set,clear}` (per-agent auth rotation overrides). — thanks @steipete
|
- Models/Auth: show per-agent auth candidates in `/model status`, and add `clawdbot models auth order {get,set,clear}` (per-agent auth rotation overrides). — thanks @steipete
|
||||||
- Debugging: add raw model stream logging flags and document gateway watch mode.
|
- Debugging: add raw model stream logging flags and document gateway watch mode.
|
||||||
|
|||||||
@@ -56,6 +56,30 @@ describe("runWithModelFallback", () => {
|
|||||||
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("falls back on billing errors", async () => {
|
||||||
|
const cfg = makeCfg();
|
||||||
|
const run = vi
|
||||||
|
.fn()
|
||||||
|
.mockRejectedValueOnce(
|
||||||
|
new Error(
|
||||||
|
"LLM request rejected: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.mockResolvedValueOnce("ok");
|
||||||
|
|
||||||
|
const result = await runWithModelFallback({
|
||||||
|
cfg,
|
||||||
|
provider: "openai",
|
||||||
|
model: "gpt-4.1-mini",
|
||||||
|
run,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.result).toBe("ok");
|
||||||
|
expect(run).toHaveBeenCalledTimes(2);
|
||||||
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
||||||
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
||||||
|
});
|
||||||
|
|
||||||
it("appends the configured primary as a last fallback", async () => {
|
it("appends the configured primary as a last fallback", async () => {
|
||||||
const cfg = makeCfg({
|
const cfg = makeCfg({
|
||||||
agents: {
|
agents: {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
} from "./model-selection.js";
|
} from "./model-selection.js";
|
||||||
import {
|
import {
|
||||||
isAuthErrorMessage,
|
isAuthErrorMessage,
|
||||||
|
isBillingErrorMessage,
|
||||||
isRateLimitErrorMessage,
|
isRateLimitErrorMessage,
|
||||||
} from "./pi-embedded-helpers.js";
|
} from "./pi-embedded-helpers.js";
|
||||||
|
|
||||||
@@ -82,7 +83,7 @@ function isTimeoutErrorMessage(raw: string): boolean {
|
|||||||
|
|
||||||
function shouldFallbackForError(err: unknown): boolean {
|
function shouldFallbackForError(err: unknown): boolean {
|
||||||
const statusCode = getStatusCode(err);
|
const statusCode = getStatusCode(err);
|
||||||
if (statusCode && [401, 403, 429].includes(statusCode)) return true;
|
if (statusCode && [401, 402, 403, 429].includes(statusCode)) return true;
|
||||||
const code = getErrorCode(err).toUpperCase();
|
const code = getErrorCode(err).toUpperCase();
|
||||||
if (
|
if (
|
||||||
["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "ECONNABORTED"].includes(
|
["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "ECONNABORTED"].includes(
|
||||||
@@ -96,6 +97,7 @@ function shouldFallbackForError(err: unknown): boolean {
|
|||||||
return (
|
return (
|
||||||
isAuthErrorMessage(message) ||
|
isAuthErrorMessage(message) ||
|
||||||
isRateLimitErrorMessage(message) ||
|
isRateLimitErrorMessage(message) ||
|
||||||
|
isBillingErrorMessage(message) ||
|
||||||
isTimeoutErrorMessage(message)
|
isTimeoutErrorMessage(message)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest";
|
|||||||
import {
|
import {
|
||||||
buildBootstrapContextFiles,
|
buildBootstrapContextFiles,
|
||||||
formatAssistantErrorText,
|
formatAssistantErrorText,
|
||||||
|
isBillingErrorMessage,
|
||||||
isContextOverflowError,
|
isContextOverflowError,
|
||||||
isMessagingToolDuplicate,
|
isMessagingToolDuplicate,
|
||||||
normalizeTextForComparison,
|
normalizeTextForComparison,
|
||||||
@@ -215,6 +216,28 @@ describe("isContextOverflowError", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("isBillingErrorMessage", () => {
|
||||||
|
it("matches credit / payment failures", () => {
|
||||||
|
const samples = [
|
||||||
|
"Your credit balance is too low to access the Anthropic API.",
|
||||||
|
"insufficient credits",
|
||||||
|
"Payment Required",
|
||||||
|
"HTTP 402 Payment Required",
|
||||||
|
"plans & billing",
|
||||||
|
"billing: please upgrade your plan",
|
||||||
|
];
|
||||||
|
for (const sample of samples) {
|
||||||
|
expect(isBillingErrorMessage(sample)).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores unrelated errors", () => {
|
||||||
|
expect(isBillingErrorMessage("rate limit exceeded")).toBe(false);
|
||||||
|
expect(isBillingErrorMessage("invalid api key")).toBe(false);
|
||||||
|
expect(isBillingErrorMessage("context length exceeded")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("formatAssistantErrorText", () => {
|
describe("formatAssistantErrorText", () => {
|
||||||
const makeAssistantError = (errorMessage: string): AssistantMessage =>
|
const makeAssistantError = (errorMessage: string): AssistantMessage =>
|
||||||
({
|
({
|
||||||
|
|||||||
@@ -261,6 +261,30 @@ export function isRateLimitErrorMessage(raw: string): boolean {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isBillingErrorMessage(raw: string): boolean {
|
||||||
|
const value = raw.toLowerCase();
|
||||||
|
if (!value) return false;
|
||||||
|
return (
|
||||||
|
/\b402\b/.test(value) ||
|
||||||
|
value.includes("payment required") ||
|
||||||
|
value.includes("insufficient credits") ||
|
||||||
|
value.includes("credit balance") ||
|
||||||
|
value.includes("plans & billing") ||
|
||||||
|
(value.includes("billing") &&
|
||||||
|
(value.includes("upgrade") ||
|
||||||
|
value.includes("credits") ||
|
||||||
|
value.includes("payment") ||
|
||||||
|
value.includes("plan")))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isBillingAssistantError(
|
||||||
|
msg: AssistantMessage | undefined,
|
||||||
|
): boolean {
|
||||||
|
if (!msg || msg.stopReason !== "error") return false;
|
||||||
|
return isBillingErrorMessage(msg.errorMessage ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
export function isAuthErrorMessage(raw: string): boolean {
|
export function isAuthErrorMessage(raw: string): boolean {
|
||||||
const value = raw.toLowerCase();
|
const value = raw.toLowerCase();
|
||||||
if (!value) return false;
|
if (!value) return false;
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ import {
|
|||||||
formatAssistantErrorText,
|
formatAssistantErrorText,
|
||||||
isAuthAssistantError,
|
isAuthAssistantError,
|
||||||
isAuthErrorMessage,
|
isAuthErrorMessage,
|
||||||
|
isBillingAssistantError,
|
||||||
|
isBillingErrorMessage,
|
||||||
isContextOverflowError,
|
isContextOverflowError,
|
||||||
isGoogleModelApi,
|
isGoogleModelApi,
|
||||||
isRateLimitAssistantError,
|
isRateLimitAssistantError,
|
||||||
@@ -1421,7 +1423,8 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
(isAuthErrorMessage(errorText) ||
|
(isAuthErrorMessage(errorText) ||
|
||||||
isRateLimitErrorMessage(errorText)) &&
|
isRateLimitErrorMessage(errorText) ||
|
||||||
|
isBillingErrorMessage(errorText)) &&
|
||||||
(await advanceAuthProfile())
|
(await advanceAuthProfile())
|
||||||
) {
|
) {
|
||||||
continue;
|
continue;
|
||||||
@@ -1464,10 +1467,12 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
0;
|
0;
|
||||||
const authFailure = isAuthAssistantError(lastAssistant);
|
const authFailure = isAuthAssistantError(lastAssistant);
|
||||||
const rateLimitFailure = isRateLimitAssistantError(lastAssistant);
|
const rateLimitFailure = isRateLimitAssistantError(lastAssistant);
|
||||||
|
const billingFailure = isBillingAssistantError(lastAssistant);
|
||||||
|
|
||||||
// Treat timeout as potential rate limit (Antigravity hangs on rate limit)
|
// Treat timeout as potential rate limit (Antigravity hangs on rate limit)
|
||||||
const shouldRotate =
|
const shouldRotate =
|
||||||
(!aborted && (authFailure || rateLimitFailure)) || timedOut;
|
(!aborted && (authFailure || rateLimitFailure || billingFailure)) ||
|
||||||
|
timedOut;
|
||||||
|
|
||||||
if (shouldRotate) {
|
if (shouldRotate) {
|
||||||
// Mark current profile for cooldown before rotating
|
// Mark current profile for cooldown before rotating
|
||||||
@@ -1496,6 +1501,8 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
? "LLM request timed out."
|
? "LLM request timed out."
|
||||||
: rateLimitFailure
|
: rateLimitFailure
|
||||||
? "LLM request rate limited."
|
? "LLM request rate limited."
|
||||||
|
: billingFailure
|
||||||
|
? "LLM request payment required."
|
||||||
: "LLM request unauthorized.");
|
: "LLM request unauthorized.");
|
||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user