diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index d3a438800..4024e9d4a 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -58,6 +58,8 @@ If you have both an OAuth profile and an API key profile for the same provider, When a profile fails due to auth/rate‑limit errors (or a timeout that looks like rate limiting), Clawdbot marks it in cooldown and moves to the next profile. +Format/invalid‑request errors (for example Cloud Code Assist tool call ID +validation failures) are treated as failover‑worthy and use the same cooldowns. Cooldowns use exponential backoff: - 1 minute diff --git a/src/agents/auth-profiles.ts b/src/agents/auth-profiles.ts index 099d04d47..c5507e96a 100644 --- a/src/agents/auth-profiles.ts +++ b/src/agents/auth-profiles.ts @@ -74,6 +74,7 @@ export type AuthProfileCredential = export type AuthProfileFailureReason = | "auth" + | "format" | "rate_limit" | "billing" | "timeout" diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index a94fb8b3f..3bb59cbdd 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -15,6 +15,14 @@ describe("failover-error", () => { expect(resolveFailoverReasonFromError({ status: 408 })).toBe("timeout"); }); + it("infers format errors from error messages", () => { + expect( + resolveFailoverReasonFromError({ + message: "invalid request format: messages.1.content.1.tool_use.id", + }), + ).toBe("format"); + }); + it("infers timeout from common node error codes", () => { expect(resolveFailoverReasonFromError({ code: "ETIMEDOUT" })).toBe( "timeout", @@ -36,6 +44,15 @@ describe("failover-error", () => { expect(err?.model).toBe("claude-opus-4-5"); }); + it("coerces format errors with a 400 status", () => { + const err = coerceToFailoverError("invalid request format", { + provider: "google", + model: "cloud-code-assist", + }); + expect(err?.reason).toBe("format"); + expect(err?.status).toBe(400); + }); + it("describes non-Error values consistently", () => { const described = describeFailoverError(123); expect(described.message).toBe("123"); diff --git a/src/agents/failover-error.ts b/src/agents/failover-error.ts index fdadab0a7..2a9113f61 100644 --- a/src/agents/failover-error.ts +++ b/src/agents/failover-error.ts @@ -50,6 +50,8 @@ export function resolveFailoverStatus( return 401; case "timeout": return 408; + case "format": + return 400; default: return undefined; } diff --git a/src/agents/pi-embedded-helpers.test.ts b/src/agents/pi-embedded-helpers.test.ts index c280c1b86..a9f63fc9f 100644 --- a/src/agents/pi-embedded-helpers.test.ts +++ b/src/agents/pi-embedded-helpers.test.ts @@ -249,6 +249,7 @@ describe("isFailoverErrorMessage", () => { "429 rate limit exceeded", "Your credit balance is too low", "request timed out", + "invalid request format", ]; for (const sample of samples) { expect(isFailoverErrorMessage(sample)).toBe(true); @@ -263,9 +264,12 @@ describe("classifyFailoverReason", () => { expect(classifyFailoverReason("resource has been exhausted")).toBe( "rate_limit", ); + expect(classifyFailoverReason("invalid request format")).toBe("format"); expect(classifyFailoverReason("credit balance too low")).toBe("billing"); expect(classifyFailoverReason("deadline exceeded")).toBe("timeout"); - expect(classifyFailoverReason("string should match pattern")).toBeNull(); + expect(classifyFailoverReason("string should match pattern")).toBe( + "format", + ); expect(classifyFailoverReason("bad request")).toBeNull(); }); }); diff --git a/src/agents/pi-embedded-helpers.ts b/src/agents/pi-embedded-helpers.ts index 2e10029bf..1fdf46f84 100644 --- a/src/agents/pi-embedded-helpers.ts +++ b/src/agents/pi-embedded-helpers.ts @@ -276,47 +276,84 @@ export function isRateLimitAssistantError( msg: AssistantMessage | undefined, ): boolean { if (!msg || msg.stopReason !== "error") return false; - const raw = (msg.errorMessage ?? "").toLowerCase(); + return isRateLimitErrorMessage(msg.errorMessage ?? ""); +} + +type ErrorPattern = RegExp | string; + +const ERROR_PATTERNS = { + rateLimit: [ + /rate[_ ]limit|too many requests|429/, + "exceeded your current quota", + "resource has been exhausted", + "quota exceeded", + "resource_exhausted", + ], + timeout: [ + "timeout", + "timed out", + "deadline exceeded", + "context deadline exceeded", + ], + billing: [ + /\b402\b/, + "payment required", + "insufficient credits", + "credit balance", + "plans & billing", + ], + auth: [ + /invalid[_ ]?api[_ ]?key/, + "incorrect api key", + "invalid token", + "authentication", + "unauthorized", + "forbidden", + "access denied", + "expired", + "token has expired", + /\b401\b/, + /\b403\b/, + ], + format: [ + "invalid_request_error", + "string should match pattern", + "tool_use.id", + "tool_use_id", + "messages.1.content.1.tool_use.id", + "invalid request format", + ], +} as const; + +function matchesErrorPatterns( + raw: string, + patterns: readonly ErrorPattern[], +): boolean { if (!raw) return false; - return isRateLimitErrorMessage(raw); + const value = raw.toLowerCase(); + return patterns.some((pattern) => + pattern instanceof RegExp ? pattern.test(value) : value.includes(pattern), + ); } export function isRateLimitErrorMessage(raw: string): boolean { - const value = raw.toLowerCase(); - return ( - /rate[_ ]limit|too many requests|429/.test(value) || - value.includes("exceeded your current quota") || - value.includes("resource has been exhausted") || - value.includes("quota exceeded") || - value.includes("resource_exhausted") - ); + return matchesErrorPatterns(raw, ERROR_PATTERNS.rateLimit); } export function isTimeoutErrorMessage(raw: string): boolean { - const value = raw.toLowerCase(); - if (!value) return false; - return ( - value.includes("timeout") || - value.includes("timed out") || - value.includes("deadline exceeded") || - value.includes("context deadline exceeded") - ); + return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout); } export function isBillingErrorMessage(raw: string): boolean { const value = raw.toLowerCase(); if (!value) return false; + if (matchesErrorPatterns(value, ERROR_PATTERNS.billing)) return true; return ( - /\b402\b/.test(value) || - value.includes("payment required") || - value.includes("insufficient credits") || - value.includes("credit balance") || - value.includes("plans & billing") || - (value.includes("billing") && - (value.includes("upgrade") || - value.includes("credits") || - value.includes("payment") || - value.includes("plan"))) + value.includes("billing") && + (value.includes("upgrade") || + value.includes("credits") || + value.includes("payment") || + value.includes("plan")) ); } @@ -328,34 +365,11 @@ export function isBillingAssistantError( } export function isAuthErrorMessage(raw: string): boolean { - const value = raw.toLowerCase(); - if (!value) return false; - return ( - /invalid[_ ]?api[_ ]?key/.test(value) || - value.includes("incorrect api key") || - value.includes("invalid token") || - value.includes("authentication") || - value.includes("unauthorized") || - value.includes("forbidden") || - value.includes("access denied") || - value.includes("expired") || - value.includes("token has expired") || - /\b401\b/.test(value) || - /\b403\b/.test(value) - ); + return matchesErrorPatterns(raw, ERROR_PATTERNS.auth); } export function isCloudCodeAssistFormatError(raw: string): boolean { - const value = raw.toLowerCase(); - if (!value) return false; - return ( - value.includes("invalid_request_error") || - value.includes("string should match pattern") || - value.includes("tool_use.id") || - value.includes("tool_use_id") || - value.includes("messages.1.content.1.tool_use.id") || - value.includes("invalid request format") - ); + return matchesErrorPatterns(raw, ERROR_PATTERNS.format); } export function isAuthAssistantError( @@ -367,16 +381,18 @@ export function isAuthAssistantError( export type FailoverReason = | "auth" + | "format" | "rate_limit" | "billing" | "timeout" | "unknown"; export function classifyFailoverReason(raw: string): FailoverReason | null { - if (isAuthErrorMessage(raw)) return "auth"; if (isRateLimitErrorMessage(raw)) return "rate_limit"; + if (isCloudCodeAssistFormatError(raw)) return "format"; if (isBillingErrorMessage(raw)) return "billing"; if (isTimeoutErrorMessage(raw)) return "timeout"; + if (isAuthErrorMessage(raw)) return "auth"; return null; } diff --git a/src/agents/pi-embedded-runner.ts b/src/agents/pi-embedded-runner.ts index 444b084de..a3c2fd545 100644 --- a/src/agents/pi-embedded-runner.ts +++ b/src/agents/pi-embedded-runner.ts @@ -1533,9 +1533,7 @@ export async function runEmbeddedPiAgent(params: { : false; // Treat timeout as potential rate limit (Antigravity hangs on rate limit) - const shouldRotate = - (!aborted && (failoverFailure || cloudCodeAssistFormatError)) || - timedOut; + const shouldRotate = (!aborted && failoverFailure) || timedOut; if (shouldRotate) { // Mark current profile for cooldown before rotating