import type { ClawdbotConfig } from "../config/config.js"; import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js"; import { buildModelAliasIndex, modelKey, parseModelRef, resolveConfiguredModelRef, resolveModelRefFromString, } from "./model-selection.js"; import { isAuthErrorMessage, isRateLimitErrorMessage, } from "./pi-embedded-helpers.js"; type ModelCandidate = { provider: string; model: string; }; type FallbackAttempt = { provider: string; model: string; error: string; }; function isAbortError(err: unknown): boolean { if (!err || typeof err !== "object") return false; const name = "name" in err ? String(err.name) : ""; if (name === "AbortError") return true; const message = "message" in err && typeof err.message === "string" ? err.message.toLowerCase() : ""; return message.includes("aborted"); } function getStatusCode(err: unknown): number | null { if (!err || typeof err !== "object") return null; const candidate = (err as { status?: unknown; statusCode?: unknown }).status ?? (err as { statusCode?: unknown }).statusCode; if (typeof candidate === "number") return candidate; if (typeof candidate === "string" && /^\d+$/.test(candidate)) { return Number(candidate); } return null; } function getErrorCode(err: unknown): string { if (!err || typeof err !== "object") return ""; const candidate = (err as { code?: unknown }).code; return typeof candidate === "string" ? candidate : ""; } function getErrorMessage(err: unknown): string { if (err instanceof Error) return err.message; if (typeof err === "string") return err; if ( typeof err === "number" || typeof err === "boolean" || typeof err === "bigint" ) { return String(err); } if (typeof err === "symbol") return err.description ?? ""; if (err && typeof err === "object") { const message = (err as { message?: unknown }).message; if (typeof message === "string") return message; } return ""; } function isTimeoutErrorMessage(raw: string): boolean { const value = raw.toLowerCase(); return ( value.includes("timeout") || value.includes("timed out") || value.includes("deadline exceeded") || value.includes("context deadline exceeded") ); } function shouldFallbackForError(err: unknown): boolean { const statusCode = getStatusCode(err); if (statusCode && [401, 403, 429].includes(statusCode)) return true; const code = getErrorCode(err).toUpperCase(); if ( ["ETIMEDOUT", "ESOCKETTIMEDOUT", "ECONNRESET", "ECONNABORTED"].includes( code, ) ) { return true; } const message = getErrorMessage(err); if (!message) return false; return ( isAuthErrorMessage(message) || isRateLimitErrorMessage(message) || isTimeoutErrorMessage(message) ); } function buildAllowedModelKeys( cfg: ClawdbotConfig | undefined, defaultProvider: string, ): Set | null { const rawAllowlist = (() => { const modelMap = cfg?.agents?.defaults?.models ?? {}; return Object.keys(modelMap); })(); if (rawAllowlist.length === 0) return null; const keys = new Set(); for (const raw of rawAllowlist) { const parsed = parseModelRef(String(raw ?? ""), defaultProvider); if (!parsed) continue; keys.add(modelKey(parsed.provider, parsed.model)); } return keys.size > 0 ? keys : null; } function resolveImageFallbackCandidates(params: { cfg: ClawdbotConfig | undefined; defaultProvider: string; modelOverride?: string; }): ModelCandidate[] { const aliasIndex = buildModelAliasIndex({ cfg: params.cfg ?? {}, defaultProvider: params.defaultProvider, }); const allowlist = buildAllowedModelKeys(params.cfg, params.defaultProvider); const seen = new Set(); const candidates: ModelCandidate[] = []; const addCandidate = ( candidate: ModelCandidate, enforceAllowlist: boolean, ) => { if (!candidate.provider || !candidate.model) return; const key = modelKey(candidate.provider, candidate.model); if (seen.has(key)) return; if (enforceAllowlist && allowlist && !allowlist.has(key)) return; seen.add(key); candidates.push(candidate); }; const addRaw = (raw: string, enforceAllowlist: boolean) => { const resolved = resolveModelRefFromString({ raw: String(raw ?? ""), defaultProvider: params.defaultProvider, aliasIndex, }); if (!resolved) return; addCandidate(resolved.ref, enforceAllowlist); }; if (params.modelOverride?.trim()) { addRaw(params.modelOverride, false); } else { const imageModel = params.cfg?.agents?.defaults?.imageModel as | { primary?: string } | string | undefined; const primary = typeof imageModel === "string" ? imageModel.trim() : imageModel?.primary; if (primary?.trim()) addRaw(primary, false); } const imageFallbacks = (() => { const imageModel = params.cfg?.agents?.defaults?.imageModel as | { fallbacks?: string[] } | string | undefined; if (imageModel && typeof imageModel === "object") { return imageModel.fallbacks ?? []; } return []; })(); for (const raw of imageFallbacks) { addRaw(raw, true); } return candidates; } function resolveFallbackCandidates(params: { cfg: ClawdbotConfig | undefined; provider: string; model: string; }): ModelCandidate[] { const provider = params.provider.trim() || DEFAULT_PROVIDER; const model = params.model.trim() || DEFAULT_MODEL; const primary = params.cfg ? resolveConfiguredModelRef({ cfg: params.cfg, defaultProvider: DEFAULT_PROVIDER, defaultModel: DEFAULT_MODEL, }) : null; const aliasIndex = buildModelAliasIndex({ cfg: params.cfg ?? {}, defaultProvider: DEFAULT_PROVIDER, }); const allowlist = buildAllowedModelKeys(params.cfg, DEFAULT_PROVIDER); const seen = new Set(); const candidates: ModelCandidate[] = []; const addCandidate = ( candidate: ModelCandidate, enforceAllowlist: boolean, ) => { if (!candidate.provider || !candidate.model) return; const key = modelKey(candidate.provider, candidate.model); if (seen.has(key)) return; if (enforceAllowlist && allowlist && !allowlist.has(key)) return; seen.add(key); candidates.push(candidate); }; addCandidate({ provider, model }, false); const modelFallbacks = (() => { const model = params.cfg?.agents?.defaults?.model as | { fallbacks?: string[] } | string | undefined; if (model && typeof model === "object") return model.fallbacks ?? []; return []; })(); for (const raw of modelFallbacks) { const resolved = resolveModelRefFromString({ raw: String(raw ?? ""), defaultProvider: DEFAULT_PROVIDER, aliasIndex, }); if (!resolved) continue; addCandidate(resolved.ref, true); } if (primary?.provider && primary.model) { addCandidate({ provider: primary.provider, model: primary.model }, false); } return candidates; } export async function runWithModelFallback(params: { cfg: ClawdbotConfig | undefined; provider: string; model: string; run: (provider: string, model: string) => Promise; onError?: (attempt: { provider: string; model: string; error: unknown; attempt: number; total: number; }) => void | Promise; }): Promise<{ result: T; provider: string; model: string; attempts: FallbackAttempt[]; }> { const candidates = resolveFallbackCandidates(params); const attempts: FallbackAttempt[] = []; let lastError: unknown; for (let i = 0; i < candidates.length; i += 1) { const candidate = candidates[i] as ModelCandidate; try { const result = await params.run(candidate.provider, candidate.model); return { result, provider: candidate.provider, model: candidate.model, attempts, }; } catch (err) { if (isAbortError(err)) throw err; const shouldFallback = shouldFallbackForError(err); if (!shouldFallback) throw err; lastError = err; attempts.push({ provider: candidate.provider, model: candidate.model, error: err instanceof Error ? err.message : String(err), }); await params.onError?.({ provider: candidate.provider, model: candidate.model, error: err, attempt: i + 1, total: candidates.length, }); } } if (attempts.length <= 1 && lastError) throw lastError; const summary = attempts.length > 0 ? attempts .map( (attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`, ) .join(" | ") : "unknown"; throw new Error( `All models failed (${attempts.length || candidates.length}): ${summary}`, { cause: lastError instanceof Error ? lastError : undefined }, ); } export async function runWithImageModelFallback(params: { cfg: ClawdbotConfig | undefined; modelOverride?: string; run: (provider: string, model: string) => Promise; onError?: (attempt: { provider: string; model: string; error: unknown; attempt: number; total: number; }) => void | Promise; }): Promise<{ result: T; provider: string; model: string; attempts: FallbackAttempt[]; }> { const candidates = resolveImageFallbackCandidates({ cfg: params.cfg, defaultProvider: DEFAULT_PROVIDER, modelOverride: params.modelOverride, }); if (candidates.length === 0) { throw new Error( "No image model configured. Set agents.defaults.imageModel.primary or agents.defaults.imageModel.fallbacks.", ); } const attempts: FallbackAttempt[] = []; let lastError: unknown; for (let i = 0; i < candidates.length; i += 1) { const candidate = candidates[i] as ModelCandidate; try { const result = await params.run(candidate.provider, candidate.model); return { result, provider: candidate.provider, model: candidate.model, attempts, }; } catch (err) { if (isAbortError(err)) throw err; lastError = err; attempts.push({ provider: candidate.provider, model: candidate.model, error: err instanceof Error ? err.message : String(err), }); await params.onError?.({ provider: candidate.provider, model: candidate.model, error: err, attempt: i + 1, total: candidates.length, }); } } if (attempts.length <= 1 && lastError) throw lastError; const summary = attempts.length > 0 ? attempts .map( (attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`, ) .join(" | ") : "unknown"; throw new Error( `All image models failed (${attempts.length || candidates.length}): ${summary}`, { cause: lastError instanceof Error ? lastError : undefined }, ); }