diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index fcaef512f..4dc1dbf61 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -83,7 +83,10 @@ State is stored in `auth-profiles.json` under `usageStats`: If all profiles for a provider fail, Clawdbot moves to the next model in `agents.defaults.model.fallbacks`. This applies to auth failures, rate limits, and -timeouts that exhausted profile rotation. +timeouts that exhausted profile rotation (other errors do not advance fallback). + +When a run starts with a model override (hooks or CLI), fallbacks still end at +`agents.defaults.model.primary` after trying any configured fallbacks. ## Related config diff --git a/src/agents/model-selection.test.ts b/src/agents/model-selection.test.ts index a36a2e139..da6f30c5a 100644 --- a/src/agents/model-selection.test.ts +++ b/src/agents/model-selection.test.ts @@ -6,6 +6,7 @@ import { buildAllowedModelSet, modelKey, parseModelRef, + resolveAllowedModelRef, resolveHooksGmailModel, } from "./model-selection.js"; @@ -143,3 +144,62 @@ describe("resolveHooksGmailModel", () => { }); }); }); + +describe("resolveAllowedModelRef", () => { + it("resolves aliases when allowed", () => { + const cfg = { + agents: { + defaults: { + models: { + "anthropic/claude-sonnet-4-1": { alias: "Sonnet" }, + }, + }, + }, + } satisfies ClawdbotConfig; + const resolved = resolveAllowedModelRef({ + cfg, + catalog: [ + { + provider: "anthropic", + id: "claude-sonnet-4-1", + name: "Sonnet", + }, + ], + raw: "Sonnet", + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-5", + }); + expect("error" in resolved).toBe(false); + if ("ref" in resolved) { + expect(resolved.ref).toEqual({ + provider: "anthropic", + model: "claude-sonnet-4-1", + }); + } + }); + + it("rejects disallowed models", () => { + const cfg = { + agents: { + defaults: { + models: { + "openai/gpt-4": { alias: "GPT4" }, + }, + }, + }, + } satisfies ClawdbotConfig; + const resolved = resolveAllowedModelRef({ + cfg, + catalog: [ + { provider: "openai", id: "gpt-4", name: "GPT-4" }, + { provider: "anthropic", id: "claude-sonnet-4-1", name: "Sonnet" }, + ], + raw: "anthropic/claude-sonnet-4-1", + defaultProvider: "openai", + defaultModel: "gpt-4", + }); + expect(resolved).toEqual({ + error: "model not allowed: anthropic/claude-sonnet-4-1", + }); + }); +}); diff --git a/src/agents/model-selection.ts b/src/agents/model-selection.ts index 75b2035ad..57841c6fa 100644 --- a/src/agents/model-selection.ts +++ b/src/agents/model-selection.ts @@ -197,6 +197,76 @@ export function buildAllowedModelSet(params: { return { allowAny: false, allowedCatalog, allowedKeys }; } +export type ModelRefStatus = { + key: string; + inCatalog: boolean; + allowAny: boolean; + allowed: boolean; +}; + +export function getModelRefStatus(params: { + cfg: ClawdbotConfig; + catalog: ModelCatalogEntry[]; + ref: ModelRef; + defaultProvider: string; + defaultModel?: string; +}): ModelRefStatus { + const allowed = buildAllowedModelSet({ + cfg: params.cfg, + catalog: params.catalog, + defaultProvider: params.defaultProvider, + defaultModel: params.defaultModel, + }); + const key = modelKey(params.ref.provider, params.ref.model); + return { + key, + inCatalog: params.catalog.some( + (entry) => modelKey(entry.provider, entry.id) === key, + ), + allowAny: allowed.allowAny, + allowed: allowed.allowAny || allowed.allowedKeys.has(key), + }; +} + +export function resolveAllowedModelRef(params: { + cfg: ClawdbotConfig; + catalog: ModelCatalogEntry[]; + raw: string; + defaultProvider: string; + defaultModel?: string; +}): + | { ref: ModelRef; key: string } + | { + error: string; + } { + const trimmed = params.raw.trim(); + if (!trimmed) return { error: "invalid model: empty" }; + + const aliasIndex = buildModelAliasIndex({ + cfg: params.cfg, + defaultProvider: params.defaultProvider, + }); + const resolved = resolveModelRefFromString({ + raw: trimmed, + defaultProvider: params.defaultProvider, + aliasIndex, + }); + if (!resolved) return { error: `invalid model: ${trimmed}` }; + + const status = getModelRefStatus({ + cfg: params.cfg, + catalog: params.catalog, + ref: resolved.ref, + defaultProvider: params.defaultProvider, + defaultModel: params.defaultModel, + }); + if (!status.allowed) { + return { error: `model not allowed: ${status.key}` }; + } + + return { ref: resolved.ref, key: status.key }; +} + export function resolveThinkingDefault(params: { cfg: ClawdbotConfig; provider: string; diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index f26bc61bb..64807b20b 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -8,6 +8,13 @@ import { resolveAgentWorkspaceDir, resolveDefaultAgentId, } from "../agents/agent-scope.js"; +import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js"; +import { loadModelCatalog } from "../agents/model-catalog.js"; +import { + getModelRefStatus, + resolveConfiguredModelRef, + resolveHooksGmailModel, +} from "../agents/model-selection.js"; import { buildWorkspaceSkillStatus } from "../agents/skills-status.js"; import type { ClawdbotConfig } from "../config/config.js"; import { @@ -201,6 +208,48 @@ export async function doctorCommand( await noteSecurityWarnings(cfg); + if (cfg.hooks?.gmail?.model?.trim()) { + const hooksModelRef = resolveHooksGmailModel({ + cfg, + defaultProvider: DEFAULT_PROVIDER, + }); + if (!hooksModelRef) { + note( + `- hooks.gmail.model "${cfg.hooks.gmail.model}" could not be resolved`, + "Hooks", + ); + } else { + const { provider: defaultProvider, model: defaultModel } = + resolveConfiguredModelRef({ + cfg, + defaultProvider: DEFAULT_PROVIDER, + defaultModel: DEFAULT_MODEL, + }); + const catalog = await loadModelCatalog({ config: cfg }); + const status = getModelRefStatus({ + cfg, + catalog, + ref: hooksModelRef, + defaultProvider, + defaultModel, + }); + const warnings: string[] = []; + if (!status.allowed) { + warnings.push( + `- hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`, + ); + } + if (!status.inCatalog) { + warnings.push( + `- hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`, + ); + } + if (warnings.length > 0) { + note(warnings.join("\n"), "Hooks"); + } + } + } + if ( options.nonInteractive !== true && process.platform === "linux" && diff --git a/src/cron/isolated-agent.ts b/src/cron/isolated-agent.ts index 139252a69..e4b393daa 100644 --- a/src/cron/isolated-agent.ts +++ b/src/cron/isolated-agent.ts @@ -9,12 +9,10 @@ import { import { loadModelCatalog } from "../agents/model-catalog.js"; import { runWithModelFallback } from "../agents/model-fallback.js"; import { - buildAllowedModelSet, - buildModelAliasIndex, - modelKey, + getModelRefStatus, + resolveAllowedModelRef, resolveConfiguredModelRef, resolveHooksGmailModel, - resolveModelRefFromString, resolveThinkingDefault, } from "../agents/model-selection.js"; import { runEmbeddedPiAgent } from "../agents/pi-embedded.js"; @@ -301,14 +299,14 @@ export async function runCronIsolatedAgentTurn(params: { }) : null; if (hooksGmailModelRef) { - const allowed = buildAllowedModelSet({ + const status = getModelRefStatus({ cfg: params.cfg, catalog: await loadCatalog(), + ref: hooksGmailModelRef, defaultProvider: resolvedDefault.provider, defaultModel: resolvedDefault.model, }); - const key = modelKey(hooksGmailModelRef.provider, hooksGmailModelRef.model); - if (allowed.allowAny || allowed.allowedKeys.has(key)) { + if (status.allowed) { provider = hooksGmailModelRef.provider; model = hooksGmailModelRef.model; } @@ -321,34 +319,15 @@ export async function runCronIsolatedAgentTurn(params: { if (typeof modelOverrideRaw !== "string") { return { status: "error", error: "invalid model: expected string" }; } - const trimmed = modelOverrideRaw.trim(); - if (!trimmed) { - return { status: "error", error: "invalid model: empty" }; - } - const aliasIndex = buildModelAliasIndex({ - cfg: params.cfg, - defaultProvider: resolvedDefault.provider, - }); - const resolvedOverride = resolveModelRefFromString({ - raw: trimmed, - defaultProvider: resolvedDefault.provider, - aliasIndex, - }); - if (!resolvedOverride) { - return { status: "error", error: `invalid model: ${trimmed}` }; - } - const allowed = buildAllowedModelSet({ + const resolvedOverride = resolveAllowedModelRef({ cfg: params.cfg, catalog: await loadCatalog(), + raw: modelOverrideRaw, defaultProvider: resolvedDefault.provider, defaultModel: resolvedDefault.model, }); - const key = modelKey( - resolvedOverride.ref.provider, - resolvedOverride.ref.model, - ); - if (!allowed.allowAny && !allowed.allowedKeys.has(key)) { - return { status: "error", error: `model not allowed: ${key}` }; + if ("error" in resolvedOverride) { + return { status: "error", error: resolvedOverride.error }; } provider = resolvedOverride.ref.provider; model = resolvedOverride.ref.model; diff --git a/src/gateway/server.ts b/src/gateway/server.ts index 55587b79b..4bd840e66 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -10,8 +10,7 @@ import { resetModelCatalogCacheForTest, } from "../agents/model-catalog.js"; import { - buildAllowedModelSet, - modelKey, + getModelRefStatus, resolveConfiguredModelRef, resolveHooksGmailModel, } from "../agents/model-selection.js"; @@ -1782,22 +1781,21 @@ export async function startGatewayServer( defaultModel: DEFAULT_MODEL, }); const catalog = await loadModelCatalog({ config: cfgAtStart }); - const key = modelKey(hooksModelRef.provider, hooksModelRef.model); - const allowed = buildAllowedModelSet({ + const status = getModelRefStatus({ cfg: cfgAtStart, catalog, + ref: hooksModelRef, defaultProvider, defaultModel, }); - if (!allowed.allowAny && !allowed.allowedKeys.has(key)) { + if (!status.allowed) { logHooks.warn( - `hooks.gmail.model "${key}" not in agents.defaults.models allowlist (will use primary instead)`, + `hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`, ); } - const inCatalog = catalog.some((e) => modelKey(e.provider, e.id) === key); - if (!inCatalog) { + if (!status.inCatalog) { logHooks.warn( - `hooks.gmail.model "${key}" not in the model catalog (may fail at runtime)`, + `hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`, ); } } diff --git a/src/gateway/sessions-patch.ts b/src/gateway/sessions-patch.ts index 5b6b78bde..f28c8d2e0 100644 --- a/src/gateway/sessions-patch.ts +++ b/src/gateway/sessions-patch.ts @@ -3,11 +3,8 @@ import { randomUUID } from "node:crypto"; import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js"; import type { ModelCatalogEntry } from "../agents/model-catalog.js"; import { - buildAllowedModelSet, - buildModelAliasIndex, - modelKey, + resolveAllowedModelRef, resolveConfiguredModelRef, - resolveModelRefFromString, } from "../agents/model-selection.js"; import { normalizeGroupActivation } from "../auto-reply/group-activation.js"; import { @@ -168,17 +165,6 @@ export async function applySessionsPatchToStore(params: { defaultProvider: DEFAULT_PROVIDER, defaultModel: DEFAULT_MODEL, }); - const aliasIndex = buildModelAliasIndex({ - cfg, - defaultProvider: resolvedDefault.provider, - }); - const resolved = resolveModelRefFromString({ - raw: trimmed, - defaultProvider: resolvedDefault.provider, - aliasIndex, - }); - if (!resolved) return invalid(`invalid model: ${trimmed}`); - if (!params.loadGatewayModelCatalog) { return { ok: false, @@ -189,15 +175,15 @@ export async function applySessionsPatchToStore(params: { }; } const catalog = await params.loadGatewayModelCatalog(); - const allowed = buildAllowedModelSet({ + const resolved = resolveAllowedModelRef({ cfg, catalog, + raw: trimmed, defaultProvider: resolvedDefault.provider, defaultModel: resolvedDefault.model, }); - const key = modelKey(resolved.ref.provider, resolved.ref.model); - if (!allowed.allowAny && !allowed.allowedKeys.has(key)) { - return invalid(`model not allowed: ${key}`); + if ("error" in resolved) { + return invalid(resolved.error); } if ( resolved.ref.provider === resolvedDefault.provider &&