refactor: centralize model override validation

This commit is contained in:
Peter Steinberger
2026-01-09 20:07:20 +01:00
parent f0a909f6dd
commit 7e81980747
7 changed files with 204 additions and 59 deletions

View File

@@ -83,7 +83,10 @@ State is stored in `auth-profiles.json` under `usageStats`:
If all profiles for a provider fail, Clawdbot moves to the next model in
`agents.defaults.model.fallbacks`. This applies to auth failures, rate limits, and
timeouts that exhausted profile rotation.
timeouts that exhausted profile rotation (other errors do not advance fallback).
When a run starts with a model override (hooks or CLI), fallbacks still end at
`agents.defaults.model.primary` after trying any configured fallbacks.
## Related config

View File

@@ -6,6 +6,7 @@ import {
buildAllowedModelSet,
modelKey,
parseModelRef,
resolveAllowedModelRef,
resolveHooksGmailModel,
} from "./model-selection.js";
@@ -143,3 +144,62 @@ describe("resolveHooksGmailModel", () => {
});
});
});
describe("resolveAllowedModelRef", () => {
it("resolves aliases when allowed", () => {
const cfg = {
agents: {
defaults: {
models: {
"anthropic/claude-sonnet-4-1": { alias: "Sonnet" },
},
},
},
} satisfies ClawdbotConfig;
const resolved = resolveAllowedModelRef({
cfg,
catalog: [
{
provider: "anthropic",
id: "claude-sonnet-4-1",
name: "Sonnet",
},
],
raw: "Sonnet",
defaultProvider: "anthropic",
defaultModel: "claude-opus-4-5",
});
expect("error" in resolved).toBe(false);
if ("ref" in resolved) {
expect(resolved.ref).toEqual({
provider: "anthropic",
model: "claude-sonnet-4-1",
});
}
});
it("rejects disallowed models", () => {
const cfg = {
agents: {
defaults: {
models: {
"openai/gpt-4": { alias: "GPT4" },
},
},
},
} satisfies ClawdbotConfig;
const resolved = resolveAllowedModelRef({
cfg,
catalog: [
{ provider: "openai", id: "gpt-4", name: "GPT-4" },
{ provider: "anthropic", id: "claude-sonnet-4-1", name: "Sonnet" },
],
raw: "anthropic/claude-sonnet-4-1",
defaultProvider: "openai",
defaultModel: "gpt-4",
});
expect(resolved).toEqual({
error: "model not allowed: anthropic/claude-sonnet-4-1",
});
});
});

View File

@@ -197,6 +197,76 @@ export function buildAllowedModelSet(params: {
return { allowAny: false, allowedCatalog, allowedKeys };
}
export type ModelRefStatus = {
key: string;
inCatalog: boolean;
allowAny: boolean;
allowed: boolean;
};
export function getModelRefStatus(params: {
cfg: ClawdbotConfig;
catalog: ModelCatalogEntry[];
ref: ModelRef;
defaultProvider: string;
defaultModel?: string;
}): ModelRefStatus {
const allowed = buildAllowedModelSet({
cfg: params.cfg,
catalog: params.catalog,
defaultProvider: params.defaultProvider,
defaultModel: params.defaultModel,
});
const key = modelKey(params.ref.provider, params.ref.model);
return {
key,
inCatalog: params.catalog.some(
(entry) => modelKey(entry.provider, entry.id) === key,
),
allowAny: allowed.allowAny,
allowed: allowed.allowAny || allowed.allowedKeys.has(key),
};
}
export function resolveAllowedModelRef(params: {
cfg: ClawdbotConfig;
catalog: ModelCatalogEntry[];
raw: string;
defaultProvider: string;
defaultModel?: string;
}):
| { ref: ModelRef; key: string }
| {
error: string;
} {
const trimmed = params.raw.trim();
if (!trimmed) return { error: "invalid model: empty" };
const aliasIndex = buildModelAliasIndex({
cfg: params.cfg,
defaultProvider: params.defaultProvider,
});
const resolved = resolveModelRefFromString({
raw: trimmed,
defaultProvider: params.defaultProvider,
aliasIndex,
});
if (!resolved) return { error: `invalid model: ${trimmed}` };
const status = getModelRefStatus({
cfg: params.cfg,
catalog: params.catalog,
ref: resolved.ref,
defaultProvider: params.defaultProvider,
defaultModel: params.defaultModel,
});
if (!status.allowed) {
return { error: `model not allowed: ${status.key}` };
}
return { ref: resolved.ref, key: status.key };
}
export function resolveThinkingDefault(params: {
cfg: ClawdbotConfig;
provider: string;

View File

@@ -8,6 +8,13 @@ import {
resolveAgentWorkspaceDir,
resolveDefaultAgentId,
} from "../agents/agent-scope.js";
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
import { loadModelCatalog } from "../agents/model-catalog.js";
import {
getModelRefStatus,
resolveConfiguredModelRef,
resolveHooksGmailModel,
} from "../agents/model-selection.js";
import { buildWorkspaceSkillStatus } from "../agents/skills-status.js";
import type { ClawdbotConfig } from "../config/config.js";
import {
@@ -201,6 +208,48 @@ export async function doctorCommand(
await noteSecurityWarnings(cfg);
if (cfg.hooks?.gmail?.model?.trim()) {
const hooksModelRef = resolveHooksGmailModel({
cfg,
defaultProvider: DEFAULT_PROVIDER,
});
if (!hooksModelRef) {
note(
`- hooks.gmail.model "${cfg.hooks.gmail.model}" could not be resolved`,
"Hooks",
);
} else {
const { provider: defaultProvider, model: defaultModel } =
resolveConfiguredModelRef({
cfg,
defaultProvider: DEFAULT_PROVIDER,
defaultModel: DEFAULT_MODEL,
});
const catalog = await loadModelCatalog({ config: cfg });
const status = getModelRefStatus({
cfg,
catalog,
ref: hooksModelRef,
defaultProvider,
defaultModel,
});
const warnings: string[] = [];
if (!status.allowed) {
warnings.push(
`- hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
);
}
if (!status.inCatalog) {
warnings.push(
`- hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
);
}
if (warnings.length > 0) {
note(warnings.join("\n"), "Hooks");
}
}
}
if (
options.nonInteractive !== true &&
process.platform === "linux" &&

View File

@@ -9,12 +9,10 @@ import {
import { loadModelCatalog } from "../agents/model-catalog.js";
import { runWithModelFallback } from "../agents/model-fallback.js";
import {
buildAllowedModelSet,
buildModelAliasIndex,
modelKey,
getModelRefStatus,
resolveAllowedModelRef,
resolveConfiguredModelRef,
resolveHooksGmailModel,
resolveModelRefFromString,
resolveThinkingDefault,
} from "../agents/model-selection.js";
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
@@ -301,14 +299,14 @@ export async function runCronIsolatedAgentTurn(params: {
})
: null;
if (hooksGmailModelRef) {
const allowed = buildAllowedModelSet({
const status = getModelRefStatus({
cfg: params.cfg,
catalog: await loadCatalog(),
ref: hooksGmailModelRef,
defaultProvider: resolvedDefault.provider,
defaultModel: resolvedDefault.model,
});
const key = modelKey(hooksGmailModelRef.provider, hooksGmailModelRef.model);
if (allowed.allowAny || allowed.allowedKeys.has(key)) {
if (status.allowed) {
provider = hooksGmailModelRef.provider;
model = hooksGmailModelRef.model;
}
@@ -321,34 +319,15 @@ export async function runCronIsolatedAgentTurn(params: {
if (typeof modelOverrideRaw !== "string") {
return { status: "error", error: "invalid model: expected string" };
}
const trimmed = modelOverrideRaw.trim();
if (!trimmed) {
return { status: "error", error: "invalid model: empty" };
}
const aliasIndex = buildModelAliasIndex({
cfg: params.cfg,
defaultProvider: resolvedDefault.provider,
});
const resolvedOverride = resolveModelRefFromString({
raw: trimmed,
defaultProvider: resolvedDefault.provider,
aliasIndex,
});
if (!resolvedOverride) {
return { status: "error", error: `invalid model: ${trimmed}` };
}
const allowed = buildAllowedModelSet({
const resolvedOverride = resolveAllowedModelRef({
cfg: params.cfg,
catalog: await loadCatalog(),
raw: modelOverrideRaw,
defaultProvider: resolvedDefault.provider,
defaultModel: resolvedDefault.model,
});
const key = modelKey(
resolvedOverride.ref.provider,
resolvedOverride.ref.model,
);
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
return { status: "error", error: `model not allowed: ${key}` };
if ("error" in resolvedOverride) {
return { status: "error", error: resolvedOverride.error };
}
provider = resolvedOverride.ref.provider;
model = resolvedOverride.ref.model;

View File

@@ -10,8 +10,7 @@ import {
resetModelCatalogCacheForTest,
} from "../agents/model-catalog.js";
import {
buildAllowedModelSet,
modelKey,
getModelRefStatus,
resolveConfiguredModelRef,
resolveHooksGmailModel,
} from "../agents/model-selection.js";
@@ -1782,22 +1781,21 @@ export async function startGatewayServer(
defaultModel: DEFAULT_MODEL,
});
const catalog = await loadModelCatalog({ config: cfgAtStart });
const key = modelKey(hooksModelRef.provider, hooksModelRef.model);
const allowed = buildAllowedModelSet({
const status = getModelRefStatus({
cfg: cfgAtStart,
catalog,
ref: hooksModelRef,
defaultProvider,
defaultModel,
});
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
if (!status.allowed) {
logHooks.warn(
`hooks.gmail.model "${key}" not in agents.defaults.models allowlist (will use primary instead)`,
`hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
);
}
const inCatalog = catalog.some((e) => modelKey(e.provider, e.id) === key);
if (!inCatalog) {
if (!status.inCatalog) {
logHooks.warn(
`hooks.gmail.model "${key}" not in the model catalog (may fail at runtime)`,
`hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
);
}
}

View File

@@ -3,11 +3,8 @@ import { randomUUID } from "node:crypto";
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
import {
buildAllowedModelSet,
buildModelAliasIndex,
modelKey,
resolveAllowedModelRef,
resolveConfiguredModelRef,
resolveModelRefFromString,
} from "../agents/model-selection.js";
import { normalizeGroupActivation } from "../auto-reply/group-activation.js";
import {
@@ -168,17 +165,6 @@ export async function applySessionsPatchToStore(params: {
defaultProvider: DEFAULT_PROVIDER,
defaultModel: DEFAULT_MODEL,
});
const aliasIndex = buildModelAliasIndex({
cfg,
defaultProvider: resolvedDefault.provider,
});
const resolved = resolveModelRefFromString({
raw: trimmed,
defaultProvider: resolvedDefault.provider,
aliasIndex,
});
if (!resolved) return invalid(`invalid model: ${trimmed}`);
if (!params.loadGatewayModelCatalog) {
return {
ok: false,
@@ -189,15 +175,15 @@ export async function applySessionsPatchToStore(params: {
};
}
const catalog = await params.loadGatewayModelCatalog();
const allowed = buildAllowedModelSet({
const resolved = resolveAllowedModelRef({
cfg,
catalog,
raw: trimmed,
defaultProvider: resolvedDefault.provider,
defaultModel: resolvedDefault.model,
});
const key = modelKey(resolved.ref.provider, resolved.ref.model);
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
return invalid(`model not allowed: ${key}`);
if ("error" in resolved) {
return invalid(resolved.error);
}
if (
resolved.ref.provider === resolvedDefault.provider &&