refactor: centralize model override validation
This commit is contained in:
@@ -83,7 +83,10 @@ State is stored in `auth-profiles.json` under `usageStats`:
|
|||||||
|
|
||||||
If all profiles for a provider fail, Clawdbot moves to the next model in
|
If all profiles for a provider fail, Clawdbot moves to the next model in
|
||||||
`agents.defaults.model.fallbacks`. This applies to auth failures, rate limits, and
|
`agents.defaults.model.fallbacks`. This applies to auth failures, rate limits, and
|
||||||
timeouts that exhausted profile rotation.
|
timeouts that exhausted profile rotation (other errors do not advance fallback).
|
||||||
|
|
||||||
|
When a run starts with a model override (hooks or CLI), fallbacks still end at
|
||||||
|
`agents.defaults.model.primary` after trying any configured fallbacks.
|
||||||
|
|
||||||
## Related config
|
## Related config
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
buildAllowedModelSet,
|
buildAllowedModelSet,
|
||||||
modelKey,
|
modelKey,
|
||||||
parseModelRef,
|
parseModelRef,
|
||||||
|
resolveAllowedModelRef,
|
||||||
resolveHooksGmailModel,
|
resolveHooksGmailModel,
|
||||||
} from "./model-selection.js";
|
} from "./model-selection.js";
|
||||||
|
|
||||||
@@ -143,3 +144,62 @@ describe("resolveHooksGmailModel", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("resolveAllowedModelRef", () => {
|
||||||
|
it("resolves aliases when allowed", () => {
|
||||||
|
const cfg = {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
models: {
|
||||||
|
"anthropic/claude-sonnet-4-1": { alias: "Sonnet" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies ClawdbotConfig;
|
||||||
|
const resolved = resolveAllowedModelRef({
|
||||||
|
cfg,
|
||||||
|
catalog: [
|
||||||
|
{
|
||||||
|
provider: "anthropic",
|
||||||
|
id: "claude-sonnet-4-1",
|
||||||
|
name: "Sonnet",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
raw: "Sonnet",
|
||||||
|
defaultProvider: "anthropic",
|
||||||
|
defaultModel: "claude-opus-4-5",
|
||||||
|
});
|
||||||
|
expect("error" in resolved).toBe(false);
|
||||||
|
if ("ref" in resolved) {
|
||||||
|
expect(resolved.ref).toEqual({
|
||||||
|
provider: "anthropic",
|
||||||
|
model: "claude-sonnet-4-1",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects disallowed models", () => {
|
||||||
|
const cfg = {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
models: {
|
||||||
|
"openai/gpt-4": { alias: "GPT4" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} satisfies ClawdbotConfig;
|
||||||
|
const resolved = resolveAllowedModelRef({
|
||||||
|
cfg,
|
||||||
|
catalog: [
|
||||||
|
{ provider: "openai", id: "gpt-4", name: "GPT-4" },
|
||||||
|
{ provider: "anthropic", id: "claude-sonnet-4-1", name: "Sonnet" },
|
||||||
|
],
|
||||||
|
raw: "anthropic/claude-sonnet-4-1",
|
||||||
|
defaultProvider: "openai",
|
||||||
|
defaultModel: "gpt-4",
|
||||||
|
});
|
||||||
|
expect(resolved).toEqual({
|
||||||
|
error: "model not allowed: anthropic/claude-sonnet-4-1",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -197,6 +197,76 @@ export function buildAllowedModelSet(params: {
|
|||||||
return { allowAny: false, allowedCatalog, allowedKeys };
|
return { allowAny: false, allowedCatalog, allowedKeys };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type ModelRefStatus = {
|
||||||
|
key: string;
|
||||||
|
inCatalog: boolean;
|
||||||
|
allowAny: boolean;
|
||||||
|
allowed: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function getModelRefStatus(params: {
|
||||||
|
cfg: ClawdbotConfig;
|
||||||
|
catalog: ModelCatalogEntry[];
|
||||||
|
ref: ModelRef;
|
||||||
|
defaultProvider: string;
|
||||||
|
defaultModel?: string;
|
||||||
|
}): ModelRefStatus {
|
||||||
|
const allowed = buildAllowedModelSet({
|
||||||
|
cfg: params.cfg,
|
||||||
|
catalog: params.catalog,
|
||||||
|
defaultProvider: params.defaultProvider,
|
||||||
|
defaultModel: params.defaultModel,
|
||||||
|
});
|
||||||
|
const key = modelKey(params.ref.provider, params.ref.model);
|
||||||
|
return {
|
||||||
|
key,
|
||||||
|
inCatalog: params.catalog.some(
|
||||||
|
(entry) => modelKey(entry.provider, entry.id) === key,
|
||||||
|
),
|
||||||
|
allowAny: allowed.allowAny,
|
||||||
|
allowed: allowed.allowAny || allowed.allowedKeys.has(key),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveAllowedModelRef(params: {
|
||||||
|
cfg: ClawdbotConfig;
|
||||||
|
catalog: ModelCatalogEntry[];
|
||||||
|
raw: string;
|
||||||
|
defaultProvider: string;
|
||||||
|
defaultModel?: string;
|
||||||
|
}):
|
||||||
|
| { ref: ModelRef; key: string }
|
||||||
|
| {
|
||||||
|
error: string;
|
||||||
|
} {
|
||||||
|
const trimmed = params.raw.trim();
|
||||||
|
if (!trimmed) return { error: "invalid model: empty" };
|
||||||
|
|
||||||
|
const aliasIndex = buildModelAliasIndex({
|
||||||
|
cfg: params.cfg,
|
||||||
|
defaultProvider: params.defaultProvider,
|
||||||
|
});
|
||||||
|
const resolved = resolveModelRefFromString({
|
||||||
|
raw: trimmed,
|
||||||
|
defaultProvider: params.defaultProvider,
|
||||||
|
aliasIndex,
|
||||||
|
});
|
||||||
|
if (!resolved) return { error: `invalid model: ${trimmed}` };
|
||||||
|
|
||||||
|
const status = getModelRefStatus({
|
||||||
|
cfg: params.cfg,
|
||||||
|
catalog: params.catalog,
|
||||||
|
ref: resolved.ref,
|
||||||
|
defaultProvider: params.defaultProvider,
|
||||||
|
defaultModel: params.defaultModel,
|
||||||
|
});
|
||||||
|
if (!status.allowed) {
|
||||||
|
return { error: `model not allowed: ${status.key}` };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { ref: resolved.ref, key: status.key };
|
||||||
|
}
|
||||||
|
|
||||||
export function resolveThinkingDefault(params: {
|
export function resolveThinkingDefault(params: {
|
||||||
cfg: ClawdbotConfig;
|
cfg: ClawdbotConfig;
|
||||||
provider: string;
|
provider: string;
|
||||||
|
|||||||
@@ -8,6 +8,13 @@ import {
|
|||||||
resolveAgentWorkspaceDir,
|
resolveAgentWorkspaceDir,
|
||||||
resolveDefaultAgentId,
|
resolveDefaultAgentId,
|
||||||
} from "../agents/agent-scope.js";
|
} from "../agents/agent-scope.js";
|
||||||
|
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
|
||||||
|
import { loadModelCatalog } from "../agents/model-catalog.js";
|
||||||
|
import {
|
||||||
|
getModelRefStatus,
|
||||||
|
resolveConfiguredModelRef,
|
||||||
|
resolveHooksGmailModel,
|
||||||
|
} from "../agents/model-selection.js";
|
||||||
import { buildWorkspaceSkillStatus } from "../agents/skills-status.js";
|
import { buildWorkspaceSkillStatus } from "../agents/skills-status.js";
|
||||||
import type { ClawdbotConfig } from "../config/config.js";
|
import type { ClawdbotConfig } from "../config/config.js";
|
||||||
import {
|
import {
|
||||||
@@ -201,6 +208,48 @@ export async function doctorCommand(
|
|||||||
|
|
||||||
await noteSecurityWarnings(cfg);
|
await noteSecurityWarnings(cfg);
|
||||||
|
|
||||||
|
if (cfg.hooks?.gmail?.model?.trim()) {
|
||||||
|
const hooksModelRef = resolveHooksGmailModel({
|
||||||
|
cfg,
|
||||||
|
defaultProvider: DEFAULT_PROVIDER,
|
||||||
|
});
|
||||||
|
if (!hooksModelRef) {
|
||||||
|
note(
|
||||||
|
`- hooks.gmail.model "${cfg.hooks.gmail.model}" could not be resolved`,
|
||||||
|
"Hooks",
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
const { provider: defaultProvider, model: defaultModel } =
|
||||||
|
resolveConfiguredModelRef({
|
||||||
|
cfg,
|
||||||
|
defaultProvider: DEFAULT_PROVIDER,
|
||||||
|
defaultModel: DEFAULT_MODEL,
|
||||||
|
});
|
||||||
|
const catalog = await loadModelCatalog({ config: cfg });
|
||||||
|
const status = getModelRefStatus({
|
||||||
|
cfg,
|
||||||
|
catalog,
|
||||||
|
ref: hooksModelRef,
|
||||||
|
defaultProvider,
|
||||||
|
defaultModel,
|
||||||
|
});
|
||||||
|
const warnings: string[] = [];
|
||||||
|
if (!status.allowed) {
|
||||||
|
warnings.push(
|
||||||
|
`- hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!status.inCatalog) {
|
||||||
|
warnings.push(
|
||||||
|
`- hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (warnings.length > 0) {
|
||||||
|
note(warnings.join("\n"), "Hooks");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
options.nonInteractive !== true &&
|
options.nonInteractive !== true &&
|
||||||
process.platform === "linux" &&
|
process.platform === "linux" &&
|
||||||
|
|||||||
@@ -9,12 +9,10 @@ import {
|
|||||||
import { loadModelCatalog } from "../agents/model-catalog.js";
|
import { loadModelCatalog } from "../agents/model-catalog.js";
|
||||||
import { runWithModelFallback } from "../agents/model-fallback.js";
|
import { runWithModelFallback } from "../agents/model-fallback.js";
|
||||||
import {
|
import {
|
||||||
buildAllowedModelSet,
|
getModelRefStatus,
|
||||||
buildModelAliasIndex,
|
resolveAllowedModelRef,
|
||||||
modelKey,
|
|
||||||
resolveConfiguredModelRef,
|
resolveConfiguredModelRef,
|
||||||
resolveHooksGmailModel,
|
resolveHooksGmailModel,
|
||||||
resolveModelRefFromString,
|
|
||||||
resolveThinkingDefault,
|
resolveThinkingDefault,
|
||||||
} from "../agents/model-selection.js";
|
} from "../agents/model-selection.js";
|
||||||
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
|
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
|
||||||
@@ -301,14 +299,14 @@ export async function runCronIsolatedAgentTurn(params: {
|
|||||||
})
|
})
|
||||||
: null;
|
: null;
|
||||||
if (hooksGmailModelRef) {
|
if (hooksGmailModelRef) {
|
||||||
const allowed = buildAllowedModelSet({
|
const status = getModelRefStatus({
|
||||||
cfg: params.cfg,
|
cfg: params.cfg,
|
||||||
catalog: await loadCatalog(),
|
catalog: await loadCatalog(),
|
||||||
|
ref: hooksGmailModelRef,
|
||||||
defaultProvider: resolvedDefault.provider,
|
defaultProvider: resolvedDefault.provider,
|
||||||
defaultModel: resolvedDefault.model,
|
defaultModel: resolvedDefault.model,
|
||||||
});
|
});
|
||||||
const key = modelKey(hooksGmailModelRef.provider, hooksGmailModelRef.model);
|
if (status.allowed) {
|
||||||
if (allowed.allowAny || allowed.allowedKeys.has(key)) {
|
|
||||||
provider = hooksGmailModelRef.provider;
|
provider = hooksGmailModelRef.provider;
|
||||||
model = hooksGmailModelRef.model;
|
model = hooksGmailModelRef.model;
|
||||||
}
|
}
|
||||||
@@ -321,34 +319,15 @@ export async function runCronIsolatedAgentTurn(params: {
|
|||||||
if (typeof modelOverrideRaw !== "string") {
|
if (typeof modelOverrideRaw !== "string") {
|
||||||
return { status: "error", error: "invalid model: expected string" };
|
return { status: "error", error: "invalid model: expected string" };
|
||||||
}
|
}
|
||||||
const trimmed = modelOverrideRaw.trim();
|
const resolvedOverride = resolveAllowedModelRef({
|
||||||
if (!trimmed) {
|
|
||||||
return { status: "error", error: "invalid model: empty" };
|
|
||||||
}
|
|
||||||
const aliasIndex = buildModelAliasIndex({
|
|
||||||
cfg: params.cfg,
|
|
||||||
defaultProvider: resolvedDefault.provider,
|
|
||||||
});
|
|
||||||
const resolvedOverride = resolveModelRefFromString({
|
|
||||||
raw: trimmed,
|
|
||||||
defaultProvider: resolvedDefault.provider,
|
|
||||||
aliasIndex,
|
|
||||||
});
|
|
||||||
if (!resolvedOverride) {
|
|
||||||
return { status: "error", error: `invalid model: ${trimmed}` };
|
|
||||||
}
|
|
||||||
const allowed = buildAllowedModelSet({
|
|
||||||
cfg: params.cfg,
|
cfg: params.cfg,
|
||||||
catalog: await loadCatalog(),
|
catalog: await loadCatalog(),
|
||||||
|
raw: modelOverrideRaw,
|
||||||
defaultProvider: resolvedDefault.provider,
|
defaultProvider: resolvedDefault.provider,
|
||||||
defaultModel: resolvedDefault.model,
|
defaultModel: resolvedDefault.model,
|
||||||
});
|
});
|
||||||
const key = modelKey(
|
if ("error" in resolvedOverride) {
|
||||||
resolvedOverride.ref.provider,
|
return { status: "error", error: resolvedOverride.error };
|
||||||
resolvedOverride.ref.model,
|
|
||||||
);
|
|
||||||
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
|
|
||||||
return { status: "error", error: `model not allowed: ${key}` };
|
|
||||||
}
|
}
|
||||||
provider = resolvedOverride.ref.provider;
|
provider = resolvedOverride.ref.provider;
|
||||||
model = resolvedOverride.ref.model;
|
model = resolvedOverride.ref.model;
|
||||||
|
|||||||
@@ -10,8 +10,7 @@ import {
|
|||||||
resetModelCatalogCacheForTest,
|
resetModelCatalogCacheForTest,
|
||||||
} from "../agents/model-catalog.js";
|
} from "../agents/model-catalog.js";
|
||||||
import {
|
import {
|
||||||
buildAllowedModelSet,
|
getModelRefStatus,
|
||||||
modelKey,
|
|
||||||
resolveConfiguredModelRef,
|
resolveConfiguredModelRef,
|
||||||
resolveHooksGmailModel,
|
resolveHooksGmailModel,
|
||||||
} from "../agents/model-selection.js";
|
} from "../agents/model-selection.js";
|
||||||
@@ -1782,22 +1781,21 @@ export async function startGatewayServer(
|
|||||||
defaultModel: DEFAULT_MODEL,
|
defaultModel: DEFAULT_MODEL,
|
||||||
});
|
});
|
||||||
const catalog = await loadModelCatalog({ config: cfgAtStart });
|
const catalog = await loadModelCatalog({ config: cfgAtStart });
|
||||||
const key = modelKey(hooksModelRef.provider, hooksModelRef.model);
|
const status = getModelRefStatus({
|
||||||
const allowed = buildAllowedModelSet({
|
|
||||||
cfg: cfgAtStart,
|
cfg: cfgAtStart,
|
||||||
catalog,
|
catalog,
|
||||||
|
ref: hooksModelRef,
|
||||||
defaultProvider,
|
defaultProvider,
|
||||||
defaultModel,
|
defaultModel,
|
||||||
});
|
});
|
||||||
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
|
if (!status.allowed) {
|
||||||
logHooks.warn(
|
logHooks.warn(
|
||||||
`hooks.gmail.model "${key}" not in agents.defaults.models allowlist (will use primary instead)`,
|
`hooks.gmail.model "${status.key}" not in agents.defaults.models allowlist (will use primary instead)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const inCatalog = catalog.some((e) => modelKey(e.provider, e.id) === key);
|
if (!status.inCatalog) {
|
||||||
if (!inCatalog) {
|
|
||||||
logHooks.warn(
|
logHooks.warn(
|
||||||
`hooks.gmail.model "${key}" not in the model catalog (may fail at runtime)`,
|
`hooks.gmail.model "${status.key}" not in the model catalog (may fail at runtime)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,8 @@ import { randomUUID } from "node:crypto";
|
|||||||
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
|
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js";
|
||||||
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
|
import type { ModelCatalogEntry } from "../agents/model-catalog.js";
|
||||||
import {
|
import {
|
||||||
buildAllowedModelSet,
|
resolveAllowedModelRef,
|
||||||
buildModelAliasIndex,
|
|
||||||
modelKey,
|
|
||||||
resolveConfiguredModelRef,
|
resolveConfiguredModelRef,
|
||||||
resolveModelRefFromString,
|
|
||||||
} from "../agents/model-selection.js";
|
} from "../agents/model-selection.js";
|
||||||
import { normalizeGroupActivation } from "../auto-reply/group-activation.js";
|
import { normalizeGroupActivation } from "../auto-reply/group-activation.js";
|
||||||
import {
|
import {
|
||||||
@@ -168,17 +165,6 @@ export async function applySessionsPatchToStore(params: {
|
|||||||
defaultProvider: DEFAULT_PROVIDER,
|
defaultProvider: DEFAULT_PROVIDER,
|
||||||
defaultModel: DEFAULT_MODEL,
|
defaultModel: DEFAULT_MODEL,
|
||||||
});
|
});
|
||||||
const aliasIndex = buildModelAliasIndex({
|
|
||||||
cfg,
|
|
||||||
defaultProvider: resolvedDefault.provider,
|
|
||||||
});
|
|
||||||
const resolved = resolveModelRefFromString({
|
|
||||||
raw: trimmed,
|
|
||||||
defaultProvider: resolvedDefault.provider,
|
|
||||||
aliasIndex,
|
|
||||||
});
|
|
||||||
if (!resolved) return invalid(`invalid model: ${trimmed}`);
|
|
||||||
|
|
||||||
if (!params.loadGatewayModelCatalog) {
|
if (!params.loadGatewayModelCatalog) {
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
@@ -189,15 +175,15 @@ export async function applySessionsPatchToStore(params: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
const catalog = await params.loadGatewayModelCatalog();
|
const catalog = await params.loadGatewayModelCatalog();
|
||||||
const allowed = buildAllowedModelSet({
|
const resolved = resolveAllowedModelRef({
|
||||||
cfg,
|
cfg,
|
||||||
catalog,
|
catalog,
|
||||||
|
raw: trimmed,
|
||||||
defaultProvider: resolvedDefault.provider,
|
defaultProvider: resolvedDefault.provider,
|
||||||
defaultModel: resolvedDefault.model,
|
defaultModel: resolvedDefault.model,
|
||||||
});
|
});
|
||||||
const key = modelKey(resolved.ref.provider, resolved.ref.model);
|
if ("error" in resolved) {
|
||||||
if (!allowed.allowAny && !allowed.allowedKeys.has(key)) {
|
return invalid(resolved.error);
|
||||||
return invalid(`model not allowed: ${key}`);
|
|
||||||
}
|
}
|
||||||
if (
|
if (
|
||||||
resolved.ref.provider === resolvedDefault.provider &&
|
resolved.ref.provider === resolvedDefault.provider &&
|
||||||
|
|||||||
Reference in New Issue
Block a user