diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f6630ef3..787d6fa62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - WhatsApp: refactor vCard parsing helper and improve empty contact card summaries. (#624) — thanks @steipete - WhatsApp: include phone numbers when multiple contacts are shared. (#625) — thanks @mahmoudashraf93 +- Agents: warn on small context windows (<32k) and block unusable ones (<16k). — thanks @steipete - Pairing: cap pending DM pairing requests at 3 per provider and avoid pairing replies for outbound DMs. — thanks @steipete - macOS: replace relay smoke test with version check in packaging script. (#615) — thanks @YuriNachos - macOS: avoid clearing Launch at Login during app initialization. (#607) — thanks @wes-davis diff --git a/src/agents/context-window-guard.test.ts b/src/agents/context-window-guard.test.ts new file mode 100644 index 000000000..83315a660 --- /dev/null +++ b/src/agents/context-window-guard.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import type { ClawdbotConfig } from "../config/config.js"; +import { + CONTEXT_WINDOW_HARD_MIN_TOKENS, + CONTEXT_WINDOW_WARN_BELOW_TOKENS, + evaluateContextWindowGuard, + resolveContextWindowInfo, +} from "./context-window-guard.js"; + +describe("context-window-guard", () => { + it("blocks below 16k (model metadata)", () => { + const info = resolveContextWindowInfo({ + cfg: undefined, + provider: "openrouter", + modelId: "tiny", + modelContextWindow: 8000, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(guard.source).toBe("model"); + expect(guard.tokens).toBe(8000); + expect(guard.shouldWarn).toBe(true); + expect(guard.shouldBlock).toBe(true); + }); + + it("warns below 32k but does not block at 16k+", () => { + const info = resolveContextWindowInfo({ + cfg: undefined, + provider: "openai", + modelId: "small", + modelContextWindow: 24_000, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(guard.tokens).toBe(24_000); + expect(guard.shouldWarn).toBe(true); + expect(guard.shouldBlock).toBe(false); + }); + + it("does not warn at 32k+ (model metadata)", () => { + const info = resolveContextWindowInfo({ + cfg: undefined, + provider: "openai", + modelId: "ok", + modelContextWindow: 64_000, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(guard.shouldWarn).toBe(false); + expect(guard.shouldBlock).toBe(false); + }); + + it("uses models.providers.*.models[].contextWindow when present", () => { + const cfg = { + models: { + providers: { + openrouter: { + baseUrl: "http://localhost", + apiKey: "x", + models: [ + { + id: "tiny", + name: "tiny", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 12_000, + maxTokens: 256, + }, + ], + }, + }, + }, + } satisfies ClawdbotConfig; + + const info = resolveContextWindowInfo({ + cfg, + provider: "openrouter", + modelId: "tiny", + modelContextWindow: undefined, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(info.source).toBe("modelsConfig"); + expect(guard.shouldBlock).toBe(true); + }); + + it("falls back to agents.defaults.contextTokens", () => { + const cfg = { + agents: { defaults: { contextTokens: 20_000 } }, + } satisfies ClawdbotConfig; + const info = resolveContextWindowInfo({ + cfg, + provider: "anthropic", + modelId: "whatever", + modelContextWindow: undefined, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(info.source).toBe("agentContextTokens"); + expect(guard.shouldWarn).toBe(true); + expect(guard.shouldBlock).toBe(false); + }); + + it("uses default when nothing else is available", () => { + const info = resolveContextWindowInfo({ + cfg: undefined, + provider: "anthropic", + modelId: "unknown", + modelContextWindow: undefined, + defaultTokens: 200_000, + }); + const guard = evaluateContextWindowGuard({ info }); + expect(info.source).toBe("default"); + expect(guard.shouldWarn).toBe(false); + expect(guard.shouldBlock).toBe(false); + }); + + it("allows overriding thresholds", () => { + const info = { tokens: 10_000, source: "model" as const }; + const guard = evaluateContextWindowGuard({ + info, + warnBelowTokens: 12_000, + hardMinTokens: 9_000, + }); + expect(guard.shouldWarn).toBe(true); + expect(guard.shouldBlock).toBe(false); + }); + + it("exports thresholds as expected", () => { + expect(CONTEXT_WINDOW_HARD_MIN_TOKENS).toBe(16_000); + expect(CONTEXT_WINDOW_WARN_BELOW_TOKENS).toBe(32_000); + }); +}); diff --git a/src/agents/context-window-guard.ts b/src/agents/context-window-guard.ts new file mode 100644 index 000000000..952d6a4d5 --- /dev/null +++ b/src/agents/context-window-guard.ts @@ -0,0 +1,84 @@ +import type { ClawdbotConfig } from "../config/config.js"; + +export const CONTEXT_WINDOW_HARD_MIN_TOKENS = 16_000; +export const CONTEXT_WINDOW_WARN_BELOW_TOKENS = 32_000; + +export type ContextWindowSource = + | "model" + | "modelsConfig" + | "agentContextTokens" + | "default"; + +export type ContextWindowInfo = { + tokens: number; + source: ContextWindowSource; +}; + +function normalizePositiveInt(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value)) return null; + const int = Math.floor(value); + return int > 0 ? int : null; +} + +export function resolveContextWindowInfo(params: { + cfg: ClawdbotConfig | undefined; + provider: string; + modelId: string; + modelContextWindow?: number; + defaultTokens: number; +}): ContextWindowInfo { + const fromModel = normalizePositiveInt(params.modelContextWindow); + if (fromModel) return { tokens: fromModel, source: "model" }; + + const fromModelsConfig = (() => { + const providers = params.cfg?.models?.providers as + | Record< + string, + { models?: Array<{ id?: string; contextWindow?: number }> } + > + | undefined; + const providerEntry = providers?.[params.provider]; + const models = Array.isArray(providerEntry?.models) + ? providerEntry.models + : []; + const match = models.find((m) => m?.id === params.modelId); + return normalizePositiveInt(match?.contextWindow); + })(); + if (fromModelsConfig) + return { tokens: fromModelsConfig, source: "modelsConfig" }; + + const fromAgentConfig = normalizePositiveInt( + params.cfg?.agents?.defaults?.contextTokens, + ); + if (fromAgentConfig) + return { tokens: fromAgentConfig, source: "agentContextTokens" }; + + return { tokens: Math.floor(params.defaultTokens), source: "default" }; +} + +export type ContextWindowGuardResult = ContextWindowInfo & { + shouldWarn: boolean; + shouldBlock: boolean; +}; + +export function evaluateContextWindowGuard(params: { + info: ContextWindowInfo; + warnBelowTokens?: number; + hardMinTokens?: number; +}): ContextWindowGuardResult { + const warnBelow = Math.max( + 1, + Math.floor(params.warnBelowTokens ?? CONTEXT_WINDOW_WARN_BELOW_TOKENS), + ); + const hardMin = Math.max( + 1, + Math.floor(params.hardMinTokens ?? CONTEXT_WINDOW_HARD_MIN_TOKENS), + ); + const tokens = Math.max(0, Math.floor(params.info.tokens)); + return { + ...params.info, + tokens, + shouldWarn: tokens > 0 && tokens < warnBelow, + shouldBlock: tokens > 0 && tokens < hardMin, + }; +} diff --git a/src/agents/pi-embedded-runner.ts b/src/agents/pi-embedded-runner.ts index 230c75b25..a95d5f38d 100644 --- a/src/agents/pi-embedded-runner.ts +++ b/src/agents/pi-embedded-runner.ts @@ -42,6 +42,12 @@ import { markAuthProfileUsed, } from "./auth-profiles.js"; import type { BashElevatedDefaults } from "./bash-tools.js"; +import { + CONTEXT_WINDOW_HARD_MIN_TOKENS, + CONTEXT_WINDOW_WARN_BELOW_TOKENS, + evaluateContextWindowGuard, + resolveContextWindowInfo, +} from "./context-window-guard.js"; import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, @@ -183,41 +189,13 @@ function resolveContextWindowTokens(params: { modelId: string; model: Model | undefined; }): number { - const fromModel = - typeof params.model?.contextWindow === "number" && - Number.isFinite(params.model.contextWindow) && - params.model.contextWindow > 0 - ? params.model.contextWindow - : undefined; - if (fromModel) return fromModel; - - const fromModelsConfig = (() => { - const providers = params.cfg?.models?.providers as - | Record< - string, - { models?: Array<{ id?: string; contextWindow?: number }> } - > - | undefined; - const providerEntry = providers?.[params.provider]; - const models = Array.isArray(providerEntry?.models) - ? providerEntry.models - : []; - const match = models.find((m) => m?.id === params.modelId); - return typeof match?.contextWindow === "number" && match.contextWindow > 0 - ? match.contextWindow - : undefined; - })(); - if (fromModelsConfig) return fromModelsConfig; - - const fromAgentConfig = - typeof params.cfg?.agents?.defaults?.contextTokens === "number" && - Number.isFinite(params.cfg.agents.defaults.contextTokens) && - params.cfg.agents.defaults.contextTokens > 0 - ? Math.floor(params.cfg.agents.defaults.contextTokens) - : undefined; - if (fromAgentConfig) return fromAgentConfig; - - return DEFAULT_CONTEXT_TOKENS; + return resolveContextWindowInfo({ + cfg: params.cfg, + provider: params.provider, + modelId: params.modelId, + modelContextWindow: params.model?.contextWindow, + defaultTokens: DEFAULT_CONTEXT_TOKENS, + }).tokens; } function buildContextPruningExtension(params: { @@ -1086,6 +1064,33 @@ export async function runEmbeddedPiAgent(params: { if (!model) { throw new Error(error ?? `Unknown model: ${provider}/${modelId}`); } + + const ctxInfo = resolveContextWindowInfo({ + cfg: params.config, + provider, + modelId, + modelContextWindow: model.contextWindow, + defaultTokens: DEFAULT_CONTEXT_TOKENS, + }); + const ctxGuard = evaluateContextWindowGuard({ + info: ctxInfo, + warnBelowTokens: CONTEXT_WINDOW_WARN_BELOW_TOKENS, + hardMinTokens: CONTEXT_WINDOW_HARD_MIN_TOKENS, + }); + if (ctxGuard.shouldWarn) { + log.warn( + `low context window: ${provider}/${modelId} ctx=${ctxGuard.tokens} (warn<${CONTEXT_WINDOW_WARN_BELOW_TOKENS}) source=${ctxGuard.source}`, + ); + } + if (ctxGuard.shouldBlock) { + log.error( + `blocked model (context window too small): ${provider}/${modelId} ctx=${ctxGuard.tokens} (min=${CONTEXT_WINDOW_HARD_MIN_TOKENS}) source=${ctxGuard.source}`, + ); + throw new FailoverError( + `Model context window too small (${ctxGuard.tokens} tokens). Minimum is ${CONTEXT_WINDOW_HARD_MIN_TOKENS}.`, + { reason: "unknown", provider, model: modelId }, + ); + } const authStore = ensureAuthProfileStore(agentDir); const explicitProfileId = params.authProfileId?.trim(); const profileOrder = resolveAuthProfileOrder({