fix(model): retry with supported thinking level

This commit is contained in:
CI
2026-01-05 18:54:23 +01:00
committed by Peter Steinberger
parent 5622dfe86b
commit c627efce3e
3 changed files with 122 additions and 18 deletions

View File

@@ -1,7 +1,11 @@
import type { AssistantMessage } from "@mariozechner/pi-ai"; import type { AssistantMessage } from "@mariozechner/pi-ai";
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { isRateLimitAssistantError } from "./pi-embedded-helpers.js"; import {
isRateLimitAssistantError,
pickFallbackThinkingLevel,
} from "./pi-embedded-helpers.js";
import type { ThinkLevel } from "../auto-reply/thinking.js";
const asAssistant = (overrides: Partial<AssistantMessage>) => const asAssistant = (overrides: Partial<AssistantMessage>) =>
({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage; ({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage;
@@ -30,3 +34,34 @@ describe("isRateLimitAssistantError", () => {
expect(isRateLimitAssistantError(msg)).toBe(false); expect(isRateLimitAssistantError(msg)).toBe(false);
}); });
}); });
describe("pickFallbackThinkingLevel", () => {
it("selects the first supported thinking level", () => {
const attempted = new Set<ThinkLevel>(["low"]);
const next = pickFallbackThinkingLevel({
message:
"Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'.",
attempted,
});
expect(next).toBe("medium");
});
it("skips already attempted levels", () => {
const attempted = new Set<ThinkLevel>(["low", "medium"]);
const next = pickFallbackThinkingLevel({
message:
"Supported values are: 'medium', 'high', and 'xhigh'.",
attempted,
});
expect(next).toBe("high");
});
it("returns undefined when no supported values are found", () => {
const attempted = new Set<ThinkLevel>(["low"]);
const next = pickFallbackThinkingLevel({
message: "Request failed.",
attempted,
});
expect(next).toBeUndefined();
});
});

View File

@@ -6,6 +6,7 @@ import type {
AgentToolResult, AgentToolResult,
} from "@mariozechner/pi-agent-core"; } from "@mariozechner/pi-agent-core";
import type { AssistantMessage } from "@mariozechner/pi-ai"; import type { AssistantMessage } from "@mariozechner/pi-ai";
import { normalizeThinkLevel, type ThinkLevel } from "../auto-reply/thinking.js";
import { sanitizeContentBlocksImages } from "./tool-images.js"; import { sanitizeContentBlocksImages } from "./tool-images.js";
import type { WorkspaceBootstrapFile } from "./workspace.js"; import type { WorkspaceBootstrapFile } from "./workspace.js";
@@ -118,3 +119,38 @@ export function isRateLimitAssistantError(
if (!raw) return false; if (!raw) return false;
return /rate[_ ]limit|too many requests|429/.test(raw); return /rate[_ ]limit|too many requests|429/.test(raw);
} }
function extractSupportedValues(raw: string): string[] {
const match =
raw.match(/supported values are:\s*([^\n.]+)/i) ??
raw.match(/supported values:\s*([^\n.]+)/i);
if (!match?.[1]) return [];
const fragment = match[1];
const quoted = Array.from(fragment.matchAll(/['"]([^'"]+)['"]/g)).map(
(entry) => entry[1]?.trim(),
);
if (quoted.length > 0) {
return quoted.filter((entry): entry is string => Boolean(entry));
}
return fragment
.split(/,|\band\b/gi)
.map((entry) => entry.replace(/^[^a-zA-Z]+|[^a-zA-Z]+$/g, "").trim())
.filter(Boolean);
}
export function pickFallbackThinkingLevel(params: {
message?: string;
attempted: Set<ThinkLevel>;
}): ThinkLevel | undefined {
const raw = params.message?.trim();
if (!raw) return undefined;
const supported = extractSupportedValues(raw);
if (supported.length === 0) return undefined;
for (const entry of supported) {
const normalized = normalizeThinkLevel(entry);
if (!normalized) continue;
if (params.attempted.has(normalized)) continue;
return normalized;
}
return undefined;
}

View File

@@ -33,6 +33,7 @@ import {
ensureSessionHeader, ensureSessionHeader,
formatAssistantErrorText, formatAssistantErrorText,
isRateLimitAssistantError, isRateLimitAssistantError,
pickFallbackThinkingLevel,
sanitizeSessionMessagesImages, sanitizeSessionMessagesImages,
} from "./pi-embedded-helpers.js"; } from "./pi-embedded-helpers.js";
import { import {
@@ -319,22 +320,27 @@ export async function runEmbeddedPiAgent(params: {
const apiKey = await getApiKeyForModel(model, authStorage); const apiKey = await getApiKeyForModel(model, authStorage);
authStorage.setRuntimeApiKey(model.provider, apiKey); authStorage.setRuntimeApiKey(model.provider, apiKey);
const thinkingLevel = mapThinkingLevel(params.thinkLevel); let thinkLevel = params.thinkLevel ?? "off";
const attemptedThinking = new Set<ThinkLevel>();
log.debug( while (true) {
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} surface=${params.surface ?? "unknown"}`, const thinkingLevel = mapThinkingLevel(thinkLevel);
); attemptedThinking.add(thinkLevel);
await fs.mkdir(resolvedWorkspace, { recursive: true }); log.debug(
await ensureSessionHeader({ `embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} thinking=${thinkLevel} surface=${params.surface ?? "unknown"}`,
sessionFile: params.sessionFile, );
sessionId: params.sessionId,
cwd: resolvedWorkspace,
});
let restoreSkillEnv: (() => void) | undefined; await fs.mkdir(resolvedWorkspace, { recursive: true });
process.chdir(resolvedWorkspace); await ensureSessionHeader({
try { sessionFile: params.sessionFile,
sessionId: params.sessionId,
cwd: resolvedWorkspace,
});
let restoreSkillEnv: (() => void) | undefined;
process.chdir(resolvedWorkspace);
try {
const shouldLoadSkillEntries = const shouldLoadSkillEntries =
!params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills; !params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills;
const skillEntries = shouldLoadSkillEntries const skillEntries = shouldLoadSkillEntries
@@ -391,7 +397,7 @@ export async function runEmbeddedPiAgent(params: {
const systemPrompt = buildSystemPrompt({ const systemPrompt = buildSystemPrompt({
appendPrompt: buildAgentSystemPromptAppend({ appendPrompt: buildAgentSystemPromptAppend({
workspaceDir: resolvedWorkspace, workspaceDir: resolvedWorkspace,
defaultThinkLevel: params.thinkLevel, defaultThinkLevel: thinkLevel,
extraSystemPrompt: params.extraSystemPrompt, extraSystemPrompt: params.extraSystemPrompt,
ownerNumbers: params.ownerNumbers, ownerNumbers: params.ownerNumbers,
reasoningTagHint, reasoningTagHint,
@@ -542,6 +548,20 @@ export async function runEmbeddedPiAgent(params: {
params.abortSignal?.removeEventListener?.("abort", onAbort); params.abortSignal?.removeEventListener?.("abort", onAbort);
} }
if (promptError && !aborted) { if (promptError && !aborted) {
const fallbackThinking = pickFallbackThinkingLevel({
message:
promptError instanceof Error
? promptError.message
: String(promptError),
attempted: attemptedThinking,
});
if (fallbackThinking) {
log.warn(
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
);
thinkLevel = fallbackThinking;
continue;
}
throw promptError; throw promptError;
} }
@@ -552,6 +572,18 @@ export async function runEmbeddedPiAgent(params: {
| AssistantMessage | AssistantMessage
| undefined; | undefined;
const fallbackThinking = pickFallbackThinkingLevel({
message: lastAssistant?.errorMessage,
attempted: attemptedThinking,
});
if (fallbackThinking && !aborted) {
log.warn(
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
);
thinkLevel = fallbackThinking;
continue;
}
const fallbackConfigured = const fallbackConfigured =
(params.config?.agent?.modelFallbacks?.length ?? 0) > 0; (params.config?.agent?.modelFallbacks?.length ?? 0) > 0;
if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) { if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) {
@@ -631,9 +663,10 @@ export async function runEmbeddedPiAgent(params: {
aborted, aborted,
}, },
}; };
} finally { } finally {
restoreSkillEnv?.(); restoreSkillEnv?.();
process.chdir(prevCwd); process.chdir(prevCwd);
}
} }
}), }),
); );