fix(model): retry with supported thinking level

This commit is contained in:
CI
2026-01-05 18:54:23 +01:00
committed by Peter Steinberger
parent 5622dfe86b
commit c627efce3e
3 changed files with 122 additions and 18 deletions

View File

@@ -1,7 +1,11 @@
import type { AssistantMessage } from "@mariozechner/pi-ai";
import { describe, expect, it } from "vitest";
import { isRateLimitAssistantError } from "./pi-embedded-helpers.js";
import {
isRateLimitAssistantError,
pickFallbackThinkingLevel,
} from "./pi-embedded-helpers.js";
import type { ThinkLevel } from "../auto-reply/thinking.js";
const asAssistant = (overrides: Partial<AssistantMessage>) =>
({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage;
@@ -30,3 +34,34 @@ describe("isRateLimitAssistantError", () => {
expect(isRateLimitAssistantError(msg)).toBe(false);
});
});
describe("pickFallbackThinkingLevel", () => {
it("selects the first supported thinking level", () => {
const attempted = new Set<ThinkLevel>(["low"]);
const next = pickFallbackThinkingLevel({
message:
"Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'.",
attempted,
});
expect(next).toBe("medium");
});
it("skips already attempted levels", () => {
const attempted = new Set<ThinkLevel>(["low", "medium"]);
const next = pickFallbackThinkingLevel({
message:
"Supported values are: 'medium', 'high', and 'xhigh'.",
attempted,
});
expect(next).toBe("high");
});
it("returns undefined when no supported values are found", () => {
const attempted = new Set<ThinkLevel>(["low"]);
const next = pickFallbackThinkingLevel({
message: "Request failed.",
attempted,
});
expect(next).toBeUndefined();
});
});

View File

@@ -6,6 +6,7 @@ import type {
AgentToolResult,
} from "@mariozechner/pi-agent-core";
import type { AssistantMessage } from "@mariozechner/pi-ai";
import { normalizeThinkLevel, type ThinkLevel } from "../auto-reply/thinking.js";
import { sanitizeContentBlocksImages } from "./tool-images.js";
import type { WorkspaceBootstrapFile } from "./workspace.js";
@@ -118,3 +119,38 @@ export function isRateLimitAssistantError(
if (!raw) return false;
return /rate[_ ]limit|too many requests|429/.test(raw);
}
function extractSupportedValues(raw: string): string[] {
const match =
raw.match(/supported values are:\s*([^\n.]+)/i) ??
raw.match(/supported values:\s*([^\n.]+)/i);
if (!match?.[1]) return [];
const fragment = match[1];
const quoted = Array.from(fragment.matchAll(/['"]([^'"]+)['"]/g)).map(
(entry) => entry[1]?.trim(),
);
if (quoted.length > 0) {
return quoted.filter((entry): entry is string => Boolean(entry));
}
return fragment
.split(/,|\band\b/gi)
.map((entry) => entry.replace(/^[^a-zA-Z]+|[^a-zA-Z]+$/g, "").trim())
.filter(Boolean);
}
export function pickFallbackThinkingLevel(params: {
message?: string;
attempted: Set<ThinkLevel>;
}): ThinkLevel | undefined {
const raw = params.message?.trim();
if (!raw) return undefined;
const supported = extractSupportedValues(raw);
if (supported.length === 0) return undefined;
for (const entry of supported) {
const normalized = normalizeThinkLevel(entry);
if (!normalized) continue;
if (params.attempted.has(normalized)) continue;
return normalized;
}
return undefined;
}

View File

@@ -33,6 +33,7 @@ import {
ensureSessionHeader,
formatAssistantErrorText,
isRateLimitAssistantError,
pickFallbackThinkingLevel,
sanitizeSessionMessagesImages,
} from "./pi-embedded-helpers.js";
import {
@@ -319,22 +320,27 @@ export async function runEmbeddedPiAgent(params: {
const apiKey = await getApiKeyForModel(model, authStorage);
authStorage.setRuntimeApiKey(model.provider, apiKey);
const thinkingLevel = mapThinkingLevel(params.thinkLevel);
let thinkLevel = params.thinkLevel ?? "off";
const attemptedThinking = new Set<ThinkLevel>();
log.debug(
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} surface=${params.surface ?? "unknown"}`,
);
while (true) {
const thinkingLevel = mapThinkingLevel(thinkLevel);
attemptedThinking.add(thinkLevel);
await fs.mkdir(resolvedWorkspace, { recursive: true });
await ensureSessionHeader({
sessionFile: params.sessionFile,
sessionId: params.sessionId,
cwd: resolvedWorkspace,
});
log.debug(
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} thinking=${thinkLevel} surface=${params.surface ?? "unknown"}`,
);
let restoreSkillEnv: (() => void) | undefined;
process.chdir(resolvedWorkspace);
try {
await fs.mkdir(resolvedWorkspace, { recursive: true });
await ensureSessionHeader({
sessionFile: params.sessionFile,
sessionId: params.sessionId,
cwd: resolvedWorkspace,
});
let restoreSkillEnv: (() => void) | undefined;
process.chdir(resolvedWorkspace);
try {
const shouldLoadSkillEntries =
!params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills;
const skillEntries = shouldLoadSkillEntries
@@ -391,7 +397,7 @@ export async function runEmbeddedPiAgent(params: {
const systemPrompt = buildSystemPrompt({
appendPrompt: buildAgentSystemPromptAppend({
workspaceDir: resolvedWorkspace,
defaultThinkLevel: params.thinkLevel,
defaultThinkLevel: thinkLevel,
extraSystemPrompt: params.extraSystemPrompt,
ownerNumbers: params.ownerNumbers,
reasoningTagHint,
@@ -542,6 +548,20 @@ export async function runEmbeddedPiAgent(params: {
params.abortSignal?.removeEventListener?.("abort", onAbort);
}
if (promptError && !aborted) {
const fallbackThinking = pickFallbackThinkingLevel({
message:
promptError instanceof Error
? promptError.message
: String(promptError),
attempted: attemptedThinking,
});
if (fallbackThinking) {
log.warn(
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
);
thinkLevel = fallbackThinking;
continue;
}
throw promptError;
}
@@ -552,6 +572,18 @@ export async function runEmbeddedPiAgent(params: {
| AssistantMessage
| undefined;
const fallbackThinking = pickFallbackThinkingLevel({
message: lastAssistant?.errorMessage,
attempted: attemptedThinking,
});
if (fallbackThinking && !aborted) {
log.warn(
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
);
thinkLevel = fallbackThinking;
continue;
}
const fallbackConfigured =
(params.config?.agent?.modelFallbacks?.length ?? 0) > 0;
if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) {
@@ -631,9 +663,10 @@ export async function runEmbeddedPiAgent(params: {
aborted,
},
};
} finally {
restoreSkillEnv?.();
process.chdir(prevCwd);
} finally {
restoreSkillEnv?.();
process.chdir(prevCwd);
}
}
}),
);