fix(model): retry with supported thinking level
This commit is contained in:
@@ -1,7 +1,11 @@
|
||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { isRateLimitAssistantError } from "./pi-embedded-helpers.js";
|
||||
import {
|
||||
isRateLimitAssistantError,
|
||||
pickFallbackThinkingLevel,
|
||||
} from "./pi-embedded-helpers.js";
|
||||
import type { ThinkLevel } from "../auto-reply/thinking.js";
|
||||
|
||||
const asAssistant = (overrides: Partial<AssistantMessage>) =>
|
||||
({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage;
|
||||
@@ -30,3 +34,34 @@ describe("isRateLimitAssistantError", () => {
|
||||
expect(isRateLimitAssistantError(msg)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickFallbackThinkingLevel", () => {
|
||||
it("selects the first supported thinking level", () => {
|
||||
const attempted = new Set<ThinkLevel>(["low"]);
|
||||
const next = pickFallbackThinkingLevel({
|
||||
message:
|
||||
"Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'.",
|
||||
attempted,
|
||||
});
|
||||
expect(next).toBe("medium");
|
||||
});
|
||||
|
||||
it("skips already attempted levels", () => {
|
||||
const attempted = new Set<ThinkLevel>(["low", "medium"]);
|
||||
const next = pickFallbackThinkingLevel({
|
||||
message:
|
||||
"Supported values are: 'medium', 'high', and 'xhigh'.",
|
||||
attempted,
|
||||
});
|
||||
expect(next).toBe("high");
|
||||
});
|
||||
|
||||
it("returns undefined when no supported values are found", () => {
|
||||
const attempted = new Set<ThinkLevel>(["low"]);
|
||||
const next = pickFallbackThinkingLevel({
|
||||
message: "Request failed.",
|
||||
attempted,
|
||||
});
|
||||
expect(next).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
AgentToolResult,
|
||||
} from "@mariozechner/pi-agent-core";
|
||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||
import { normalizeThinkLevel, type ThinkLevel } from "../auto-reply/thinking.js";
|
||||
|
||||
import { sanitizeContentBlocksImages } from "./tool-images.js";
|
||||
import type { WorkspaceBootstrapFile } from "./workspace.js";
|
||||
@@ -118,3 +119,38 @@ export function isRateLimitAssistantError(
|
||||
if (!raw) return false;
|
||||
return /rate[_ ]limit|too many requests|429/.test(raw);
|
||||
}
|
||||
|
||||
function extractSupportedValues(raw: string): string[] {
|
||||
const match =
|
||||
raw.match(/supported values are:\s*([^\n.]+)/i) ??
|
||||
raw.match(/supported values:\s*([^\n.]+)/i);
|
||||
if (!match?.[1]) return [];
|
||||
const fragment = match[1];
|
||||
const quoted = Array.from(fragment.matchAll(/['"]([^'"]+)['"]/g)).map(
|
||||
(entry) => entry[1]?.trim(),
|
||||
);
|
||||
if (quoted.length > 0) {
|
||||
return quoted.filter((entry): entry is string => Boolean(entry));
|
||||
}
|
||||
return fragment
|
||||
.split(/,|\band\b/gi)
|
||||
.map((entry) => entry.replace(/^[^a-zA-Z]+|[^a-zA-Z]+$/g, "").trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function pickFallbackThinkingLevel(params: {
|
||||
message?: string;
|
||||
attempted: Set<ThinkLevel>;
|
||||
}): ThinkLevel | undefined {
|
||||
const raw = params.message?.trim();
|
||||
if (!raw) return undefined;
|
||||
const supported = extractSupportedValues(raw);
|
||||
if (supported.length === 0) return undefined;
|
||||
for (const entry of supported) {
|
||||
const normalized = normalizeThinkLevel(entry);
|
||||
if (!normalized) continue;
|
||||
if (params.attempted.has(normalized)) continue;
|
||||
return normalized;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
ensureSessionHeader,
|
||||
formatAssistantErrorText,
|
||||
isRateLimitAssistantError,
|
||||
pickFallbackThinkingLevel,
|
||||
sanitizeSessionMessagesImages,
|
||||
} from "./pi-embedded-helpers.js";
|
||||
import {
|
||||
@@ -319,22 +320,27 @@ export async function runEmbeddedPiAgent(params: {
|
||||
const apiKey = await getApiKeyForModel(model, authStorage);
|
||||
authStorage.setRuntimeApiKey(model.provider, apiKey);
|
||||
|
||||
const thinkingLevel = mapThinkingLevel(params.thinkLevel);
|
||||
let thinkLevel = params.thinkLevel ?? "off";
|
||||
const attemptedThinking = new Set<ThinkLevel>();
|
||||
|
||||
log.debug(
|
||||
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} surface=${params.surface ?? "unknown"}`,
|
||||
);
|
||||
while (true) {
|
||||
const thinkingLevel = mapThinkingLevel(thinkLevel);
|
||||
attemptedThinking.add(thinkLevel);
|
||||
|
||||
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
||||
await ensureSessionHeader({
|
||||
sessionFile: params.sessionFile,
|
||||
sessionId: params.sessionId,
|
||||
cwd: resolvedWorkspace,
|
||||
});
|
||||
log.debug(
|
||||
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} thinking=${thinkLevel} surface=${params.surface ?? "unknown"}`,
|
||||
);
|
||||
|
||||
let restoreSkillEnv: (() => void) | undefined;
|
||||
process.chdir(resolvedWorkspace);
|
||||
try {
|
||||
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
||||
await ensureSessionHeader({
|
||||
sessionFile: params.sessionFile,
|
||||
sessionId: params.sessionId,
|
||||
cwd: resolvedWorkspace,
|
||||
});
|
||||
|
||||
let restoreSkillEnv: (() => void) | undefined;
|
||||
process.chdir(resolvedWorkspace);
|
||||
try {
|
||||
const shouldLoadSkillEntries =
|
||||
!params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills;
|
||||
const skillEntries = shouldLoadSkillEntries
|
||||
@@ -391,7 +397,7 @@ export async function runEmbeddedPiAgent(params: {
|
||||
const systemPrompt = buildSystemPrompt({
|
||||
appendPrompt: buildAgentSystemPromptAppend({
|
||||
workspaceDir: resolvedWorkspace,
|
||||
defaultThinkLevel: params.thinkLevel,
|
||||
defaultThinkLevel: thinkLevel,
|
||||
extraSystemPrompt: params.extraSystemPrompt,
|
||||
ownerNumbers: params.ownerNumbers,
|
||||
reasoningTagHint,
|
||||
@@ -542,6 +548,20 @@ export async function runEmbeddedPiAgent(params: {
|
||||
params.abortSignal?.removeEventListener?.("abort", onAbort);
|
||||
}
|
||||
if (promptError && !aborted) {
|
||||
const fallbackThinking = pickFallbackThinkingLevel({
|
||||
message:
|
||||
promptError instanceof Error
|
||||
? promptError.message
|
||||
: String(promptError),
|
||||
attempted: attemptedThinking,
|
||||
});
|
||||
if (fallbackThinking) {
|
||||
log.warn(
|
||||
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
||||
);
|
||||
thinkLevel = fallbackThinking;
|
||||
continue;
|
||||
}
|
||||
throw promptError;
|
||||
}
|
||||
|
||||
@@ -552,6 +572,18 @@ export async function runEmbeddedPiAgent(params: {
|
||||
| AssistantMessage
|
||||
| undefined;
|
||||
|
||||
const fallbackThinking = pickFallbackThinkingLevel({
|
||||
message: lastAssistant?.errorMessage,
|
||||
attempted: attemptedThinking,
|
||||
});
|
||||
if (fallbackThinking && !aborted) {
|
||||
log.warn(
|
||||
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
||||
);
|
||||
thinkLevel = fallbackThinking;
|
||||
continue;
|
||||
}
|
||||
|
||||
const fallbackConfigured =
|
||||
(params.config?.agent?.modelFallbacks?.length ?? 0) > 0;
|
||||
if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) {
|
||||
@@ -631,9 +663,10 @@ export async function runEmbeddedPiAgent(params: {
|
||||
aborted,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
restoreSkillEnv?.();
|
||||
process.chdir(prevCwd);
|
||||
} finally {
|
||||
restoreSkillEnv?.();
|
||||
process.chdir(prevCwd);
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user