fix(model): retry with supported thinking level
This commit is contained in:
@@ -1,7 +1,11 @@
|
|||||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import { isRateLimitAssistantError } from "./pi-embedded-helpers.js";
|
import {
|
||||||
|
isRateLimitAssistantError,
|
||||||
|
pickFallbackThinkingLevel,
|
||||||
|
} from "./pi-embedded-helpers.js";
|
||||||
|
import type { ThinkLevel } from "../auto-reply/thinking.js";
|
||||||
|
|
||||||
const asAssistant = (overrides: Partial<AssistantMessage>) =>
|
const asAssistant = (overrides: Partial<AssistantMessage>) =>
|
||||||
({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage;
|
({ role: "assistant", stopReason: "error", ...overrides }) as AssistantMessage;
|
||||||
@@ -30,3 +34,34 @@ describe("isRateLimitAssistantError", () => {
|
|||||||
expect(isRateLimitAssistantError(msg)).toBe(false);
|
expect(isRateLimitAssistantError(msg)).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("pickFallbackThinkingLevel", () => {
|
||||||
|
it("selects the first supported thinking level", () => {
|
||||||
|
const attempted = new Set<ThinkLevel>(["low"]);
|
||||||
|
const next = pickFallbackThinkingLevel({
|
||||||
|
message:
|
||||||
|
"Unsupported value: 'low' is not supported with the 'gpt-5.2-pro' model. Supported values are: 'medium', 'high', and 'xhigh'.",
|
||||||
|
attempted,
|
||||||
|
});
|
||||||
|
expect(next).toBe("medium");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips already attempted levels", () => {
|
||||||
|
const attempted = new Set<ThinkLevel>(["low", "medium"]);
|
||||||
|
const next = pickFallbackThinkingLevel({
|
||||||
|
message:
|
||||||
|
"Supported values are: 'medium', 'high', and 'xhigh'.",
|
||||||
|
attempted,
|
||||||
|
});
|
||||||
|
expect(next).toBe("high");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns undefined when no supported values are found", () => {
|
||||||
|
const attempted = new Set<ThinkLevel>(["low"]);
|
||||||
|
const next = pickFallbackThinkingLevel({
|
||||||
|
message: "Request failed.",
|
||||||
|
attempted,
|
||||||
|
});
|
||||||
|
expect(next).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import type {
|
|||||||
AgentToolResult,
|
AgentToolResult,
|
||||||
} from "@mariozechner/pi-agent-core";
|
} from "@mariozechner/pi-agent-core";
|
||||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||||
|
import { normalizeThinkLevel, type ThinkLevel } from "../auto-reply/thinking.js";
|
||||||
|
|
||||||
import { sanitizeContentBlocksImages } from "./tool-images.js";
|
import { sanitizeContentBlocksImages } from "./tool-images.js";
|
||||||
import type { WorkspaceBootstrapFile } from "./workspace.js";
|
import type { WorkspaceBootstrapFile } from "./workspace.js";
|
||||||
@@ -118,3 +119,38 @@ export function isRateLimitAssistantError(
|
|||||||
if (!raw) return false;
|
if (!raw) return false;
|
||||||
return /rate[_ ]limit|too many requests|429/.test(raw);
|
return /rate[_ ]limit|too many requests|429/.test(raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractSupportedValues(raw: string): string[] {
|
||||||
|
const match =
|
||||||
|
raw.match(/supported values are:\s*([^\n.]+)/i) ??
|
||||||
|
raw.match(/supported values:\s*([^\n.]+)/i);
|
||||||
|
if (!match?.[1]) return [];
|
||||||
|
const fragment = match[1];
|
||||||
|
const quoted = Array.from(fragment.matchAll(/['"]([^'"]+)['"]/g)).map(
|
||||||
|
(entry) => entry[1]?.trim(),
|
||||||
|
);
|
||||||
|
if (quoted.length > 0) {
|
||||||
|
return quoted.filter((entry): entry is string => Boolean(entry));
|
||||||
|
}
|
||||||
|
return fragment
|
||||||
|
.split(/,|\band\b/gi)
|
||||||
|
.map((entry) => entry.replace(/^[^a-zA-Z]+|[^a-zA-Z]+$/g, "").trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function pickFallbackThinkingLevel(params: {
|
||||||
|
message?: string;
|
||||||
|
attempted: Set<ThinkLevel>;
|
||||||
|
}): ThinkLevel | undefined {
|
||||||
|
const raw = params.message?.trim();
|
||||||
|
if (!raw) return undefined;
|
||||||
|
const supported = extractSupportedValues(raw);
|
||||||
|
if (supported.length === 0) return undefined;
|
||||||
|
for (const entry of supported) {
|
||||||
|
const normalized = normalizeThinkLevel(entry);
|
||||||
|
if (!normalized) continue;
|
||||||
|
if (params.attempted.has(normalized)) continue;
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import {
|
|||||||
ensureSessionHeader,
|
ensureSessionHeader,
|
||||||
formatAssistantErrorText,
|
formatAssistantErrorText,
|
||||||
isRateLimitAssistantError,
|
isRateLimitAssistantError,
|
||||||
|
pickFallbackThinkingLevel,
|
||||||
sanitizeSessionMessagesImages,
|
sanitizeSessionMessagesImages,
|
||||||
} from "./pi-embedded-helpers.js";
|
} from "./pi-embedded-helpers.js";
|
||||||
import {
|
import {
|
||||||
@@ -319,22 +320,27 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
const apiKey = await getApiKeyForModel(model, authStorage);
|
const apiKey = await getApiKeyForModel(model, authStorage);
|
||||||
authStorage.setRuntimeApiKey(model.provider, apiKey);
|
authStorage.setRuntimeApiKey(model.provider, apiKey);
|
||||||
|
|
||||||
const thinkingLevel = mapThinkingLevel(params.thinkLevel);
|
let thinkLevel = params.thinkLevel ?? "off";
|
||||||
|
const attemptedThinking = new Set<ThinkLevel>();
|
||||||
|
|
||||||
log.debug(
|
while (true) {
|
||||||
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} surface=${params.surface ?? "unknown"}`,
|
const thinkingLevel = mapThinkingLevel(thinkLevel);
|
||||||
);
|
attemptedThinking.add(thinkLevel);
|
||||||
|
|
||||||
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
log.debug(
|
||||||
await ensureSessionHeader({
|
`embedded run start: runId=${params.runId} sessionId=${params.sessionId} provider=${provider} model=${modelId} thinking=${thinkLevel} surface=${params.surface ?? "unknown"}`,
|
||||||
sessionFile: params.sessionFile,
|
);
|
||||||
sessionId: params.sessionId,
|
|
||||||
cwd: resolvedWorkspace,
|
|
||||||
});
|
|
||||||
|
|
||||||
let restoreSkillEnv: (() => void) | undefined;
|
await fs.mkdir(resolvedWorkspace, { recursive: true });
|
||||||
process.chdir(resolvedWorkspace);
|
await ensureSessionHeader({
|
||||||
try {
|
sessionFile: params.sessionFile,
|
||||||
|
sessionId: params.sessionId,
|
||||||
|
cwd: resolvedWorkspace,
|
||||||
|
});
|
||||||
|
|
||||||
|
let restoreSkillEnv: (() => void) | undefined;
|
||||||
|
process.chdir(resolvedWorkspace);
|
||||||
|
try {
|
||||||
const shouldLoadSkillEntries =
|
const shouldLoadSkillEntries =
|
||||||
!params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills;
|
!params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills;
|
||||||
const skillEntries = shouldLoadSkillEntries
|
const skillEntries = shouldLoadSkillEntries
|
||||||
@@ -391,7 +397,7 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
const systemPrompt = buildSystemPrompt({
|
const systemPrompt = buildSystemPrompt({
|
||||||
appendPrompt: buildAgentSystemPromptAppend({
|
appendPrompt: buildAgentSystemPromptAppend({
|
||||||
workspaceDir: resolvedWorkspace,
|
workspaceDir: resolvedWorkspace,
|
||||||
defaultThinkLevel: params.thinkLevel,
|
defaultThinkLevel: thinkLevel,
|
||||||
extraSystemPrompt: params.extraSystemPrompt,
|
extraSystemPrompt: params.extraSystemPrompt,
|
||||||
ownerNumbers: params.ownerNumbers,
|
ownerNumbers: params.ownerNumbers,
|
||||||
reasoningTagHint,
|
reasoningTagHint,
|
||||||
@@ -542,6 +548,20 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
params.abortSignal?.removeEventListener?.("abort", onAbort);
|
params.abortSignal?.removeEventListener?.("abort", onAbort);
|
||||||
}
|
}
|
||||||
if (promptError && !aborted) {
|
if (promptError && !aborted) {
|
||||||
|
const fallbackThinking = pickFallbackThinkingLevel({
|
||||||
|
message:
|
||||||
|
promptError instanceof Error
|
||||||
|
? promptError.message
|
||||||
|
: String(promptError),
|
||||||
|
attempted: attemptedThinking,
|
||||||
|
});
|
||||||
|
if (fallbackThinking) {
|
||||||
|
log.warn(
|
||||||
|
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
||||||
|
);
|
||||||
|
thinkLevel = fallbackThinking;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
throw promptError;
|
throw promptError;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -552,6 +572,18 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
| AssistantMessage
|
| AssistantMessage
|
||||||
| undefined;
|
| undefined;
|
||||||
|
|
||||||
|
const fallbackThinking = pickFallbackThinkingLevel({
|
||||||
|
message: lastAssistant?.errorMessage,
|
||||||
|
attempted: attemptedThinking,
|
||||||
|
});
|
||||||
|
if (fallbackThinking && !aborted) {
|
||||||
|
log.warn(
|
||||||
|
`unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`,
|
||||||
|
);
|
||||||
|
thinkLevel = fallbackThinking;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const fallbackConfigured =
|
const fallbackConfigured =
|
||||||
(params.config?.agent?.modelFallbacks?.length ?? 0) > 0;
|
(params.config?.agent?.modelFallbacks?.length ?? 0) > 0;
|
||||||
if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) {
|
if (fallbackConfigured && isRateLimitAssistantError(lastAssistant)) {
|
||||||
@@ -631,9 +663,10 @@ export async function runEmbeddedPiAgent(params: {
|
|||||||
aborted,
|
aborted,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
restoreSkillEnv?.();
|
restoreSkillEnv?.();
|
||||||
process.chdir(prevCwd);
|
process.chdir(prevCwd);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|||||||
Reference in New Issue
Block a user