feat: add models scan and fallbacks
This commit is contained in:
Peter Steinberger
150
src/agents/model-fallback.ts
Normal file
150
src/agents/model-fallback.ts
Normal file
@@ -0,0 +1,150 @@
|
||||
import type { ClawdbotConfig } from "../config/config.js";
|
||||
import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js";
|
||||
import {
|
||||
buildModelAliasIndex,
|
||||
modelKey,
|
||||
parseModelRef,
|
||||
resolveModelRefFromString,
|
||||
} from "./model-selection.js";
|
||||
|
||||
type ModelCandidate = {
|
||||
provider: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type FallbackAttempt = {
|
||||
provider: string;
|
||||
model: string;
|
||||
error: string;
|
||||
};
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
if (!err || typeof err !== "object") return false;
|
||||
const name = "name" in err ? String(err.name) : "";
|
||||
if (name === "AbortError") return true;
|
||||
const message =
|
||||
"message" in err && typeof err.message === "string"
|
||||
? err.message.toLowerCase()
|
||||
: "";
|
||||
return message.includes("aborted");
|
||||
}
|
||||
|
||||
function buildAllowedModelKeys(
|
||||
cfg: ClawdbotConfig | undefined,
|
||||
defaultProvider: string,
|
||||
): Set<string> | null {
|
||||
const rawAllowlist = cfg?.agent?.allowedModels ?? [];
|
||||
if (rawAllowlist.length === 0) return null;
|
||||
const keys = new Set<string>();
|
||||
for (const raw of rawAllowlist) {
|
||||
const parsed = parseModelRef(String(raw ?? ""), defaultProvider);
|
||||
if (!parsed) continue;
|
||||
keys.add(modelKey(parsed.provider, parsed.model));
|
||||
}
|
||||
return keys.size > 0 ? keys : null;
|
||||
}
|
||||
|
||||
function resolveFallbackCandidates(params: {
|
||||
cfg: ClawdbotConfig | undefined;
|
||||
provider: string;
|
||||
model: string;
|
||||
}): ModelCandidate[] {
|
||||
const provider = params.provider.trim() || DEFAULT_PROVIDER;
|
||||
const model = params.model.trim() || DEFAULT_MODEL;
|
||||
const aliasIndex = buildModelAliasIndex({
|
||||
cfg: params.cfg ?? {},
|
||||
defaultProvider: DEFAULT_PROVIDER,
|
||||
});
|
||||
const allowlist = buildAllowedModelKeys(params.cfg, DEFAULT_PROVIDER);
|
||||
const seen = new Set<string>();
|
||||
const candidates: ModelCandidate[] = [];
|
||||
|
||||
const addCandidate = (candidate: ModelCandidate, enforceAllowlist: boolean) => {
|
||||
if (!candidate.provider || !candidate.model) return;
|
||||
const key = modelKey(candidate.provider, candidate.model);
|
||||
if (seen.has(key)) return;
|
||||
if (enforceAllowlist && allowlist && !allowlist.has(key)) return;
|
||||
seen.add(key);
|
||||
candidates.push(candidate);
|
||||
};
|
||||
|
||||
addCandidate({ provider, model }, false);
|
||||
|
||||
for (const raw of params.cfg?.agent?.modelFallbacks ?? []) {
|
||||
const resolved = resolveModelRefFromString({
|
||||
raw: String(raw ?? ""),
|
||||
defaultProvider: DEFAULT_PROVIDER,
|
||||
aliasIndex,
|
||||
});
|
||||
if (!resolved) continue;
|
||||
addCandidate(resolved.ref, true);
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
export async function runWithModelFallback<T>(params: {
|
||||
cfg: ClawdbotConfig | undefined;
|
||||
provider: string;
|
||||
model: string;
|
||||
run: (provider: string, model: string) => Promise<T>;
|
||||
onError?: (attempt: {
|
||||
provider: string;
|
||||
model: string;
|
||||
error: unknown;
|
||||
attempt: number;
|
||||
total: number;
|
||||
}) => void | Promise<void>;
|
||||
}): Promise<{
|
||||
result: T;
|
||||
provider: string;
|
||||
model: string;
|
||||
attempts: FallbackAttempt[];
|
||||
}> {
|
||||
const candidates = resolveFallbackCandidates(params);
|
||||
const attempts: FallbackAttempt[] = [];
|
||||
let lastError: unknown;
|
||||
|
||||
for (let i = 0; i < candidates.length; i += 1) {
|
||||
const candidate = candidates[i] as ModelCandidate;
|
||||
try {
|
||||
const result = await params.run(candidate.provider, candidate.model);
|
||||
return {
|
||||
result,
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
attempts,
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
lastError = err;
|
||||
attempts.push({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
await params.onError?.({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error: err,
|
||||
attempt: i + 1,
|
||||
total: candidates.length,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (attempts.length <= 1 && lastError) throw lastError;
|
||||
const summary =
|
||||
attempts.length > 0
|
||||
? attempts
|
||||
.map(
|
||||
(attempt) =>
|
||||
`${attempt.provider}/${attempt.model}: ${attempt.error}`,
|
||||
)
|
||||
.join(" | ")
|
||||
: "unknown";
|
||||
throw new Error(
|
||||
`All models failed (${attempts.length || candidates.length}): ${summary}`,
|
||||
{ cause: lastError instanceof Error ? lastError : undefined },
|
||||
);
|
||||
}
|
||||
379
src/agents/model-scan.ts
Normal file
379
src/agents/model-scan.ts
Normal file
@@ -0,0 +1,379 @@
|
||||
import { Type } from "@sinclair/typebox";
|
||||
import {
|
||||
complete,
|
||||
getEnvApiKey,
|
||||
getModel,
|
||||
type Context,
|
||||
type Model,
|
||||
type Tool,
|
||||
type OpenAICompletionsOptions,
|
||||
} from "@mariozechner/pi-ai";
|
||||
|
||||
const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models";
|
||||
const DEFAULT_TIMEOUT_MS = 12_000;
|
||||
const DEFAULT_CONCURRENCY = 3;
|
||||
|
||||
const BASE_IMAGE_PNG =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+X3mIAAAAASUVORK5CYII=";
|
||||
|
||||
const TOOL_PING: Tool = {
|
||||
name: "ping",
|
||||
description: "Return OK.",
|
||||
parameters: Type.Object({}),
|
||||
};
|
||||
|
||||
type OpenRouterModelMeta = {
|
||||
id: string;
|
||||
name: string;
|
||||
contextLength: number | null;
|
||||
maxCompletionTokens: number | null;
|
||||
supportedParametersCount: number;
|
||||
modality: string | null;
|
||||
inferredParamB: number | null;
|
||||
createdAtMs: number | null;
|
||||
};
|
||||
|
||||
export type ProbeResult = {
|
||||
ok: boolean;
|
||||
latencyMs: number | null;
|
||||
error?: string;
|
||||
skipped?: boolean;
|
||||
};
|
||||
|
||||
export type ModelScanResult = {
|
||||
id: string;
|
||||
name: string;
|
||||
provider: string;
|
||||
modelRef: string;
|
||||
contextLength: number | null;
|
||||
maxCompletionTokens: number | null;
|
||||
supportedParametersCount: number;
|
||||
modality: string | null;
|
||||
inferredParamB: number | null;
|
||||
createdAtMs: number | null;
|
||||
tool: ProbeResult;
|
||||
image: ProbeResult;
|
||||
};
|
||||
|
||||
export type OpenRouterScanOptions = {
|
||||
apiKey?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
timeoutMs?: number;
|
||||
concurrency?: number;
|
||||
minParamB?: number;
|
||||
maxAgeDays?: number;
|
||||
providerFilter?: string;
|
||||
};
|
||||
|
||||
type OpenAIModel = Model<"openai-completions">;
|
||||
|
||||
function normalizeCreatedAtMs(value: unknown): number | null {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return null;
|
||||
if (value <= 0) return null;
|
||||
if (value > 1e12) return Math.round(value);
|
||||
return Math.round(value * 1000);
|
||||
}
|
||||
|
||||
function inferParamBFromIdOrName(text: string): number | null {
|
||||
const raw = text.toLowerCase();
|
||||
const matches = raw.matchAll(
|
||||
/(?:^|[^a-z0-9])[a-z]?(\d+(?:\.\d+)?)b(?:[^a-z0-9]|$)/g,
|
||||
);
|
||||
let best: number | null = null;
|
||||
for (const match of matches) {
|
||||
const numRaw = match[1];
|
||||
if (!numRaw) continue;
|
||||
const value = Number(numRaw);
|
||||
if (!Number.isFinite(value) || value <= 0) continue;
|
||||
if (best === null || value > best) best = value;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function parseModality(modality: string | null): Array<"text" | "image"> {
|
||||
if (!modality) return ["text"];
|
||||
const normalized = modality.toLowerCase();
|
||||
const parts = normalized.split(/[^a-z]+/).filter(Boolean);
|
||||
const hasImage = parts.includes("image");
|
||||
return hasImage ? ["text", "image"] : ["text"];
|
||||
}
|
||||
|
||||
async function withTimeout<T>(
|
||||
timeoutMs: number,
|
||||
fn: (signal: AbortSignal) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fn(controller.signal);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchOpenRouterModels(
|
||||
fetchImpl: typeof fetch,
|
||||
): Promise<OpenRouterModelMeta[]> {
|
||||
const res = await fetchImpl(OPENROUTER_MODELS_URL, {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`OpenRouter /models failed: HTTP ${res.status}`);
|
||||
}
|
||||
const payload = (await res.json()) as { data?: unknown };
|
||||
const entries = Array.isArray(payload.data) ? payload.data : [];
|
||||
|
||||
return entries
|
||||
.map((entry) => {
|
||||
if (!entry || typeof entry !== "object") return null;
|
||||
const obj = entry as Record<string, unknown>;
|
||||
const id = typeof obj.id === "string" ? obj.id.trim() : "";
|
||||
if (!id) return null;
|
||||
const name =
|
||||
typeof obj.name === "string" && obj.name.trim()
|
||||
? obj.name.trim()
|
||||
: id;
|
||||
|
||||
const contextLength =
|
||||
typeof obj.context_length === "number" &&
|
||||
Number.isFinite(obj.context_length)
|
||||
? obj.context_length
|
||||
: null;
|
||||
|
||||
const maxCompletionTokens =
|
||||
typeof obj.max_completion_tokens === "number" &&
|
||||
Number.isFinite(obj.max_completion_tokens)
|
||||
? obj.max_completion_tokens
|
||||
: typeof obj.max_output_tokens === "number" &&
|
||||
Number.isFinite(obj.max_output_tokens)
|
||||
? obj.max_output_tokens
|
||||
: null;
|
||||
|
||||
const supportedParametersCount = Array.isArray(obj.supported_parameters)
|
||||
? obj.supported_parameters.length
|
||||
: 0;
|
||||
|
||||
const modality =
|
||||
typeof obj.modality === "string" && obj.modality.trim()
|
||||
? obj.modality.trim()
|
||||
: null;
|
||||
|
||||
const inferredParamB = inferParamBFromIdOrName(`${id} ${name}`);
|
||||
const createdAtMs = normalizeCreatedAtMs(obj.created_at);
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
contextLength,
|
||||
maxCompletionTokens,
|
||||
supportedParametersCount,
|
||||
modality,
|
||||
inferredParamB,
|
||||
createdAtMs,
|
||||
} satisfies OpenRouterModelMeta;
|
||||
})
|
||||
.filter((entry): entry is OpenRouterModelMeta => Boolean(entry));
|
||||
}
|
||||
|
||||
async function probeTool(
|
||||
model: OpenAIModel,
|
||||
apiKey: string,
|
||||
timeoutMs: number,
|
||||
): Promise<ProbeResult> {
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: "Call the ping tool with {} and nothing else.",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
tools: [TOOL_PING],
|
||||
};
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const message = await withTimeout(timeoutMs, (signal) =>
|
||||
complete(model, context, {
|
||||
apiKey,
|
||||
maxTokens: 32,
|
||||
temperature: 0,
|
||||
toolChoice: "required",
|
||||
signal,
|
||||
} satisfies OpenAICompletionsOptions),
|
||||
);
|
||||
|
||||
const hasToolCall = message.content.some(
|
||||
(block) => block.type === "toolCall",
|
||||
);
|
||||
if (!hasToolCall) {
|
||||
return {
|
||||
ok: false,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
error: "No tool call returned",
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: true, latencyMs: Date.now() - startedAt };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function probeImage(
|
||||
model: OpenAIModel,
|
||||
apiKey: string,
|
||||
timeoutMs: number,
|
||||
): Promise<ProbeResult> {
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Reply with OK." },
|
||||
{ type: "image", data: BASE_IMAGE_PNG, mimeType: "image/png" },
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
};
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
await withTimeout(timeoutMs, (signal) =>
|
||||
complete(model, context, {
|
||||
apiKey,
|
||||
maxTokens: 16,
|
||||
temperature: 0,
|
||||
signal,
|
||||
} satisfies OpenAICompletionsOptions),
|
||||
);
|
||||
return { ok: true, latencyMs: Date.now() - startedAt };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
latencyMs: Date.now() - startedAt,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function ensureImageInput(model: OpenAIModel): OpenAIModel {
|
||||
if (model.input.includes("image")) return model;
|
||||
return {
|
||||
...model,
|
||||
input: Array.from(new Set([...model.input, "image"])),
|
||||
};
|
||||
}
|
||||
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const limit = Math.max(1, Math.floor(concurrency));
|
||||
const results: R[] = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
|
||||
const worker = async () => {
|
||||
while (true) {
|
||||
const current = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (current >= items.length) return;
|
||||
results[current] = await fn(items[current] as T, current);
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, () => worker()),
|
||||
);
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function scanOpenRouterModels(
|
||||
options: OpenRouterScanOptions = {},
|
||||
): Promise<ModelScanResult[]> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
const apiKey =
|
||||
options.apiKey?.trim() || getEnvApiKey("openrouter") || "";
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"Missing OpenRouter API key. Set OPENROUTER_API_KEY to run models scan.",
|
||||
);
|
||||
}
|
||||
|
||||
const timeoutMs = Math.max(
|
||||
1,
|
||||
Math.floor(options.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
||||
);
|
||||
const concurrency = Math.max(
|
||||
1,
|
||||
Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY),
|
||||
);
|
||||
const minParamB = Math.max(0, Math.floor(options.minParamB ?? 0));
|
||||
const maxAgeDays = Math.max(0, Math.floor(options.maxAgeDays ?? 0));
|
||||
const providerFilter = options.providerFilter?.trim().toLowerCase() ?? "";
|
||||
|
||||
const catalog = await fetchOpenRouterModels(fetchImpl);
|
||||
const now = Date.now();
|
||||
|
||||
const filtered = catalog.filter((entry) => {
|
||||
if (!entry.id.endsWith(":free")) return false;
|
||||
if (providerFilter) {
|
||||
const prefix = entry.id.split("/")[0]?.toLowerCase() ?? "";
|
||||
if (prefix !== providerFilter) return false;
|
||||
}
|
||||
if (minParamB > 0) {
|
||||
const params = entry.inferredParamB ?? 0;
|
||||
if (params < minParamB) return false;
|
||||
}
|
||||
if (maxAgeDays > 0 && entry.createdAtMs) {
|
||||
const ageMs = now - entry.createdAtMs;
|
||||
const ageDays = ageMs / (24 * 60 * 60 * 1000);
|
||||
if (ageDays > maxAgeDays) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const baseModel = getModel(
|
||||
"openrouter",
|
||||
"openrouter/auto",
|
||||
) as OpenAIModel;
|
||||
|
||||
return mapWithConcurrency(filtered, concurrency, async (entry) => {
|
||||
const model: OpenAIModel = {
|
||||
...baseModel,
|
||||
id: entry.id,
|
||||
name: entry.name || entry.id,
|
||||
contextWindow: entry.contextLength ?? baseModel.contextWindow,
|
||||
maxTokens: entry.maxCompletionTokens ?? baseModel.maxTokens,
|
||||
input: parseModality(entry.modality),
|
||||
reasoning: baseModel.reasoning,
|
||||
};
|
||||
|
||||
const toolResult = await probeTool(model, apiKey, timeoutMs);
|
||||
const imageResult = toolResult.ok
|
||||
? await probeImage(ensureImageInput(model), apiKey, timeoutMs)
|
||||
: { ok: false, latencyMs: null, skipped: true };
|
||||
|
||||
return {
|
||||
id: entry.id,
|
||||
name: entry.name,
|
||||
provider: "openrouter",
|
||||
modelRef: `openrouter/${entry.id}`,
|
||||
contextLength: entry.contextLength,
|
||||
maxCompletionTokens: entry.maxCompletionTokens,
|
||||
supportedParametersCount: entry.supportedParametersCount,
|
||||
modality: entry.modality,
|
||||
inferredParamB: entry.inferredParamB,
|
||||
createdAtMs: entry.createdAtMs,
|
||||
tool: toolResult,
|
||||
image: imageResult,
|
||||
} satisfies ModelScanResult;
|
||||
});
|
||||
}
|
||||
|
||||
export { OPENROUTER_MODELS_URL };
|
||||
export type { OpenRouterModelMeta };
|
||||
Reference in New Issue
Block a user