From 3178a3014dbba92bcbcbf5b342e19e9baac8fce9 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 8 Jan 2026 04:20:34 +0100 Subject: [PATCH] feat(models): improve OpenRouter free scan --- src/agents/model-scan.test.ts | 84 ++++++++++++++++++++++++++++ src/agents/model-scan.ts | 100 ++++++++++++++++++++++++++++++++-- src/cli/models-cli.ts | 17 +++++- src/commands/models/scan.ts | 33 ++++++++--- 4 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 src/agents/model-scan.test.ts diff --git a/src/agents/model-scan.test.ts b/src/agents/model-scan.test.ts new file mode 100644 index 000000000..fc2a34a1f --- /dev/null +++ b/src/agents/model-scan.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; + +import { scanOpenRouterModels } from "./model-scan.js"; + +function createFetchFixture(payload: unknown): typeof fetch { + return async () => + new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} + +describe("scanOpenRouterModels", () => { + it("lists free models without probing", async () => { + const fetchImpl = createFetchFixture({ + data: [ + { + id: "acme/free-by-pricing", + name: "Free By Pricing", + context_length: 16_384, + max_completion_tokens: 1024, + supported_parameters: ["tools", "tool_choice", "temperature"], + modality: "text", + pricing: { prompt: "0", completion: "0", request: "0", image: "0" }, + created_at: 1_700_000_000, + }, + { + id: "acme/free-by-suffix:free", + name: "Free By Suffix", + context_length: 8_192, + supported_parameters: [], + modality: "text", + pricing: { prompt: "0", completion: "0" }, + }, + { + id: "acme/paid", + name: "Paid", + context_length: 4_096, + supported_parameters: ["tools"], + modality: "text", + pricing: { prompt: "0.000001", completion: "0.000002" }, + }, + ], + }); + + const results = await scanOpenRouterModels({ + fetchImpl, + probe: false, + }); + + expect(results.map((entry) => entry.id)).toEqual([ + "acme/free-by-pricing", + "acme/free-by-suffix:free", + ]); + + const byPricing = results[0]!; + expect(byPricing.supportsToolsMeta).toBe(true); + expect(byPricing.supportedParametersCount).toBe(3); + expect(byPricing.isFree).toBe(true); + expect(byPricing.tool.skipped).toBe(true); + expect(byPricing.image.skipped).toBe(true); + }); + + it("requires an API key when probing", async () => { + const fetchImpl = createFetchFixture({ data: [] }); + const previousKey = process.env.OPENROUTER_API_KEY; + try { + delete process.env.OPENROUTER_API_KEY; + await expect( + scanOpenRouterModels({ + fetchImpl, + probe: true, + apiKey: "", + }), + ).rejects.toThrow(/Missing OpenRouter API key/); + } finally { + if (previousKey === undefined) { + delete process.env.OPENROUTER_API_KEY; + } else { + process.env.OPENROUTER_API_KEY = previousKey; + } + } + }); +}); diff --git a/src/agents/model-scan.ts b/src/agents/model-scan.ts index ff6825659..32727997a 100644 --- a/src/agents/model-scan.ts +++ b/src/agents/model-scan.ts @@ -27,10 +27,22 @@ type OpenRouterModelMeta = { name: string; contextLength: number | null; maxCompletionTokens: number | null; + supportedParameters: string[]; supportedParametersCount: number; + supportsToolsMeta: boolean; modality: string | null; inferredParamB: number | null; createdAtMs: number | null; + pricing: OpenRouterModelPricing | null; +}; + +type OpenRouterModelPricing = { + prompt: number; + completion: number; + request: number; + image: number; + webSearch: number; + internalReasoning: number; }; export type ProbeResult = { @@ -48,9 +60,12 @@ export type ModelScanResult = { contextLength: number | null; maxCompletionTokens: number | null; supportedParametersCount: number; + supportsToolsMeta: boolean; modality: string | null; inferredParamB: number | null; createdAtMs: number | null; + pricing: OpenRouterModelPricing | null; + isFree: boolean; tool: ProbeResult; image: ProbeResult; }; @@ -63,6 +78,7 @@ export type OpenRouterScanOptions = { minParamB?: number; maxAgeDays?: number; providerFilter?: string; + probe?: boolean; }; type OpenAIModel = Model<"openai-completions">; @@ -98,6 +114,43 @@ function parseModality(modality: string | null): Array<"text" | "image"> { return hasImage ? ["text", "image"] : ["text"]; } +function parseNumberString(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed) return null; + const num = Number(trimmed); + if (!Number.isFinite(num)) return null; + return num; +} + +function parseOpenRouterPricing(value: unknown): OpenRouterModelPricing | null { + if (!value || typeof value !== "object") return null; + const obj = value as Record; + const prompt = parseNumberString(obj.prompt); + const completion = parseNumberString(obj.completion); + const request = parseNumberString(obj.request) ?? 0; + const image = parseNumberString(obj.image) ?? 0; + const webSearch = parseNumberString(obj.web_search) ?? 0; + const internalReasoning = parseNumberString(obj.internal_reasoning) ?? 0; + + if (prompt === null || completion === null) return null; + return { + prompt, + completion, + request, + image, + webSearch, + internalReasoning, + }; +} + +function isFreeOpenRouterModel(entry: OpenRouterModelMeta): boolean { + if (entry.id.endsWith(":free")) return true; + if (!entry.pricing) return false; + return entry.pricing.prompt === 0 && entry.pricing.completion === 0; +} + async function withTimeout( timeoutMs: number, fn: (signal: AbortSignal) => Promise, @@ -147,9 +200,15 @@ async function fetchOpenRouterModels( ? obj.max_output_tokens : null; - const supportedParametersCount = Array.isArray(obj.supported_parameters) - ? obj.supported_parameters.length - : 0; + const supportedParameters = Array.isArray(obj.supported_parameters) + ? obj.supported_parameters + .filter((value): value is string => typeof value === "string") + .map((value) => value.trim()) + .filter(Boolean) + : []; + + const supportedParametersCount = supportedParameters.length; + const supportsToolsMeta = supportedParameters.includes("tools"); const modality = typeof obj.modality === "string" && obj.modality.trim() @@ -158,16 +217,20 @@ async function fetchOpenRouterModels( const inferredParamB = inferParamBFromIdOrName(`${id} ${name}`); const createdAtMs = normalizeCreatedAtMs(obj.created_at); + const pricing = parseOpenRouterPricing(obj.pricing); return { id, name, contextLength, maxCompletionTokens, + supportedParameters, supportedParametersCount, + supportsToolsMeta, modality, inferredParamB, createdAtMs, + pricing, } satisfies OpenRouterModelMeta; }) .filter((entry): entry is OpenRouterModelMeta => Boolean(entry)); @@ -294,8 +357,9 @@ export async function scanOpenRouterModels( options: OpenRouterScanOptions = {}, ): Promise { const fetchImpl = options.fetchImpl ?? fetch; + const probe = options.probe ?? true; const apiKey = options.apiKey?.trim() || getEnvApiKey("openrouter") || ""; - if (!apiKey) { + if (probe && !apiKey) { throw new Error( "Missing OpenRouter API key. Set OPENROUTER_API_KEY to run models scan.", ); @@ -317,7 +381,7 @@ export async function scanOpenRouterModels( const now = Date.now(); const filtered = catalog.filter((entry) => { - if (!entry.id.endsWith(":free")) return false; + if (!isFreeOpenRouterModel(entry)) return false; if (providerFilter) { const prefix = entry.id.split("/")[0]?.toLowerCase() ?? ""; if (prefix !== providerFilter) return false; @@ -337,6 +401,27 @@ export async function scanOpenRouterModels( const baseModel = getModel("openrouter", "openrouter/auto") as OpenAIModel; return mapWithConcurrency(filtered, concurrency, async (entry) => { + const isFree = isFreeOpenRouterModel(entry); + if (!probe) { + return { + id: entry.id, + name: entry.name, + provider: "openrouter", + modelRef: `openrouter/${entry.id}`, + contextLength: entry.contextLength, + maxCompletionTokens: entry.maxCompletionTokens, + supportedParametersCount: entry.supportedParametersCount, + supportsToolsMeta: entry.supportsToolsMeta, + modality: entry.modality, + inferredParamB: entry.inferredParamB, + createdAtMs: entry.createdAtMs, + pricing: entry.pricing, + isFree, + tool: { ok: false, latencyMs: null, skipped: true }, + image: { ok: false, latencyMs: null, skipped: true }, + } satisfies ModelScanResult; + } + const model: OpenAIModel = { ...baseModel, id: entry.id, @@ -360,9 +445,12 @@ export async function scanOpenRouterModels( contextLength: entry.contextLength, maxCompletionTokens: entry.maxCompletionTokens, supportedParametersCount: entry.supportedParametersCount, + supportsToolsMeta: entry.supportsToolsMeta, modality: entry.modality, inferredParamB: entry.inferredParamB, createdAtMs: entry.createdAtMs, + pricing: entry.pricing, + isFree, tool: toolResult, image: imageResult, } satisfies ModelScanResult; @@ -370,4 +458,4 @@ export async function scanOpenRouterModels( } export { OPENROUTER_MODELS_URL }; -export type { OpenRouterModelMeta }; +export type { OpenRouterModelMeta, OpenRouterModelPricing }; diff --git a/src/cli/models-cli.ts b/src/cli/models-cli.ts index 9ee9e64f4..484f512ea 100644 --- a/src/cli/models-cli.ts +++ b/src/cli/models-cli.ts @@ -24,9 +24,13 @@ export function registerModelsCli(program: Command) { const models = program .command("models") .description("Model discovery, scanning, and configuration") - .option("--json", "Output JSON (alias for `models status --json`)", false) .option( - "--plain", + "--status-json", + "Output JSON (alias for `models status --json`)", + false, + ) + .option( + "--status-plain", "Plain output (alias for `models status --plain`)", false, ); @@ -252,6 +256,7 @@ export function registerModelsCli(program: Command) { .option("--max-candidates ", "Max fallback candidates", "6") .option("--timeout ", "Per-probe timeout in ms") .option("--concurrency ", "Probe concurrency") + .option("--no-probe", "Skip live probes; list free candidates only") .option("--yes", "Accept defaults without prompting", false) .option("--no-input", "Disable prompts (use defaults)") .option("--set-default", "Set agent.model to the first selection", false) @@ -272,7 +277,13 @@ export function registerModelsCli(program: Command) { models.action(async (opts) => { try { - await modelsStatusCommand(opts ?? {}, defaultRuntime); + await modelsStatusCommand( + { + json: Boolean(opts?.statusJson), + plain: Boolean(opts?.statusPlain), + }, + defaultRuntime, + ); } catch (err) { defaultRuntime.error(String(err)); defaultRuntime.exit(1); diff --git a/src/commands/models/scan.ts b/src/commands/models/scan.ts index 6a2a058e2..cd77ad340 100644 --- a/src/commands/models/scan.ts +++ b/src/commands/models/scan.ts @@ -140,6 +140,7 @@ export async function modelsScanCommand( setDefault?: boolean; setImage?: boolean; json?: boolean; + probe?: boolean; }, runtime: RuntimeEnv, ) { @@ -174,15 +175,18 @@ export async function modelsScanCommand( } const cfg = loadConfig(); + const probe = opts.probe ?? true; let storedKey: string | undefined; - try { - const resolved = await resolveApiKeyForProvider({ - provider: "openrouter", - cfg, - }); - storedKey = resolved.apiKey; - } catch { - storedKey = undefined; + if (probe) { + try { + const resolved = await resolveApiKeyForProvider({ + provider: "openrouter", + cfg, + }); + storedKey = resolved.apiKey; + } catch { + storedKey = undefined; + } } const results = await scanOpenRouterModels({ apiKey: storedKey ?? undefined, @@ -191,8 +195,21 @@ export async function modelsScanCommand( providerFilter: opts.provider, timeoutMs: timeout, concurrency, + probe, }); + if (!probe) { + if (!opts.json) { + runtime.log( + `Found ${results.length} OpenRouter free models (metadata only; pass --probe to test tools/images).`, + ); + printScanTable(sortScanResults(results), runtime); + } else { + runtime.log(JSON.stringify(results, null, 2)); + } + return; + } + const toolOk = results.filter((entry) => entry.tool.ok); if (toolOk.length === 0) { throw new Error("No tool-capable OpenRouter free models found.");