From 1bf44bf30cc0a8f23cb7a095c7a3ae80117a4e8d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 6 Jan 2026 20:07:04 +0000 Subject: [PATCH] feat(models): show auth overview --- CHANGELOG.md | 1 + README.md | 5 + docs/model-failover.md | 22 ++- docs/models.md | 2 + src/cli/models-cli.ts | 12 +- src/commands/models/list.status.test.ts | 155 +++++++++++++++ src/commands/models/list.ts | 247 ++++++++++++++++++++++++ 7 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 src/commands/models/list.status.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aae0dc3c..2e6bcac62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ - Bash tool: inherit gateway PATH so Nix-provided tools resolve during commands. Thanks @joshp123 for PR #202. - Delivery chunking: keep Markdown fenced code blocks valid when splitting long replies (close + reopen fences). - Auth: prefer OAuth profiles over API keys during round-robin selection (prevents OAuth “lost after one message” when both are configured). +- Models: extend `clawdbot models` status output with a masked auth overview (profiles, env sources, and OAuth counts). ### Maintenance - Agent: add `skipBootstrap` config option. Thanks @onutc for PR #292. diff --git a/README.md b/README.md index 165056ef5..70c595068 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ New install? Start here: https://docs.clawd.bot/getting-started Model note: while any model is supported, I strongly recommend **Anthropic Pro/Max (100/200) + Opus 4.5** for long‑context strength and better prompt‑injection resistance. See [Onboarding](https://docs.clawd.bot/onboarding). +## Models (selection + auth) + +- Models config + CLI: https://docs.clawd.bot/models +- Auth profile rotation (OAuth vs API keys) + fallbacks: https://docs.clawd.bot/model-failover + ## Recommended setup (from source) Do **not** download prebuilt binaries. Run from source. diff --git a/docs/model-failover.md b/docs/model-failover.md index 5ce684dff..39617981f 100644 --- a/docs/model-failover.md +++ b/docs/model-failover.md @@ -13,6 +13,18 @@ Clawdbot handles failures in two stages: This doc explains the runtime rules and the data that backs them. +## Auth storage (keys + OAuth) + +Clawdbot uses **auth profiles** for both API keys and OAuth tokens. + +- Secrets live in `~/.clawdbot/agent/auth-profiles.json` (default agent; multi-agent stores under `~/.clawdbot/agents//agent/auth-profiles.json`). +- Config `auth.profiles` / `auth.order` are **metadata + routing only** (no secrets). +- Legacy import-only OAuth file: `~/.clawdbot/credentials/oauth.json` (imported into `auth-profiles.json` on first use). + +Credential types: +- `type: "api_key"` → `{ provider, key }` +- `type: "oauth"` → `{ provider, access, refresh, expires, email? }` (+ `projectId`/`enterpriseUrl` for some providers) + ## Profile IDs OAuth logins create distinct profiles so multiple accounts can coexist. @@ -30,10 +42,16 @@ When a provider has multiple profiles, Clawdbot chooses an order like this: 3) **Stored profiles**: entries in `auth-profiles.json` for the provider. If no explicit order is configured, Clawdbot uses a round‑robin order: -- **Primary key:** `usageStats.lastUsed` (oldest first). -- **Secondary key:** profile type (OAuth before API keys). +- **Primary key:** profile type (**OAuth before API keys**). +- **Secondary key:** `usageStats.lastUsed` (oldest first, within each type). - **Cooldown profiles** are moved to the end, ordered by soonest cooldown expiry. +### Why OAuth can “look lost” + +If you have both an OAuth profile and an API key profile for the same provider, round‑robin can switch between them across messages unless pinned. To force a single profile: +- Pin with `auth.order[provider] = ["provider:profileId"]`, or +- Use a per-session override via `/model …` with a profile override (when supported by your UI/chat surface). + ## Cooldowns When a profile fails due to auth/rate‑limit errors (or a timeout that looks diff --git a/docs/models.md b/docs/models.md index 069304966..0b8d6206a 100644 --- a/docs/models.md +++ b/docs/models.md @@ -7,6 +7,8 @@ read_when: --- # Models CLI plan +See [`docs/model-failover.md`](https://docs.clawd.bot/model-failover) for how auth profiles rotate (OAuth vs API keys), cooldowns, and how that interacts with model fallbacks. + Goal: give clear model visibility + control (configured vs available), plus scan tooling that prefers tool-call + image-capable models and maintains ordered fallbacks. diff --git a/src/cli/models-cli.ts b/src/cli/models-cli.ts index cc6adf9c3..9ee9e64f4 100644 --- a/src/cli/models-cli.ts +++ b/src/cli/models-cli.ts @@ -23,7 +23,13 @@ import { defaultRuntime } from "../runtime.js"; export function registerModelsCli(program: Command) { const models = program .command("models") - .description("Model discovery, scanning, and configuration"); + .description("Model discovery, scanning, and configuration") + .option("--json", "Output JSON (alias for `models status --json`)", false) + .option( + "--plain", + "Plain output (alias for `models status --plain`)", + false, + ); models .command("list") @@ -264,9 +270,9 @@ export function registerModelsCli(program: Command) { } }); - models.action(async () => { + models.action(async (opts) => { try { - await modelsStatusCommand({}, defaultRuntime); + await modelsStatusCommand(opts ?? {}, defaultRuntime); } catch (err) { defaultRuntime.error(String(err)); defaultRuntime.exit(1); diff --git a/src/commands/models/list.status.test.ts b/src/commands/models/list.status.test.ts new file mode 100644 index 000000000..b4310fad1 --- /dev/null +++ b/src/commands/models/list.status.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const store = { + version: 1, + profiles: { + "anthropic:default": { + type: "oauth", + provider: "anthropic", + access: "sk-ant-oat01-ACCESS-TOKEN-1234567890", + refresh: "sk-ant-ort01-REFRESH-TOKEN-1234567890", + expires: Date.now() + 60_000, + email: "peter@example.com", + }, + "anthropic:work": { + type: "api_key", + provider: "anthropic", + key: "sk-ant-api-0123456789abcdefghijklmnopqrstuvwxyz", + }, + "openai-codex:default": { + type: "oauth", + provider: "openai-codex", + access: "eyJhbGciOi-ACCESS", + refresh: "oai-refresh-1234567890", + expires: Date.now() + 60_000, + }, + }, + }; + + return { + store, + resolveClawdbotAgentDir: vi.fn().mockReturnValue("/tmp/clawdbot-agent"), + ensureAuthProfileStore: vi.fn().mockReturnValue(store), + listProfilesForProvider: vi.fn((s: typeof store, provider: string) => { + return Object.entries(s.profiles) + .filter(([, cred]) => cred.provider === provider) + .map(([id]) => id); + }), + resolveAuthProfileDisplayLabel: vi.fn( + ({ profileId }: { profileId: string }) => profileId, + ), + resolveAuthStorePathForDisplay: vi + .fn() + .mockReturnValue("/tmp/clawdbot-agent/auth-profiles.json"), + resolveEnvApiKey: vi.fn((provider: string) => { + if (provider === "openai") { + return { + apiKey: "sk-openai-0123456789abcdefghijklmnopqrstuvwxyz", + source: "shell env: OPENAI_API_KEY", + }; + } + if (provider === "anthropic") { + return { + apiKey: "sk-ant-oat01-ACCESS-TOKEN-1234567890", + source: "env: ANTHROPIC_OAUTH_TOKEN", + }; + } + return null; + }), + getCustomProviderApiKey: vi.fn().mockReturnValue(undefined), + getShellEnvAppliedKeys: vi + .fn() + .mockReturnValue(["OPENAI_API_KEY", "ANTHROPIC_OAUTH_TOKEN"]), + shouldEnableShellEnvFallback: vi.fn().mockReturnValue(true), + loadConfig: vi.fn().mockReturnValue({ + agent: { + model: { primary: "anthropic/claude-opus-4-5", fallbacks: [] }, + models: { "anthropic/claude-opus-4-5": { alias: "Opus" } }, + }, + models: { providers: {} }, + env: { shellEnv: { enabled: true } }, + }), + }; +}); + +vi.mock("../../agents/agent-paths.js", () => ({ + resolveClawdbotAgentDir: mocks.resolveClawdbotAgentDir, +})); + +vi.mock("../../agents/auth-profiles.js", () => ({ + ensureAuthProfileStore: mocks.ensureAuthProfileStore, + listProfilesForProvider: mocks.listProfilesForProvider, + resolveAuthProfileDisplayLabel: mocks.resolveAuthProfileDisplayLabel, + resolveAuthStorePathForDisplay: mocks.resolveAuthStorePathForDisplay, +})); + +vi.mock("../../agents/model-auth.js", () => ({ + resolveEnvApiKey: mocks.resolveEnvApiKey, + getCustomProviderApiKey: mocks.getCustomProviderApiKey, +})); + +vi.mock("../../infra/shell-env.js", () => ({ + getShellEnvAppliedKeys: mocks.getShellEnvAppliedKeys, + shouldEnableShellEnvFallback: mocks.shouldEnableShellEnvFallback, +})); + +vi.mock("../../config/config.js", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + loadConfig: mocks.loadConfig, + }; +}); + +import { modelsStatusCommand } from "./list.js"; + +const runtime = { + log: vi.fn(), + error: vi.fn(), + exit: vi.fn(), +}; + +describe("modelsStatusCommand auth overview", () => { + it("includes masked auth sources in JSON output", async () => { + await modelsStatusCommand({ json: true }, runtime as never); + const payload = JSON.parse( + String((runtime.log as vi.Mock).mock.calls[0][0]), + ); + + expect(payload.defaultModel).toBe("anthropic/claude-opus-4-5"); + expect(payload.auth.storePath).toBe( + "/tmp/clawdbot-agent/auth-profiles.json", + ); + expect(payload.auth.shellEnvFallback.enabled).toBe(true); + expect(payload.auth.shellEnvFallback.appliedKeys).toContain( + "OPENAI_API_KEY", + ); + + const providers = payload.auth.providers as Array<{ + provider: string; + profiles: { labels: string[] }; + env?: { value: string; source: string }; + }>; + const anthropic = providers.find((p) => p.provider === "anthropic"); + expect(anthropic).toBeTruthy(); + expect(anthropic?.profiles.labels.join(" ")).toContain("OAuth"); + expect(anthropic?.profiles.labels.join(" ")).toContain("..."); + + const openai = providers.find((p) => p.provider === "openai"); + expect(openai?.env?.source).toContain("OPENAI_API_KEY"); + expect(openai?.env?.value).toContain("..."); + + expect( + (payload.auth.providersWithOAuth as string[]).some((e) => + e.startsWith("anthropic"), + ), + ).toBe(true); + expect( + (payload.auth.providersWithOAuth as string[]).some((e) => + e.startsWith("openai-codex"), + ), + ).toBe(true); + }); +}); diff --git a/src/commands/models/list.ts b/src/commands/models/list.ts index 7a8fb7858..56baa5e62 100644 --- a/src/commands/models/list.ts +++ b/src/commands/models/list.ts @@ -1,3 +1,5 @@ +import path from "node:path"; + import type { Api, Model } from "@mariozechner/pi-ai"; import { discoverAuthStorage, @@ -10,6 +12,8 @@ import { type AuthProfileStore, ensureAuthProfileStore, listProfilesForProvider, + resolveAuthProfileDisplayLabel, + resolveAuthStorePathForDisplay, } from "../../agents/auth-profiles.js"; import { getCustomProviderApiKey, @@ -28,7 +32,12 @@ import { loadConfig, } from "../../config/config.js"; import { info } from "../../globals.js"; +import { + getShellEnvAppliedKeys, + shouldEnableShellEnvFallback, +} from "../../infra/shell-env.js"; import type { RuntimeEnv } from "../../runtime.js"; +import { shortenHomePath } from "../../utils.js"; import { DEFAULT_MODEL, DEFAULT_PROVIDER, @@ -56,6 +65,13 @@ const truncate = (value: string, max: number) => { return `${value.slice(0, max - 3)}...`; }; +const maskApiKey = (value: string): string => { + const trimmed = value.trim(); + if (!trimmed) return "missing"; + if (trimmed.length <= 16) return trimmed; + return `${trimmed.slice(0, 8)}...${trimmed.slice(-8)}`; +}; + type ConfiguredEntry = { key: string; ref: { provider: string; model: string }; @@ -101,6 +117,109 @@ const hasAuthForProvider = ( return false; }; +type ProviderAuthOverview = { + provider: string; + effective: { + kind: "profiles" | "env" | "models.json" | "missing"; + detail: string; + }; + profiles: { + count: number; + oauth: number; + apiKey: number; + labels: string[]; + }; + env?: { value: string; source: string }; + modelsJson?: { value: string; source: string }; +}; + +function resolveProviderAuthOverview(params: { + provider: string; + cfg: ClawdbotConfig; + store: AuthProfileStore; + modelsPath: string; +}): ProviderAuthOverview { + const { provider, cfg, store } = params; + const profiles = listProfilesForProvider(store, provider); + const labels = profiles.map((profileId) => { + const profile = store.profiles[profileId]; + if (!profile) return `${profileId}=missing`; + if (profile.type === "api_key") { + return `${profileId}=${maskApiKey(profile.key)}`; + } + const display = resolveAuthProfileDisplayLabel({ cfg, store, profileId }); + const suffix = + display === profileId + ? "" + : display.startsWith(profileId) + ? display.slice(profileId.length).trim() + : `(${display})`; + return `${profileId}=OAuth${suffix ? ` ${suffix}` : ""}`; + }); + const oauthCount = profiles.filter( + (id) => store.profiles[id]?.type === "oauth", + ).length; + const apiKeyCount = profiles.filter( + (id) => store.profiles[id]?.type === "api_key", + ).length; + + const envKey = resolveEnvApiKey(provider); + const customKey = getCustomProviderApiKey(cfg, provider); + + const effective: ProviderAuthOverview["effective"] = (() => { + if (profiles.length > 0) { + return { + kind: "profiles", + detail: shortenHomePath(resolveAuthStorePathForDisplay()), + }; + } + if (envKey) { + const isOAuthEnv = + envKey.source.includes("OAUTH_TOKEN") || + envKey.source.toLowerCase().includes("oauth"); + return { + kind: "env", + detail: isOAuthEnv ? "OAuth (env)" : maskApiKey(envKey.apiKey), + }; + } + if (customKey) { + return { kind: "models.json", detail: maskApiKey(customKey) }; + } + return { kind: "missing", detail: "missing" }; + })(); + + return { + provider, + effective, + profiles: { + count: profiles.length, + oauth: oauthCount, + apiKey: apiKeyCount, + labels, + }, + ...(envKey + ? { + env: { + value: + envKey.source.includes("OAUTH_TOKEN") || + envKey.source.toLowerCase().includes("oauth") + ? "OAuth (env)" + : maskApiKey(envKey.apiKey), + source: envKey.source, + }, + } + : {}), + ...(customKey + ? { + modelsJson: { + value: maskApiKey(customKey), + source: `models.json: ${shortenHomePath(params.modelsPath)}`, + }, + } + : {}), + }; +} + const resolveConfiguredEntries = (cfg: ClawdbotConfig) => { const resolvedDefault = resolveConfiguredModelRef({ cfg, @@ -462,11 +581,97 @@ export async function modelsStatusCommand( }, {}); const allowed = Object.keys(cfg.agent?.models ?? {}); + const agentDir = resolveClawdbotAgentDir(); + const store = ensureAuthProfileStore(); + const modelsPath = path.join(agentDir, "models.json"); + + const providersFromStore = new Set( + Object.values(store.profiles) + .map((profile) => profile.provider) + .filter((p): p is string => Boolean(p)), + ); + const providersFromConfig = new Set( + Object.keys(cfg.models?.providers ?? {}) + .map((p) => p.trim()) + .filter(Boolean), + ); + const providersFromModels = new Set(); + for (const raw of [ + defaultLabel, + ...fallbacks, + imageModel, + ...imageFallbacks, + ...allowed, + ]) { + const parsed = parseModelRef(String(raw ?? ""), DEFAULT_PROVIDER); + if (parsed?.provider) providersFromModels.add(parsed.provider); + } + + const providersFromEnv = new Set(); + // Keep in sync with resolveEnvApiKey() mappings (we want visibility even when + // a provider isn't currently selected in config/models). + const envProbeProviders = [ + "anthropic", + "github-copilot", + "google-vertex", + "openai", + "google", + "groq", + "cerebras", + "xai", + "openrouter", + "zai", + "mistral", + ]; + for (const provider of envProbeProviders) { + if (resolveEnvApiKey(provider)) providersFromEnv.add(provider); + } + + const providers = Array.from( + new Set([ + ...providersFromStore, + ...providersFromConfig, + ...providersFromModels, + ...providersFromEnv, + ]), + ) + .map((p) => p.trim()) + .filter(Boolean) + .sort((a, b) => a.localeCompare(b)); + + const applied = getShellEnvAppliedKeys(); + const shellFallbackEnabled = + shouldEnableShellEnvFallback(process.env) || + cfg.env?.shellEnv?.enabled === true; + + const providerAuth = providers + .map((provider) => + resolveProviderAuthOverview({ provider, cfg, store, modelsPath }), + ) + .filter((entry) => { + const hasAny = + entry.profiles.count > 0 || + Boolean(entry.env) || + Boolean(entry.modelsJson); + return hasAny; + }); + + const providersWithOauth = providerAuth + .filter( + (entry) => entry.profiles.oauth > 0 || entry.env?.value === "OAuth (env)", + ) + .map((entry) => { + const count = + entry.profiles.oauth || (entry.env?.value === "OAuth (env)" ? 1 : 0); + return `${entry.provider} (${count})`; + }); + if (opts.json) { runtime.log( JSON.stringify( { configPath: CONFIG_PATH_CLAWDBOT, + agentDir, defaultModel: defaultLabel, resolvedDefault: `${resolved.provider}/${resolved.model}`, fallbacks, @@ -474,6 +679,15 @@ export async function modelsStatusCommand( imageFallbacks, aliases, allowed, + auth: { + storePath: resolveAuthStorePathForDisplay(), + shellEnvFallback: { + enabled: shellFallbackEnabled, + appliedKeys: applied, + }, + providersWithOAuth: providersWithOauth, + providers: providerAuth, + }, }, null, 2, @@ -488,6 +702,7 @@ export async function modelsStatusCommand( } runtime.log(info(`Config: ${CONFIG_PATH_CLAWDBOT}`)); + runtime.log(info(`Agent dir: ${shortenHomePath(agentDir)}`)); runtime.log(`Default: ${defaultLabel}`); runtime.log( `Fallbacks (${fallbacks.length || 0}): ${fallbacks.join(", ") || "-"}`, @@ -512,4 +727,36 @@ export async function modelsStatusCommand( allowed.length ? allowed.join(", ") : "all" }`, ); + + runtime.log(""); + runtime.log(info("Auth overview")); + runtime.log( + `Auth store: ${shortenHomePath(resolveAuthStorePathForDisplay())}`, + ); + runtime.log( + `Shell env fallback: ${shellFallbackEnabled ? "on" : "off"}${ + applied.length ? ` (applied: ${applied.join(", ")})` : "" + }`, + ); + runtime.log( + `Providers with OAuth (${providersWithOauth.length || 0}): ${ + providersWithOauth.length ? providersWithOauth.join(", ") : "-" + }`, + ); + + for (const entry of providerAuth) { + const bits: string[] = []; + bits.push(`effective=${entry.effective.kind}:${entry.effective.detail}`); + if (entry.profiles.count > 0) { + bits.push( + `profiles=${entry.profiles.count} (oauth=${entry.profiles.oauth}, api_key=${entry.profiles.apiKey})`, + ); + if (entry.profiles.labels.length > 0) { + bits.push(entry.profiles.labels.join(", ")); + } + } + if (entry.env) bits.push(`env=${entry.env.value} (${entry.env.source})`); + if (entry.modelsJson) bits.push(`models.json=${entry.modelsJson.value}`); + runtime.log(`${entry.provider}: ${bits.join(" | ")}`); + } }