From 40181afdedb04ce05f9d28d0a34440e810e8c07e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 23 Jan 2026 19:25:58 +0000 Subject: [PATCH] feat: add models status auth probes --- CHANGELOG.md | 1 + docs/cli/index.md | 7 + docs/cli/models.md | 14 + src/cli/models-cli.ts | 29 +- src/commands/models.list.test.ts | 1 + src/commands/models/list.probe.ts | 414 ++++++++++++++++++ src/commands/models/list.status-command.ts | 256 ++++++++--- ...patterns-match-without-botusername.test.ts | 13 +- ...topic-skill-filters-system-prompts.test.ts | 13 +- ...-all-group-messages-grouppolicy-is.test.ts | 13 +- ...e-callback-query-updates-by-update.test.ts | 13 +- ...gram-bot.installs-grammy-throttler.test.ts | 14 +- ...lowfrom-entries-case-insensitively.test.ts | 13 +- ...-case-insensitively-grouppolicy-is.test.ts | 13 +- ...-dms-by-telegram-accountid-binding.test.ts | 13 +- ...ies-without-native-reply-threading.test.ts | 13 +- src/telegram/bot.test.ts | 20 +- 17 files changed, 754 insertions(+), 106 deletions(-) create mode 100644 src/commands/models/list.probe.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4106f7827..3612e9686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.clawd.bot ### Changes - CLI: restart the gateway by default after `clawdbot update`; add `--no-restart` to skip it. +- CLI: add live auth probes to `clawdbot models status` for per-profile verification. - Markdown: add per-channel table conversion (bullets for Signal/WhatsApp, code blocks elsewhere). (#1495) Thanks @odysseus0. ### Fixes diff --git a/docs/cli/index.md b/docs/cli/index.md index 46f6d173e..fcc013fdc 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -700,8 +700,15 @@ Options: - `--json` - `--plain` - `--check` (exit 1=expired/missing, 2=expiring) +- `--probe` (live probe of configured auth profiles) +- `--probe-provider ` +- `--probe-profile ` (repeat or comma-separated) +- `--probe-timeout ` +- `--probe-concurrency ` +- `--probe-max-tokens ` Always includes the auth overview and OAuth expiry status for profiles in the auth store. +`--probe` runs live requests (may consume tokens and trigger rate limits). ### `models set ` Set `agents.defaults.model.primary`. diff --git a/docs/cli/models.md b/docs/cli/models.md index f394a44f9..ba4600ce4 100644 --- a/docs/cli/models.md +++ b/docs/cli/models.md @@ -25,12 +25,26 @@ clawdbot models scan `clawdbot models status` shows the resolved default/fallbacks plus an auth overview. When provider usage snapshots are available, the OAuth/token status section includes provider usage headers. +Add `--probe` to run live auth probes against each configured provider profile. +Probes are real requests (may consume tokens and trigger rate limits). Notes: - `models set ` accepts `provider/model` or an alias. - Model refs are parsed by splitting on the **first** `/`. If the model ID includes `/` (OpenRouter-style), include the provider prefix (example: `openrouter/moonshotai/kimi-k2`). - If you omit the provider, Clawdbot treats the input as an alias or a model for the **default provider** (only works when there is no `/` in the model ID). +### `models status` +Options: +- `--json` +- `--plain` +- `--check` (exit 1=expired/missing, 2=expiring) +- `--probe` (live probe of configured auth profiles) +- `--probe-provider ` (probe one provider) +- `--probe-profile ` (repeat or comma-separated profile ids) +- `--probe-timeout ` +- `--probe-concurrency ` +- `--probe-max-tokens ` + ## Aliases + fallbacks ```bash diff --git a/src/cli/models-cli.ts b/src/cli/models-cli.ts index a2674d94a..20a476f81 100644 --- a/src/cli/models-cli.ts +++ b/src/cli/models-cli.ts @@ -71,9 +71,36 @@ export function registerModelsCli(program: Command) { "Exit non-zero if auth is expiring/expired (1=expired/missing, 2=expiring)", false, ) + .option("--probe", "Probe configured provider auth (live)", false) + .option("--probe-provider ", "Only probe a single provider") + .option( + "--probe-profile ", + "Only probe specific auth profile ids (repeat or comma-separated)", + (value, previous) => { + const next = Array.isArray(previous) ? previous : previous ? [previous] : []; + next.push(value); + return next; + }, + ) + .option("--probe-timeout ", "Per-probe timeout in ms") + .option("--probe-concurrency ", "Concurrent probes") + .option("--probe-max-tokens ", "Probe max tokens (best-effort)") .action(async (opts) => { await runModelsCommand(async () => { - await modelsStatusCommand(opts, defaultRuntime); + await modelsStatusCommand( + { + json: Boolean(opts.json), + plain: Boolean(opts.plain), + check: Boolean(opts.check), + probe: Boolean(opts.probe), + probeProvider: opts.probeProvider as string | undefined, + probeProfile: opts.probeProfile as string | string[] | undefined, + probeTimeout: opts.probeTimeout as string | undefined, + probeConcurrency: opts.probeConcurrency as string | undefined, + probeMaxTokens: opts.probeMaxTokens as string | undefined, + }, + defaultRuntime, + ); }); }); diff --git a/src/commands/models.list.test.ts b/src/commands/models.list.test.ts index 47ebfe2f5..850f27246 100644 --- a/src/commands/models.list.test.ts +++ b/src/commands/models.list.test.ts @@ -17,6 +17,7 @@ const discoverModels = vi.fn(); vi.mock("../config/config.js", () => ({ CONFIG_PATH_CLAWDBOT: "/tmp/clawdbot.json", + STATE_DIR_CLAWDBOT: "/tmp/clawdbot-state", loadConfig, })); diff --git a/src/commands/models/list.probe.ts b/src/commands/models/list.probe.ts new file mode 100644 index 000000000..fbd172b57 --- /dev/null +++ b/src/commands/models/list.probe.ts @@ -0,0 +1,414 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; + +import { resolveClawdbotAgentDir } from "../../agents/agent-paths.js"; +import { + ensureAuthProfileStore, + listProfilesForProvider, + resolveAuthProfileDisplayLabel, +} from "../../agents/auth-profiles.js"; +import { runEmbeddedPiAgent } from "../../agents/pi-embedded.js"; +import { describeFailoverError } from "../../agents/failover-error.js"; +import { loadModelCatalog } from "../../agents/model-catalog.js"; +import { getCustomProviderApiKey, resolveEnvApiKey } from "../../agents/model-auth.js"; +import { normalizeProviderId, parseModelRef } from "../../agents/model-selection.js"; +import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../../agents/agent-scope.js"; +import { resolveDefaultAgentWorkspaceDir } from "../../agents/workspace.js"; +import type { ClawdbotConfig } from "../../config/config.js"; +import { + resolveSessionTranscriptPath, + resolveSessionTranscriptsDirForAgent, +} from "../../config/sessions/paths.js"; +import { redactSecrets } from "../status-all/format.js"; +import { DEFAULT_PROVIDER, formatMs } from "./shared.js"; + +const PROBE_PROMPT = "Reply with OK. Do not use tools."; + +export type AuthProbeStatus = + | "ok" + | "auth" + | "rate_limit" + | "billing" + | "timeout" + | "format" + | "unknown" + | "no_model"; + +export type AuthProbeResult = { + provider: string; + model?: string; + profileId?: string; + label: string; + source: "profile" | "env" | "models.json"; + mode?: string; + status: AuthProbeStatus; + error?: string; + latencyMs?: number; +}; + +type AuthProbeTarget = { + provider: string; + model?: { provider: string; model: string } | null; + profileId?: string; + label: string; + source: "profile" | "env" | "models.json"; + mode?: string; +}; + +export type AuthProbeSummary = { + startedAt: number; + finishedAt: number; + durationMs: number; + totalTargets: number; + options: { + provider?: string; + profileIds?: string[]; + timeoutMs: number; + concurrency: number; + maxTokens: number; + }; + results: AuthProbeResult[]; +}; + +export type AuthProbeOptions = { + provider?: string; + profileIds?: string[]; + timeoutMs: number; + concurrency: number; + maxTokens: number; +}; + +const toStatus = (reason?: string | null): AuthProbeStatus => { + if (!reason) return "unknown"; + if (reason === "auth") return "auth"; + if (reason === "rate_limit") return "rate_limit"; + if (reason === "billing") return "billing"; + if (reason === "timeout") return "timeout"; + if (reason === "format") return "format"; + return "unknown"; +}; + +function buildCandidateMap(modelCandidates: string[]): Map { + const map = new Map(); + for (const raw of modelCandidates) { + const parsed = parseModelRef(String(raw ?? ""), DEFAULT_PROVIDER); + if (!parsed) continue; + const list = map.get(parsed.provider) ?? []; + if (!list.includes(parsed.model)) list.push(parsed.model); + map.set(parsed.provider, list); + } + return map; +} + +function selectProbeModel(params: { + provider: string; + candidates: Map; + catalog: Array<{ provider: string; id: string }>; +}): { provider: string; model: string } | null { + const { provider, candidates, catalog } = params; + const direct = candidates.get(provider); + if (direct && direct.length > 0) { + return { provider, model: direct[0] }; + } + const fromCatalog = catalog.find((entry) => entry.provider === provider); + if (fromCatalog) return { provider: fromCatalog.provider, model: fromCatalog.id }; + return null; +} + +function buildProbeTargets(params: { + cfg: ClawdbotConfig; + providers: string[]; + modelCandidates: string[]; + options: AuthProbeOptions; +}): Promise<{ targets: AuthProbeTarget[]; results: AuthProbeResult[] }> { + const { cfg, providers, modelCandidates, options } = params; + const store = ensureAuthProfileStore(); + const providerFilter = options.provider?.trim(); + const providerFilterKey = providerFilter ? normalizeProviderId(providerFilter) : null; + const profileFilter = new Set((options.profileIds ?? []).map((id) => id.trim()).filter(Boolean)); + + return loadModelCatalog({ config: cfg }).then((catalog) => { + const candidates = buildCandidateMap(modelCandidates); + const targets: AuthProbeTarget[] = []; + const results: AuthProbeResult[] = []; + + for (const provider of providers) { + const providerKey = normalizeProviderId(provider); + if (providerFilterKey && providerKey !== providerFilterKey) continue; + + const model = selectProbeModel({ + provider: providerKey, + candidates, + catalog, + }); + + const profileIds = listProfilesForProvider(store, providerKey); + const filteredProfiles = profileFilter.size + ? profileIds.filter((id) => profileFilter.has(id)) + : profileIds; + + if (filteredProfiles.length > 0) { + for (const profileId of filteredProfiles) { + const profile = store.profiles[profileId]; + const mode = profile?.type; + const label = resolveAuthProfileDisplayLabel({ cfg, store, profileId }); + if (!model) { + results.push({ + provider: providerKey, + model: undefined, + profileId, + label, + source: "profile", + mode, + status: "no_model", + error: "No model available for probe", + }); + continue; + } + targets.push({ + provider: providerKey, + model, + profileId, + label, + source: "profile", + mode, + }); + } + continue; + } + + if (profileFilter.size > 0) continue; + + const envKey = resolveEnvApiKey(providerKey); + const customKey = getCustomProviderApiKey(cfg, providerKey); + if (!envKey && !customKey) continue; + + const label = envKey ? "env" : "models.json"; + const source = envKey ? "env" : "models.json"; + const mode = envKey?.source.includes("OAUTH_TOKEN") ? "oauth" : "api_key"; + + if (!model) { + results.push({ + provider: providerKey, + model: undefined, + label, + source, + mode, + status: "no_model", + error: "No model available for probe", + }); + continue; + } + + targets.push({ + provider: providerKey, + model, + label, + source, + mode, + }); + } + + return { targets, results }; + }); +} + +async function probeTarget(params: { + cfg: ClawdbotConfig; + agentId: string; + agentDir: string; + workspaceDir: string; + sessionDir: string; + target: AuthProbeTarget; + timeoutMs: number; + maxTokens: number; +}): Promise { + const { cfg, agentId, agentDir, workspaceDir, sessionDir, target, timeoutMs, maxTokens } = params; + if (!target.model) { + return { + provider: target.provider, + model: undefined, + profileId: target.profileId, + label: target.label, + source: target.source, + mode: target.mode, + status: "no_model", + error: "No model available for probe", + }; + } + + const sessionId = `probe-${target.provider}-${crypto.randomUUID()}`; + const sessionFile = resolveSessionTranscriptPath(sessionId, agentId); + await fs.mkdir(sessionDir, { recursive: true }); + + const start = Date.now(); + try { + await runEmbeddedPiAgent({ + sessionId, + sessionFile, + workspaceDir, + agentDir, + config: cfg, + prompt: PROBE_PROMPT, + provider: target.model.provider, + model: target.model.model, + authProfileId: target.profileId, + authProfileIdSource: target.profileId ? "user" : undefined, + timeoutMs, + runId: `probe-${crypto.randomUUID()}`, + lane: `auth-probe:${target.provider}:${target.profileId ?? target.source}`, + thinkLevel: "off", + reasoningLevel: "off", + verboseLevel: "off", + streamParams: { maxTokens }, + }); + return { + provider: target.provider, + model: `${target.model.provider}/${target.model.model}`, + profileId: target.profileId, + label: target.label, + source: target.source, + mode: target.mode, + status: "ok", + latencyMs: Date.now() - start, + }; + } catch (err) { + const described = describeFailoverError(err); + return { + provider: target.provider, + model: `${target.model.provider}/${target.model.model}`, + profileId: target.profileId, + label: target.label, + source: target.source, + mode: target.mode, + status: toStatus(described.reason), + error: redactSecrets(described.message), + latencyMs: Date.now() - start, + }; + } +} + +async function runTargetsWithConcurrency(params: { + cfg: ClawdbotConfig; + targets: AuthProbeTarget[]; + timeoutMs: number; + maxTokens: number; + concurrency: number; + onProgress?: (update: { completed: number; total: number; label?: string }) => void; +}): Promise { + const { cfg, targets, timeoutMs, maxTokens, onProgress } = params; + const concurrency = Math.max(1, Math.min(targets.length || 1, params.concurrency)); + + const agentId = resolveDefaultAgentId(cfg); + const agentDir = resolveClawdbotAgentDir(); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId) ?? resolveDefaultAgentWorkspaceDir(); + const sessionDir = resolveSessionTranscriptsDirForAgent(agentId); + + await fs.mkdir(workspaceDir, { recursive: true }); + + let completed = 0; + const results: Array = Array.from({ length: targets.length }); + let cursor = 0; + + const worker = async () => { + while (true) { + const index = cursor; + cursor += 1; + if (index >= targets.length) return; + const target = targets[index]; + onProgress?.({ + completed, + total: targets.length, + label: `Probing ${target.provider}${target.profileId ? ` (${target.label})` : ""}`, + }); + const result = await probeTarget({ + cfg, + agentId, + agentDir, + workspaceDir, + sessionDir, + target, + timeoutMs, + maxTokens, + }); + results[index] = result; + completed += 1; + onProgress?.({ completed, total: targets.length }); + } + }; + + await Promise.all(Array.from({ length: concurrency }, () => worker())); + + return results.filter((entry): entry is AuthProbeResult => Boolean(entry)); +} + +export async function runAuthProbes(params: { + cfg: ClawdbotConfig; + providers: string[]; + modelCandidates: string[]; + options: AuthProbeOptions; + onProgress?: (update: { completed: number; total: number; label?: string }) => void; +}): Promise { + const startedAt = Date.now(); + const plan = await buildProbeTargets({ + cfg: params.cfg, + providers: params.providers, + modelCandidates: params.modelCandidates, + options: params.options, + }); + + const totalTargets = plan.targets.length; + params.onProgress?.({ completed: 0, total: totalTargets }); + + const results = totalTargets + ? await runTargetsWithConcurrency({ + cfg: params.cfg, + targets: plan.targets, + timeoutMs: params.options.timeoutMs, + maxTokens: params.options.maxTokens, + concurrency: params.options.concurrency, + onProgress: params.onProgress, + }) + : []; + + const finishedAt = Date.now(); + + return { + startedAt, + finishedAt, + durationMs: finishedAt - startedAt, + totalTargets, + options: params.options, + results: [...plan.results, ...results], + }; +} + +export function formatProbeLatency(latencyMs?: number | null) { + if (!latencyMs && latencyMs !== 0) return "-"; + return formatMs(latencyMs); +} + +export function groupProbeResults(results: AuthProbeResult[]): Map { + const map = new Map(); + for (const result of results) { + const list = map.get(result.provider) ?? []; + list.push(result); + map.set(result.provider, list); + } + return map; +} + +export function sortProbeResults(results: AuthProbeResult[]): AuthProbeResult[] { + return results.slice().sort((a, b) => { + const provider = a.provider.localeCompare(b.provider); + if (provider !== 0) return provider; + const aLabel = a.label || a.profileId || ""; + const bLabel = b.label || b.profileId || ""; + return aLabel.localeCompare(bLabel); + }); +} + +export function describeProbeSummary(summary: AuthProbeSummary): string { + if (summary.totalTargets === 0) return "No probe targets."; + return `Probed ${summary.totalTargets} target${summary.totalTargets === 1 ? "" : "s"} in ${formatMs(summary.durationMs)}`; +} diff --git a/src/commands/models/list.status-command.ts b/src/commands/models/list.status-command.ts index 0bd8f16e9..41c126460 100644 --- a/src/commands/models/list.status-command.ts +++ b/src/commands/models/list.status-command.ts @@ -11,9 +11,15 @@ import { resolveProfileUnusableUntilForDisplay, } from "../../agents/auth-profiles.js"; import { resolveEnvApiKey } from "../../agents/model-auth.js"; -import { parseModelRef, resolveConfiguredModelRef } from "../../agents/model-selection.js"; +import { + buildModelAliasIndex, + parseModelRef, + resolveConfiguredModelRef, + resolveModelRefFromString, +} from "../../agents/model-selection.js"; import { CONFIG_PATH_CLAWDBOT, loadConfig } from "../../config/config.js"; import { getShellEnvAppliedKeys, shouldEnableShellEnvFallback } from "../../infra/shell-env.js"; +import { withProgressTotals } from "../../cli/progress.js"; import { formatUsageWindowSummary, loadProviderUsageSummary, @@ -26,13 +32,34 @@ import { formatCliCommand } from "../../cli/command-format.js"; import { shortenHomePath } from "../../utils.js"; import { resolveProviderAuthOverview } from "./list.auth-overview.js"; import { isRich } from "./list.format.js"; +import { + describeProbeSummary, + formatProbeLatency, + groupProbeResults, + runAuthProbes, + sortProbeResults, + type AuthProbeSummary, +} from "./list.probe.js"; import { DEFAULT_MODEL, DEFAULT_PROVIDER, ensureFlagCompatibility } from "./shared.js"; export async function modelsStatusCommand( - opts: { json?: boolean; plain?: boolean; check?: boolean }, + opts: { + json?: boolean; + plain?: boolean; + check?: boolean; + probe?: boolean; + probeProvider?: string; + probeProfile?: string | string[]; + probeTimeout?: string; + probeConcurrency?: string; + probeMaxTokens?: string; + }, runtime: RuntimeEnv, ) { ensureFlagCompatibility(opts); + if (opts.plain && opts.probe) { + throw new Error("--probe cannot be used with --plain output."); + } const cfg = loadConfig(); const resolved = resolveConfiguredModelRef({ cfg, @@ -139,6 +166,69 @@ export async function modelsStatusCommand( .filter((provider) => !providerAuthMap.has(provider)) .sort((a, b) => a.localeCompare(b)); + const probeProfileIds = (() => { + if (!opts.probeProfile) return []; + const raw = Array.isArray(opts.probeProfile) ? opts.probeProfile : [opts.probeProfile]; + return raw + .flatMap((value) => String(value ?? "").split(",")) + .map((value) => value.trim()) + .filter(Boolean); + })(); + const probeTimeoutMs = opts.probeTimeout ? Number(opts.probeTimeout) : 8000; + if (!Number.isFinite(probeTimeoutMs) || probeTimeoutMs <= 0) { + throw new Error("--probe-timeout must be a positive number (ms)."); + } + const probeConcurrency = opts.probeConcurrency ? Number(opts.probeConcurrency) : 2; + if (!Number.isFinite(probeConcurrency) || probeConcurrency <= 0) { + throw new Error("--probe-concurrency must be > 0."); + } + const probeMaxTokens = opts.probeMaxTokens ? Number(opts.probeMaxTokens) : 8; + if (!Number.isFinite(probeMaxTokens) || probeMaxTokens <= 0) { + throw new Error("--probe-max-tokens must be > 0."); + } + + const aliasIndex = buildModelAliasIndex({ cfg, defaultProvider: DEFAULT_PROVIDER }); + const rawCandidates = [ + rawModel || resolvedLabel, + ...fallbacks, + imageModel, + ...imageFallbacks, + ...allowed, + ].filter(Boolean); + const resolvedCandidates = rawCandidates + .map( + (raw) => + resolveModelRefFromString({ + raw: String(raw ?? ""), + defaultProvider: DEFAULT_PROVIDER, + aliasIndex, + })?.ref, + ) + .filter((ref): ref is { provider: string; model: string } => Boolean(ref)); + const modelCandidates = resolvedCandidates.map((ref) => `${ref.provider}/${ref.model}`); + + let probeSummary: AuthProbeSummary | undefined; + if (opts.probe) { + probeSummary = await withProgressTotals( + { label: "Probing auth profiles…", total: 1 }, + async (update) => { + return await runAuthProbes({ + cfg, + providers, + modelCandidates, + options: { + provider: opts.probeProvider, + profileIds: probeProfileIds, + timeoutMs: probeTimeoutMs, + concurrency: probeConcurrency, + maxTokens: probeMaxTokens, + }, + onProgress: update, + }); + }, + ); + } + const providersWithOauth = providerAuth .filter( (entry) => @@ -228,6 +318,7 @@ export async function modelsStatusCommand( profiles: authHealth.profiles, providers: authHealth.providers, }, + probes: probeSummary, }, }, null, @@ -406,72 +497,113 @@ export async function modelsStatusCommand( runtime.log(colorize(rich, theme.heading, "OAuth/token status")); if (oauthProfiles.length === 0) { runtime.log(colorize(rich, theme.muted, "- none")); - return; - } - - const usageByProvider = new Map(); - const usageProviders = Array.from( - new Set( - oauthProfiles - .map((profile) => resolveUsageProviderId(profile.provider)) - .filter((provider): provider is UsageProviderId => Boolean(provider)), - ), - ); - if (usageProviders.length > 0) { - try { - const usageSummary = await loadProviderUsageSummary({ - providers: usageProviders, - agentDir, - timeoutMs: 3500, - }); - for (const snapshot of usageSummary.providers) { - const formatted = formatUsageWindowSummary(snapshot, { - now: Date.now(), - maxWindows: 2, - includeResets: true, + } else { + const usageByProvider = new Map(); + const usageProviders = Array.from( + new Set( + oauthProfiles + .map((profile) => resolveUsageProviderId(profile.provider)) + .filter((provider): provider is UsageProviderId => Boolean(provider)), + ), + ); + if (usageProviders.length > 0) { + try { + const usageSummary = await loadProviderUsageSummary({ + providers: usageProviders, + agentDir, + timeoutMs: 3500, }); - if (formatted) { - usageByProvider.set(snapshot.provider, formatted); + for (const snapshot of usageSummary.providers) { + const formatted = formatUsageWindowSummary(snapshot, { + now: Date.now(), + maxWindows: 2, + includeResets: true, + }); + if (formatted) { + usageByProvider.set(snapshot.provider, formatted); + } } + } catch { + // ignore usage failures + } + } + + const formatStatus = (status: string) => { + if (status === "ok") return colorize(rich, theme.success, "ok"); + if (status === "static") return colorize(rich, theme.muted, "static"); + if (status === "expiring") return colorize(rich, theme.warn, "expiring"); + if (status === "missing") return colorize(rich, theme.warn, "unknown"); + return colorize(rich, theme.error, "expired"); + }; + + const profilesByProvider = new Map(); + for (const profile of oauthProfiles) { + const current = profilesByProvider.get(profile.provider); + if (current) current.push(profile); + else profilesByProvider.set(profile.provider, [profile]); + } + + for (const [provider, profiles] of profilesByProvider) { + const usageKey = resolveUsageProviderId(provider); + const usage = usageKey ? usageByProvider.get(usageKey) : undefined; + const usageSuffix = usage ? colorize(rich, theme.muted, ` usage: ${usage}`) : ""; + runtime.log(`- ${colorize(rich, theme.heading, provider)}${usageSuffix}`); + for (const profile of profiles) { + const labelText = profile.label || profile.profileId; + const label = colorize(rich, theme.accent, labelText); + const status = formatStatus(profile.status); + const expiry = + profile.status === "static" + ? "" + : profile.expiresAt + ? ` expires in ${formatRemainingShort(profile.remainingMs)}` + : " expires unknown"; + const source = + profile.source !== "store" ? colorize(rich, theme.muted, ` (${profile.source})`) : ""; + runtime.log(` - ${label} ${status}${expiry}${source}`); } - } catch { - // ignore usage failures } } - const formatStatus = (status: string) => { - if (status === "ok") return colorize(rich, theme.success, "ok"); - if (status === "static") return colorize(rich, theme.muted, "static"); - if (status === "expiring") return colorize(rich, theme.warn, "expiring"); - if (status === "missing") return colorize(rich, theme.warn, "unknown"); - return colorize(rich, theme.error, "expired"); - }; - - const profilesByProvider = new Map(); - for (const profile of oauthProfiles) { - const current = profilesByProvider.get(profile.provider); - if (current) current.push(profile); - else profilesByProvider.set(profile.provider, [profile]); - } - - for (const [provider, profiles] of profilesByProvider) { - const usageKey = resolveUsageProviderId(provider); - const usage = usageKey ? usageByProvider.get(usageKey) : undefined; - const usageSuffix = usage ? colorize(rich, theme.muted, ` usage: ${usage}`) : ""; - runtime.log(`- ${colorize(rich, theme.heading, provider)}${usageSuffix}`); - for (const profile of profiles) { - const labelText = profile.label || profile.profileId; - const label = colorize(rich, theme.accent, labelText); - const status = formatStatus(profile.status); - const expiry = - profile.status === "static" - ? "" - : profile.expiresAt - ? ` expires in ${formatRemainingShort(profile.remainingMs)}` - : " expires unknown"; - const source = - profile.source !== "store" ? colorize(rich, theme.muted, ` (${profile.source})`) : ""; - runtime.log(` - ${label} ${status}${expiry}${source}`); + if (probeSummary) { + runtime.log(""); + runtime.log(colorize(rich, theme.heading, "Auth probes")); + if (probeSummary.results.length === 0) { + runtime.log(colorize(rich, theme.muted, "- none")); + } else { + const grouped = groupProbeResults(sortProbeResults(probeSummary.results)); + const statusColor = (status: string) => { + if (status === "ok") return theme.success; + if (status === "rate_limit") return theme.warn; + if (status === "timeout" || status === "billing") return theme.warn; + if (status === "auth" || status === "format") return theme.error; + if (status === "no_model") return theme.muted; + return theme.muted; + }; + for (const [provider, results] of grouped) { + const modelLabel = results.find((r) => r.model)?.model ?? "-"; + runtime.log( + `- ${theme.heading(provider)}${colorize( + rich, + theme.muted, + modelLabel ? ` (model: ${modelLabel})` : "", + )}`, + ); + for (const result of results) { + const status = colorize(rich, statusColor(result.status), result.status); + const latency = formatProbeLatency(result.latencyMs); + const mode = result.mode ? ` (${result.mode})` : ""; + const detail = result.error ? colorize(rich, theme.muted, ` - ${result.error}`) : ""; + runtime.log( + ` - ${colorize(rich, theme.accent, result.label)}${mode} ${status} ${colorize( + rich, + theme.muted, + latency, + )}${detail}`, + ); + } + } + runtime.log(colorize(rich, theme.muted, describeProbeSummary(probeSummary))); } } diff --git a/src/telegram/bot.create-telegram-bot.accepts-group-messages-mentionpatterns-match-without-botusername.test.ts b/src/telegram/bot.create-telegram-bot.accepts-group-messages-mentionpatterns-match-without-botusername.test.ts index 4fea3521a..7024a2e52 100644 --- a/src/telegram/bot.create-telegram-bot.accepts-group-messages-mentionpatterns-match-without-botusername.test.ts +++ b/src/telegram/bot.create-telegram-bot.accepts-group-messages-mentionpatterns-match-without-botusername.test.ts @@ -1,7 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { escapeRegExp, formatEnvelopeTimestamp } from "../../test/helpers/envelope-timestamp.js"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -111,7 +112,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -121,7 +122,11 @@ const getOnHandler = (event: string) => { const ORIGINAL_TZ = process.env.TZ; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); process.env.TZ = "UTC"; resetInboundDedupe(); loadConfig.mockReturnValue({ diff --git a/src/telegram/bot.create-telegram-bot.applies-topic-skill-filters-system-prompts.test.ts b/src/telegram/bot.create-telegram-bot.applies-topic-skill-filters-system-prompts.test.ts index 2afe8cd1c..1a10ca94c 100644 --- a/src/telegram/bot.create-telegram-bot.applies-topic-skill-filters-system-prompts.test.ts +++ b/src/telegram/bot.create-telegram-bot.applies-topic-skill-filters-system-prompts.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.blocks-all-group-messages-grouppolicy-is.test.ts b/src/telegram/bot.create-telegram-bot.blocks-all-group-messages-grouppolicy-is.test.ts index 6c712ca1d..7937c1064 100644 --- a/src/telegram/bot.create-telegram-bot.blocks-all-group-messages-grouppolicy-is.test.ts +++ b/src/telegram/bot.create-telegram-bot.blocks-all-group-messages-grouppolicy-is.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.dedupes-duplicate-callback-query-updates-by-update.test.ts b/src/telegram/bot.create-telegram-bot.dedupes-duplicate-callback-query-updates-by-update.test.ts index 9ed0ed677..5e8a2dcfa 100644 --- a/src/telegram/bot.create-telegram-bot.dedupes-duplicate-callback-query-updates-by-update.test.ts +++ b/src/telegram/bot.create-telegram-bot.dedupes-duplicate-callback-query-updates-by-update.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.installs-grammy-throttler.test.ts b/src/telegram/bot.create-telegram-bot.installs-grammy-throttler.test.ts index ab43c4269..05aac6388 100644 --- a/src/telegram/bot.create-telegram-bot.installs-grammy-throttler.test.ts +++ b/src/telegram/bot.create-telegram-bot.installs-grammy-throttler.test.ts @@ -1,9 +1,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { escapeRegExp, formatEnvelopeTimestamp } from "../../test/helpers/envelope-timestamp.js"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot, getTelegramSequentialKey } from "./bot.js"; import { resolveTelegramFetch } from "./fetch.js"; +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let getTelegramSequentialKey: typeof import("./bot.js").getTelegramSequentialKey; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; + const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), })); @@ -114,7 +116,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -125,7 +127,11 @@ const getOnHandler = (event: string) => { const ORIGINAL_TZ = process.env.TZ; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot, getTelegramSequentialKey } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); process.env.TZ = "UTC"; resetInboundDedupe(); loadConfig.mockReturnValue({ diff --git a/src/telegram/bot.create-telegram-bot.matches-tg-prefixed-allowfrom-entries-case-insensitively.test.ts b/src/telegram/bot.create-telegram-bot.matches-tg-prefixed-allowfrom-entries-case-insensitively.test.ts index dfdcf43e3..2c4dfa472 100644 --- a/src/telegram/bot.create-telegram-bot.matches-tg-prefixed-allowfrom-entries-case-insensitively.test.ts +++ b/src/telegram/bot.create-telegram-bot.matches-tg-prefixed-allowfrom-entries-case-insensitively.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.matches-usernames-case-insensitively-grouppolicy-is.test.ts b/src/telegram/bot.create-telegram-bot.matches-usernames-case-insensitively-grouppolicy-is.test.ts index 1e1174fbf..2281fb407 100644 --- a/src/telegram/bot.create-telegram-bot.matches-usernames-case-insensitively-grouppolicy-is.test.ts +++ b/src/telegram/bot.create-telegram-bot.matches-usernames-case-insensitively-grouppolicy-is.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.routes-dms-by-telegram-accountid-binding.test.ts b/src/telegram/bot.create-telegram-bot.routes-dms-by-telegram-accountid-binding.test.ts index 6e83c61c3..829391727 100644 --- a/src/telegram/bot.create-telegram-bot.routes-dms-by-telegram-accountid-binding.test.ts +++ b/src/telegram/bot.create-telegram-bot.routes-dms-by-telegram-accountid-binding.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -110,7 +111,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -119,7 +120,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.create-telegram-bot.sends-replies-without-native-reply-threading.test.ts b/src/telegram/bot.create-telegram-bot.sends-replies-without-native-reply-threading.test.ts index 74f87d63b..164095a9c 100644 --- a/src/telegram/bot.create-telegram-bot.sends-replies-without-native-reply-threading.test.ts +++ b/src/telegram/bot.create-telegram-bot.sends-replies-without-native-reply-threading.test.ts @@ -2,8 +2,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import { createTelegramBot } from "./bot.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; const { loadWebMedia } = vi.hoisted(() => ({ loadWebMedia: vi.fn(), @@ -113,7 +114,7 @@ vi.mock("../auto-reply/reply.js", () => { return { getReplyFromConfig: replySpy, __replySpy: replySpy }; }); -const replyModule = await import("../auto-reply/reply.js"); +let replyModule: typeof import("../auto-reply/reply.js"); const getOnHandler = (event: string) => { const handler = onSpy.mock.calls.find((call) => call[0] === event)?.[1]; @@ -122,7 +123,11 @@ const getOnHandler = (event: string) => { }; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); resetInboundDedupe(); loadConfig.mockReturnValue({ channels: { diff --git a/src/telegram/bot.test.ts b/src/telegram/bot.test.ts index 51beb4f4b..da67c2e38 100644 --- a/src/telegram/bot.test.ts +++ b/src/telegram/bot.test.ts @@ -6,18 +6,20 @@ import { listNativeCommandSpecs, listNativeCommandSpecsForConfig, } from "../auto-reply/commands-registry.js"; +import { escapeRegExp, formatEnvelopeTimestamp } from "../../test/helpers/envelope-timestamp.js"; +import { expectInboundContextContract } from "../../test/helpers/inbound-contract.js"; +import { resolveTelegramFetch } from "./fetch.js"; + +let createTelegramBot: typeof import("./bot.js").createTelegramBot; +let getTelegramSequentialKey: typeof import("./bot.js").getTelegramSequentialKey; +let resetInboundDedupe: typeof import("../auto-reply/reply/inbound-dedupe.js").resetInboundDedupe; +let replyModule: typeof import("../auto-reply/reply.js"); const { listSkillCommandsForAgents } = vi.hoisted(() => ({ listSkillCommandsForAgents: vi.fn(() => []), })); vi.mock("../auto-reply/skill-commands.js", () => ({ listSkillCommandsForAgents, })); -import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js"; -import * as replyModule from "../auto-reply/reply.js"; -import { expectInboundContextContract } from "../../test/helpers/inbound-contract.js"; -import { escapeRegExp, formatEnvelopeTimestamp } from "../../test/helpers/envelope-timestamp.js"; -import { createTelegramBot, getTelegramSequentialKey } from "./bot.js"; -import { resolveTelegramFetch } from "./fetch.js"; function resolveSkillCommands(config: Parameters[0]) { return listSkillCommandsForAgents({ cfg: config }); @@ -155,7 +157,11 @@ const getOnHandler = (event: string) => { const ORIGINAL_TZ = process.env.TZ; describe("createTelegramBot", () => { - beforeEach(() => { + beforeEach(async () => { + vi.resetModules(); + ({ resetInboundDedupe } = await import("../auto-reply/reply/inbound-dedupe.js")); + ({ createTelegramBot, getTelegramSequentialKey } = await import("./bot.js")); + replyModule = await import("../auto-reply/reply.js"); process.env.TZ = "UTC"; resetInboundDedupe(); loadConfig.mockReturnValue({