diff --git a/CHANGELOG.md b/CHANGELOG.md index 6073b8224..01b4cb5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Debugging: add raw model stream logging flags and document gateway watch mode. - CLI: improve `logs` output (pretty/plain/JSONL), add gateway unreachable hint, and document logging. - WhatsApp: route queued replies to the original sender instead of the bot's own number. (#534) — thanks @mcinteerj - Models: add OAuth expiry checks in doctor, expanded `models status` auth output (missing auth + `--check` exit codes). (#538) — thanks @latitudeki5223 diff --git a/docs/debugging.md b/docs/debugging.md new file mode 100644 index 000000000..ac8827150 --- /dev/null +++ b/docs/debugging.md @@ -0,0 +1,86 @@ +--- +summary: "Debugging tools: watch mode, raw model streams, and tracing reasoning leakage" +read_when: + - You need to inspect raw model output for reasoning leakage + - You want to run the Gateway in watch mode while iterating + - You need a repeatable debugging workflow +--- + +# Debugging + +This page covers debugging helpers for streaming output, especially when a +provider mixes reasoning into normal text. + +## Gateway watch mode + +For fast iteration, run the gateway under the file watcher: + +```bash +pnpm gateway:watch --force +``` + +This maps to: + +```bash +tsx watch src/entry.ts gateway --force +``` + +Add any gateway CLI flags after `gateway:watch` and they will be passed through +on each restart. + +## Raw stream logging (Clawdbot) + +Clawdbot can log the **raw assistant stream** before any filtering/formatting. +This is the best way to see whether reasoning is arriving as plain text deltas +(or as separate thinking blocks). + +Enable it via CLI: + +```bash +pnpm gateway:watch --force --raw-stream +``` + +Optional path override: + +```bash +pnpm gateway:watch --force --raw-stream --raw-stream-path ~/.clawdbot/logs/raw-stream.jsonl +``` + +Equivalent env vars: + +```bash +CLAWDBOT_RAW_STREAM=1 +CLAWDBOT_RAW_STREAM_PATH=~/.clawdbot/logs/raw-stream.jsonl +``` + +Default file: + +`~/.clawdbot/logs/raw-stream.jsonl` + +## Raw chunk logging (pi-mono) + +To capture **raw OpenAI-compat chunks** before they are parsed into blocks, +pi-mono exposes a separate logger: + +```bash +PI_RAW_STREAM=1 +``` + +Optional path: + +```bash +PI_RAW_STREAM_PATH=~/.pi-mono/logs/raw-openai-completions.jsonl +``` + +Default file: + +`~/.pi-mono/logs/raw-openai-completions.jsonl` + +> Note: this is only emitted by processes using pi-mono’s +> `openai-completions` provider. + +## Safety notes + +- Raw stream logs can include full prompts, tool output, and user data. +- Keep logs local and delete them after debugging. +- If you share logs, scrub secrets and PII first. diff --git a/src/agents/pi-embedded-subscribe.ts b/src/agents/pi-embedded-subscribe.ts index a5d01251a..c3f703e40 100644 --- a/src/agents/pi-embedded-subscribe.ts +++ b/src/agents/pi-embedded-subscribe.ts @@ -1,8 +1,11 @@ +import fs from "node:fs"; +import path from "node:path"; import type { AgentEvent, AgentMessage } from "@mariozechner/pi-agent-core"; import type { AssistantMessage } from "@mariozechner/pi-ai"; import type { AgentSession } from "@mariozechner/pi-coding-agent"; import type { ReasoningLevel } from "../auto-reply/thinking.js"; import { formatToolAggregate } from "../auto-reply/tool-meta.js"; +import { resolveStateDir } from "../config/paths.js"; import { emitAgentEvent } from "../infra/agent-events.js"; import { createSubsystemLogger } from "../logging.js"; import { splitMediaFromOutput } from "../media/parse.js"; @@ -23,6 +26,31 @@ const THINKING_OPEN_GLOBAL_RE = /<\s*think(?:ing)?\s*>/gi; const THINKING_CLOSE_GLOBAL_RE = /<\s*\/\s*think(?:ing)?\s*>/gi; const TOOL_RESULT_MAX_CHARS = 8000; const log = createSubsystemLogger("agent/embedded"); +const RAW_STREAM_ENABLED = process.env.CLAWDBOT_RAW_STREAM === "1"; +const RAW_STREAM_PATH = + process.env.CLAWDBOT_RAW_STREAM_PATH?.trim() || + path.join(resolveStateDir(), "logs", "raw-stream.jsonl"); +let rawStreamReady = false; + +const appendRawStream = (payload: Record) => { + if (!RAW_STREAM_ENABLED) return; + if (!rawStreamReady) { + rawStreamReady = true; + try { + fs.mkdirSync(path.dirname(RAW_STREAM_PATH), { recursive: true }); + } catch { + // ignore raw stream mkdir failures + } + } + try { + void fs.promises.appendFile( + RAW_STREAM_PATH, + `${JSON.stringify(payload)}\n`, + ); + } catch { + // ignore raw stream write failures + } +}; export type { BlockReplyChunking } from "./pi-embedded-block-chunker.js"; @@ -664,6 +692,15 @@ export function subscribeEmbeddedPiSession(params: { typeof assistantRecord?.content === "string" ? assistantRecord.content : ""; + appendRawStream({ + ts: Date.now(), + event: "assistant_text_stream", + runId: params.runId, + sessionId: (params.session as { id?: string }).id, + evtType, + delta, + content, + }); let chunk = ""; if (evtType === "text_delta") { chunk = delta; @@ -756,6 +793,14 @@ export function subscribeEmbeddedPiSession(params: { if (msg?.role === "assistant") { const assistantMessage = msg as AssistantMessage; const rawText = extractAssistantText(assistantMessage); + appendRawStream({ + ts: Date.now(), + event: "assistant_message_end", + runId: params.runId, + sessionId: (params.session as { id?: string }).id, + rawText, + rawThinking: extractAssistantThinking(assistantMessage), + }); const cleaned = params.enforceFinalTag ? stripThinkingSegments(stripUnpairedThinkingTags(rawText)) : stripThinkingSegments(rawText); diff --git a/src/auto-reply/reply.directive.test.ts b/src/auto-reply/reply.directive.test.ts index fcb3087c8..fa8c5051c 100644 --- a/src/auto-reply/reply.directive.test.ts +++ b/src/auto-reply/reply.directive.test.ts @@ -599,7 +599,7 @@ describe("directive behavior", () => { const text = Array.isArray(res) ? res[0]?.text : res?.text; expect(text).toContain("Elevated mode disabled."); - expect(text).toContain("status agent:main:main"); + expect(text).toContain("Session: agent:main:main"); expect(runEmbeddedPiAgent).not.toHaveBeenCalled(); }); }); diff --git a/src/auto-reply/reply/commands.ts b/src/auto-reply/reply/commands.ts index 4879323e0..88e744c0e 100644 --- a/src/auto-reply/reply/commands.ts +++ b/src/auto-reply/reply/commands.ts @@ -555,7 +555,6 @@ export async function handleCommands(params: { const reply = await buildStatusReply({ cfg, command, - provider: command.provider, sessionEntry, sessionKey, sessionScope, diff --git a/src/auto-reply/status.test.ts b/src/auto-reply/status.test.ts index 8813d753d..ef0ac5bc1 100644 --- a/src/auto-reply/status.test.ts +++ b/src/auto-reply/status.test.ts @@ -65,7 +65,6 @@ describe("buildStatusMessage", () => { }, }, }, - }, } as ClawdbotConfig, agent: { model: "anthropic/pi:opus", @@ -248,7 +247,6 @@ describe("buildStatusMessage", () => { }, }, }, - }, } as ClawdbotConfig, agent: { model: "anthropic/claude-opus-4-5" }, sessionEntry: { sessionId: "c1", updatedAt: 0, inputTokens: 10 }, diff --git a/src/auto-reply/status.ts b/src/auto-reply/status.ts index 968be2d76..668304b20 100644 --- a/src/auto-reply/status.ts +++ b/src/auto-reply/status.ts @@ -296,7 +296,10 @@ export function buildStatusMessage(args: StatusArgs): string { const activationLine = activationParts.filter(Boolean).join(" · "); const authMode = resolveModelAuthMode(provider, args.config); - const showCost = authMode === "api-key"; + const authLabelValue = + args.modelAuth ?? + (authMode && authMode !== "unknown" ? authMode : undefined); + const showCost = authLabelValue === "api-key" || authLabelValue === "mixed"; const costConfig = showCost ? resolveModelCostConfig({ provider, @@ -319,9 +322,6 @@ export function buildStatusMessage(args: StatusArgs): string { const costLabel = showCost && hasUsage ? formatUsd(cost) : undefined; const modelLabel = model ? `${provider}/${model}` : "unknown"; - const authLabelValue = - args.modelAuth ?? - (authMode && authMode !== "unknown" ? authMode : undefined); const authLabel = authLabelValue ? ` · 🔑 ${authLabelValue}` : ""; const modelLine = `🧠 Model: ${modelLabel}${authLabel}`; const commit = resolveCommitHash(); diff --git a/src/cli/gateway-cli.ts b/src/cli/gateway-cli.ts index 7168a8054..5222af2b4 100644 --- a/src/cli/gateway-cli.ts +++ b/src/cli/gateway-cli.ts @@ -50,6 +50,8 @@ type GatewayRunOpts = { verbose?: boolean; wsLog?: unknown; compact?: boolean; + rawStream?: boolean; + rawStreamPath?: unknown; }; type GatewayRunParams = { @@ -300,6 +302,14 @@ async function runGatewayCommand( } setGatewayWsLogStyle(wsLogStyle); + if (opts.rawStream) { + process.env.CLAWDBOT_RAW_STREAM = "1"; + } + const rawStreamPath = toOptionString(opts.rawStreamPath); + if (rawStreamPath) { + process.env.CLAWDBOT_RAW_STREAM_PATH = rawStreamPath; + } + const cfg = loadConfig(); const portOverride = parsePort(opts.port); if (opts.port !== undefined && portOverride === null) { @@ -565,6 +575,8 @@ function addGatewayRunCommand( "auto", ) .option("--compact", 'Alias for "--ws-log compact"', false) + .option("--raw-stream", "Log raw model stream events to jsonl", false) + .option("--raw-stream-path ", "Raw stream jsonl path") .action(async (opts) => { await runGatewayCommand(opts, params); });