Merge pull request #1370 from parubets/fix-debug-ttl-cache

WIP: cache trace mvp for Anthropic

CHANGELOG.md

@@ -12,6 +12,7 @@ Docs: https://docs.clawd.bot
 - Nodes tool: include agent/node/gateway context in tool failure logs to speed approval debugging.
 - macOS: exec approvals now respect wildcard agent allowlists (`*`).
 - UI: remove the chat stop button and keep the composer aligned to the bottom edge.
+- Agents: add diagnostics cache trace config and fix cache trace logging edge cases. (#1370) — thanks @parubets.
 
 ## 2026.1.20
 

src/agents/cache-trace.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from "vitest";
+
+import type { ClawdbotConfig } from "../config/config.js";
+import { resolveUserPath } from "../utils.js";
+import { createCacheTrace } from "./cache-trace.js";
+
+describe("createCacheTrace", () => {
+  it("returns null when diagnostics cache tracing is disabled", () => {
+    const trace = createCacheTrace({
+      cfg: {} as ClawdbotConfig,
+      env: {},
+    });
+
+    expect(trace).toBeNull();
+  });
+
+  it("honors diagnostics cache trace config and expands file paths", () => {
+    const lines: string[] = [];
+    const trace = createCacheTrace({
+      cfg: {
+        diagnostics: {
+          cacheTrace: {
+            enabled: true,
+            filePath: "~/.clawdbot/logs/cache-trace.jsonl",
+          },
+        },
+      },
+      env: {},
+      writer: {
+        filePath: "memory",
+        write: (line) => lines.push(line),
+      },
+    });
+
+    expect(trace).not.toBeNull();
+    expect(trace?.filePath).toBe(resolveUserPath("~/.clawdbot/logs/cache-trace.jsonl"));
+
+    trace?.recordStage("session:loaded", {
+      messages: [],
+      system: "sys",
+    });
+
+    expect(lines.length).toBe(1);
+  });
+
+  it("records empty prompt/system values when enabled", () => {
+    const lines: string[] = [];
+    const trace = createCacheTrace({
+      cfg: {
+        diagnostics: {
+          cacheTrace: {
+            enabled: true,
+            includePrompt: true,
+            includeSystem: true,
+          },
+        },
+      },
+      env: {},
+      writer: {
+        filePath: "memory",
+        write: (line) => lines.push(line),
+      },
+    });
+
+    trace?.recordStage("prompt:before", { prompt: "", system: "" });
+
+    const event = JSON.parse(lines[0]?.trim() ?? "{}") as Record<string, unknown>;
+    expect(event.prompt).toBe("");
+    expect(event.system).toBe("");
+  });
+
+  it("respects env overrides for enablement", () => {
+    const lines: string[] = [];
+    const trace = createCacheTrace({
+      cfg: {
+        diagnostics: {
+          cacheTrace: {
+            enabled: true,
+          },
+        },
+      },
+      env: {
+        CLAWDBOT_CACHE_TRACE: "0",
+      },
+      writer: {
+        filePath: "memory",
+        write: (line) => lines.push(line),
+      },
+    });
+
+    expect(trace).toBeNull();
+  });
+});

src/agents/cache-trace.ts

@@ -0,0 +1,271 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import type { AgentMessage, StreamFn } from "@mariozechner/pi-agent-core";
+import type { Api, Model } from "@mariozechner/pi-ai";
+
+import type { ClawdbotConfig } from "../config/config.js";
+import { resolveStateDir } from "../config/paths.js";
+import { parseBooleanValue } from "../utils/boolean.js";
+import { resolveUserPath } from "../utils.js";
+
+export type CacheTraceStage =
+  | "session:loaded"
+  | "session:sanitized"
+  | "session:limited"
+  | "prompt:before"
+  | "prompt:images"
+  | "stream:context"
+  | "session:after";
+
+export type CacheTraceEvent = {
+  ts: string;
+  seq: number;
+  stage: CacheTraceStage;
+  runId?: string;
+  sessionId?: string;
+  sessionKey?: string;
+  provider?: string;
+  modelId?: string;
+  modelApi?: string | null;
+  workspaceDir?: string;
+  prompt?: string;
+  system?: unknown;
+  options?: Record<string, unknown>;
+  model?: Record<string, unknown>;
+  messages?: AgentMessage[];
+  messageCount?: number;
+  messageRoles?: Array<string | undefined>;
+  messageFingerprints?: string[];
+  messagesDigest?: string;
+  systemDigest?: string;
+  note?: string;
+  error?: string;
+};
+
+export type CacheTrace = {
+  enabled: true;
+  filePath: string;
+  recordStage: (stage: CacheTraceStage, payload?: Partial<CacheTraceEvent>) => void;
+  wrapStreamFn: (streamFn: StreamFn) => StreamFn;
+};
+
+type CacheTraceInit = {
+  cfg?: ClawdbotConfig;
+  env?: NodeJS.ProcessEnv;
+  runId?: string;
+  sessionId?: string;
+  sessionKey?: string;
+  provider?: string;
+  modelId?: string;
+  modelApi?: string | null;
+  workspaceDir?: string;
+  writer?: CacheTraceWriter;
+};
+
+type CacheTraceConfig = {
+  enabled: boolean;
+  filePath: string;
+  includeMessages: boolean;
+  includePrompt: boolean;
+  includeSystem: boolean;
+};
+
+type CacheTraceWriter = {
+  filePath: string;
+  write: (line: string) => void;
+};
+
+const writers = new Map<string, CacheTraceWriter>();
+
+function resolveCacheTraceConfig(params: CacheTraceInit): CacheTraceConfig {
+  const env = params.env ?? process.env;
+  const config = params.cfg?.diagnostics?.cacheTrace;
+  const envEnabled = parseBooleanValue(env.CLAWDBOT_CACHE_TRACE);
+  const enabled = envEnabled ?? config?.enabled ?? false;
+  const fileOverride = config?.filePath?.trim() || env.CLAWDBOT_CACHE_TRACE_FILE?.trim();
+  const filePath = fileOverride
+    ? resolveUserPath(fileOverride)
+    : path.join(resolveStateDir(env), "logs", "cache-trace.jsonl");
+
+  const includeMessages =
+    parseBooleanValue(env.CLAWDBOT_CACHE_TRACE_MESSAGES) ?? config?.includeMessages;
+  const includePrompt = parseBooleanValue(env.CLAWDBOT_CACHE_TRACE_PROMPT) ?? config?.includePrompt;
+  const includeSystem = parseBooleanValue(env.CLAWDBOT_CACHE_TRACE_SYSTEM) ?? config?.includeSystem;
+
+  return {
+    enabled,
+    filePath,
+    includeMessages: includeMessages ?? true,
+    includePrompt: includePrompt ?? true,
+    includeSystem: includeSystem ?? true,
+  };
+}
+
+function getWriter(filePath: string): CacheTraceWriter {
+  const existing = writers.get(filePath);
+  if (existing) return existing;
+
+  const dir = path.dirname(filePath);
+  const ready = fs.mkdir(dir, { recursive: true }).catch(() => undefined);
+  let queue = Promise.resolve();
+
+  const writer: CacheTraceWriter = {
+    filePath,
+    write: (line: string) => {
+      queue = queue
+        .then(() => ready)
+        .then(() => fs.appendFile(filePath, line, "utf8"))
+        .catch(() => undefined);
+    },
+  };
+
+  writers.set(filePath, writer);
+  return writer;
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null || value === undefined) return String(value);
+  if (typeof value === "number" && !Number.isFinite(value)) return JSON.stringify(String(value));
+  if (typeof value === "bigint") return JSON.stringify(value.toString());
+  if (typeof value !== "object") return JSON.stringify(value) ?? "null";
+  if (value instanceof Error) {
+    return stableStringify({
+      name: value.name,
+      message: value.message,
+      stack: value.stack,
+    });
+  }
+  if (value instanceof Uint8Array) {
+    return stableStringify({
+      type: "Uint8Array",
+      data: Buffer.from(value).toString("base64"),
+    });
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map((entry) => stableStringify(entry)).join(",")}]`;
+  }
+  const record = value as Record<string, unknown>;
+  const keys = Object.keys(record).sort();
+  const entries = keys.map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`);
+  return `{${entries.join(",")}}`;
+}
+
+function digest(value: unknown): string {
+  const serialized = stableStringify(value);
+  return crypto.createHash("sha256").update(serialized).digest("hex");
+}
+
+function summarizeMessages(messages: AgentMessage[]): {
+  messageCount: number;
+  messageRoles: Array<string | undefined>;
+  messageFingerprints: string[];
+  messagesDigest: string;
+} {
+  const messageFingerprints = messages.map((msg) => digest(msg));
+  return {
+    messageCount: messages.length,
+    messageRoles: messages.map((msg) => (msg as { role?: string }).role),
+    messageFingerprints,
+    messagesDigest: digest(messageFingerprints.join("|")),
+  };
+}
+
+function safeJsonStringify(value: unknown): string | null {
+  try {
+    return JSON.stringify(value, (_key, val) => {
+      if (typeof val === "bigint") return val.toString();
+      if (typeof val === "function") return "[Function]";
+      if (val instanceof Error) {
+        return { name: val.name, message: val.message, stack: val.stack };
+      }
+      if (val instanceof Uint8Array) {
+        return { type: "Uint8Array", data: Buffer.from(val).toString("base64") };
+      }
+      return val;
+    });
+  } catch {
+    return null;
+  }
+}
+
+export function createCacheTrace(params: CacheTraceInit): CacheTrace | null {
+  const cfg = resolveCacheTraceConfig(params);
+  if (!cfg.enabled) return null;
+
+  const writer = params.writer ?? getWriter(cfg.filePath);
+  let seq = 0;
+
+  const base: Omit<CacheTraceEvent, "ts" | "seq" | "stage"> = {
+    runId: params.runId,
+    sessionId: params.sessionId,
+    sessionKey: params.sessionKey,
+    provider: params.provider,
+    modelId: params.modelId,
+    modelApi: params.modelApi,
+    workspaceDir: params.workspaceDir,
+  };
+
+  const recordStage: CacheTrace["recordStage"] = (stage, payload = {}) => {
+    const event: CacheTraceEvent = {
+      ...base,
+      ts: new Date().toISOString(),
+      seq: (seq += 1),
+      stage,
+    };
+
+    if (payload.prompt !== undefined && cfg.includePrompt) {
+      event.prompt = payload.prompt;
+    }
+    if (payload.system !== undefined && cfg.includeSystem) {
+      event.system = payload.system;
+      event.systemDigest = digest(payload.system);
+    }
+    if (payload.options) event.options = payload.options;
+    if (payload.model) event.model = payload.model;
+
+    const messages = payload.messages;
+    if (Array.isArray(messages)) {
+      const summary = summarizeMessages(messages);
+      event.messageCount = summary.messageCount;
+      event.messageRoles = summary.messageRoles;
+      event.messageFingerprints = summary.messageFingerprints;
+      event.messagesDigest = summary.messagesDigest;
+      if (cfg.includeMessages) {
+        event.messages = messages;
+      }
+    }
+
+    if (payload.note) event.note = payload.note;
+    if (payload.error) event.error = payload.error;
+
+    const line = safeJsonStringify(event);
+    if (!line) return;
+    writer.write(`${line}\n`);
+  };
+
+  const wrapStreamFn: CacheTrace["wrapStreamFn"] = (streamFn) => {
+    const wrapped: StreamFn = (model, context, options) => {
+      recordStage("stream:context", {
+        model: {
+          id: (model as Model<Api>)?.id,
+          provider: (model as Model<Api>)?.provider,
+          api: (model as Model<Api>)?.api,
+        },
+        system: (context as { system?: unknown }).system,
+        messages: (context as { messages?: AgentMessage[] }).messages ?? [],
+        options: (options ?? {}) as Record<string, unknown>,
+      });
+      return streamFn(model, context, options);
+    };
+    return wrapped;
+  };
+
+  return {
+    enabled: true,
+    filePath: cfg.filePath,
+    recordStage,
+    wrapStreamFn,
+  };
+}

src/agents/pi-embedded-runner/run/attempt.ts

@@ -16,6 +16,7 @@ import { normalizeMessageChannel } from "../../../utils/message-channel.js";
 import { isReasoningTagProvider } from "../../../utils/provider-utils.js";
 import { isSubagentSessionKey } from "../../../routing/session-key.js";
 import { resolveUserPath } from "../../../utils.js";
+import { createCacheTrace } from "../../cache-trace.js";
 import { resolveClawdbotAgentDir } from "../../agent-paths.js";
 import { resolveSessionAgentIds } from "../../agent-scope.js";
 import { makeBootstrapWarn, resolveBootstrapContextForRun } from "../../bootstrap-files.js";
@@ -367,6 +368,17 @@ export async function runEmbeddedAttempt(
         throw new Error("Embedded agent session missing");
       }
       const activeSession = session;
+      const cacheTrace = createCacheTrace({
+        cfg: params.config,
+        env: process.env,
+        runId: params.runId,
+        sessionId: activeSession.sessionId,
+        sessionKey: params.sessionKey,
+        provider: params.provider,
+        modelId: params.modelId,
+        modelApi: params.model.api,
+        workspaceDir: params.workspaceDir,
+      });
 
       // Force a stable streamFn reference so vitest can reliably mock @mariozechner/pi-ai.
       activeSession.agent.streamFn = streamSimple;
@@ -379,6 +391,15 @@ export async function runEmbeddedAttempt(
         params.streamParams,
       );
 
+      if (cacheTrace) {
+        cacheTrace.recordStage("session:loaded", {
+          messages: activeSession.messages,
+          system: systemPrompt,
+          note: "after session create",
+        });
+        activeSession.agent.streamFn = cacheTrace.wrapStreamFn(activeSession.agent.streamFn);
+      }
+
       try {
         const prior = await sanitizeSessionHistory({
           messages: activeSession.messages,
@@ -388,12 +409,14 @@ export async function runEmbeddedAttempt(
           sessionManager,
           sessionId: params.sessionId,
         });
+        cacheTrace?.recordStage("session:sanitized", { messages: prior });
         const validatedGemini = validateGeminiTurns(prior);
         const validated = validateAnthropicTurns(validatedGemini);
         const limited = limitHistoryTurns(
           validated,
           getDmHistoryLimitFromSessionKey(params.sessionKey, params.config),
         );
+        cacheTrace?.recordStage("session:limited", { messages: limited });
         if (limited.length > 0) {
           activeSession.agent.replaceMessages(limited);
         }
@@ -564,6 +587,10 @@ export async function runEmbeddedAttempt(
         }
 
         log.debug(`embedded run prompt start: runId=${params.runId} sessionId=${params.sessionId}`);
+        cacheTrace?.recordStage("prompt:before", {
+          prompt: effectivePrompt,
+          messages: activeSession.messages,
+        });
 
         // Repair orphaned trailing user messages so new prompts don't violate role ordering.
         const leafEntry = sessionManager.getLeafEntry();
@@ -633,6 +660,12 @@ export async function runEmbeddedAttempt(
             }
           }
 
+          cacheTrace?.recordStage("prompt:images", {
+            prompt: effectivePrompt,
+            messages: activeSession.messages,
+            note: `images: prompt=${imageResult.images.length} history=${imageResult.historyImagesByIndex.size}`,
+          });
+
           // Only pass images option if there are actually images to pass
           // This avoids potential issues with models that don't expect the images parameter
           if (imageResult.images.length > 0) {
@@ -660,6 +693,10 @@ export async function runEmbeddedAttempt(
 
         messagesSnapshot = activeSession.messages.slice();
         sessionIdUsed = activeSession.sessionId;
+        cacheTrace?.recordStage("session:after", {
+          messages: messagesSnapshot,
+          note: promptError ? "prompt error" : undefined,
+        });
 
         // Run agent_end hooks to allow plugins to analyze the conversation
         // This is fire-and-forget, so we don't await

src/cli/exec-approvals-cli.test.ts

@@ -115,6 +115,10 @@ describe("exec approvals CLI", () => {
     runtimeErrors.length = 0;
     callGatewayFromCli.mockClear();
 
+    const execApprovals = await import("../infra/exec-approvals.js");
+    const saveExecApprovals = vi.mocked(execApprovals.saveExecApprovals);
+    saveExecApprovals.mockClear();
+
     const { registerExecApprovalsCli } = await import("./exec-approvals-cli.js");
     const program = new Command();
     program.exitOverride();
@@ -122,9 +126,17 @@ describe("exec approvals CLI", () => {
 
     await program.parseAsync(["approvals", "allowlist", "add", "/usr/bin/uname"], { from: "user" });
 
-    const setCall = callGatewayFromCli.mock.calls.find((call) => call[0] === "exec.approvals.set");
-    expect(setCall).toBeTruthy();
-    const params = setCall?.[2] as { file: { agents?: Record<string, unknown> } };
-    expect(params.file.agents?.["*"]).toBeTruthy();
+    expect(callGatewayFromCli).not.toHaveBeenCalledWith(
+      "exec.approvals.set",
+      expect.anything(),
+      {},
+    );
+    expect(saveExecApprovals).toHaveBeenCalledWith(
+      expect.objectContaining({
+        agents: expect.objectContaining({
+          "*": expect.anything(),
+        }),
+      }),
+    );
   });
 });

src/cli/exec-approvals-cli.ts

@@ -13,6 +13,7 @@ import { formatDocsLink } from "../terminal/links.js";
 import { isRich, theme } from "../terminal/theme.js";
 import { renderTable } from "../terminal/table.js";
 import { callGatewayFromCli } from "./gateway-rpc.js";
+import { describeUnknownError } from "./gateway-cli/shared.js";
 import { nodesCallOpts, resolveNodeId } from "./nodes-cli/rpc.js";
 import type { NodesRpcOpts } from "./nodes-cli/types.js";
 
@@ -96,7 +97,7 @@ async function loadSnapshotTarget(opts: ExecApprovalsCliOpts): Promise<{
 }
 
 function formatCliError(err: unknown): string {
-  const msg = String(err ?? "unknown error");
+  const msg = describeUnknownError(err);
   return msg.includes("\n") ? msg.split("\n")[0] : msg;
 }
 

src/config/schema.ts

@@ -114,6 +114,11 @@ const FIELD_LABELS: Record<string, string> = {
   "diagnostics.otel.logs": "OpenTelemetry Logs Enabled",
   "diagnostics.otel.sampleRate": "OpenTelemetry Trace Sample Rate",
   "diagnostics.otel.flushIntervalMs": "OpenTelemetry Flush Interval (ms)",
+  "diagnostics.cacheTrace.enabled": "Cache Trace Enabled",
+  "diagnostics.cacheTrace.filePath": "Cache Trace File Path",
+  "diagnostics.cacheTrace.includeMessages": "Cache Trace Include Messages",
+  "diagnostics.cacheTrace.includePrompt": "Cache Trace Include Prompt",
+  "diagnostics.cacheTrace.includeSystem": "Cache Trace Include System",
   "gateway.remote.url": "Remote Gateway URL",
   "gateway.remote.sshTarget": "Remote Gateway SSH Target",
   "gateway.remote.sshIdentity": "Remote Gateway SSH Identity",
@@ -345,6 +350,14 @@ const FIELD_HELP: Record<string, string> = {
     "Extra node.invoke commands to allow beyond the gateway defaults (array of command strings).",
   "gateway.nodes.denyCommands":
     "Commands to block even if present in node claims or default allowlist.",
+  "diagnostics.cacheTrace.enabled":
+    "Log cache trace snapshots for embedded agent runs (default: false).",
+  "diagnostics.cacheTrace.filePath":
+    "JSONL output path for cache trace logs (default: $CLAWDBOT_STATE_DIR/logs/cache-trace.jsonl).",
+  "diagnostics.cacheTrace.includeMessages":
+    "Include full message payloads in trace output (default: true).",
+  "diagnostics.cacheTrace.includePrompt": "Include prompt text in trace output (default: true).",
+  "diagnostics.cacheTrace.includeSystem": "Include system prompt in trace output (default: true).",
   "tools.exec.applyPatch.enabled":
     "Experimental. Enables apply_patch for OpenAI models when allowed by tool policy.",
   "tools.exec.applyPatch.allowModels":

src/config/types.base.ts

@@ -117,9 +117,18 @@ export type DiagnosticsOtelConfig = {
   flushIntervalMs?: number;
 };
 
+export type DiagnosticsCacheTraceConfig = {
+  enabled?: boolean;
+  filePath?: string;
+  includeMessages?: boolean;
+  includePrompt?: boolean;
+  includeSystem?: boolean;
+};
+
 export type DiagnosticsConfig = {
   enabled?: boolean;
   otel?: DiagnosticsOtelConfig;
+  cacheTrace?: DiagnosticsCacheTraceConfig;
 };
 
 export type WebReconnectConfig = {

src/config/zod-schema.ts

@@ -63,6 +63,16 @@ export const ClawdbotSchema = z
           })
           .strict()
           .optional(),
+        cacheTrace: z
+          .object({
+            enabled: z.boolean().optional(),
+            filePath: z.string().optional(),
+            includeMessages: z.boolean().optional(),
+            includePrompt: z.boolean().optional(),
+            includeSystem: z.boolean().optional(),
+          })
+          .strict()
+          .optional(),
       })
       .strict()
       .optional(),