feat: add experimental session memory source

This commit is contained in:
Peter Steinberger
2026-01-17 18:53:48 +00:00
parent 8ec4af4641
commit 0e49dca53c
13 changed files with 445 additions and 42 deletions

View File

@@ -57,6 +57,7 @@ Docs: https://docs.clawd.bot
- Directory: unify `clawdbot directory` across channels and plugin channels. - Directory: unify `clawdbot directory` across channels and plugin channels.
- UI: allow deleting sessions from the Control UI. - UI: allow deleting sessions from the Control UI.
- Memory: add sqlite-vec vector acceleration with CLI status details. - Memory: add sqlite-vec vector acceleration with CLI status details.
- Memory: add experimental session transcript indexing for memory_search (opt-in via memorySearch.experimental.sessionMemory + sources).
- Skills: add user-invocable skill commands and expanded skill command registration. - Skills: add user-invocable skill commands and expanded skill command registration.
- Telegram: default reaction level to minimal and enable reaction notifications by default. - Telegram: default reaction level to minimal and enable reaction notifications by default.
- Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2. - Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2.

View File

@@ -144,6 +144,28 @@ Local mode:
- Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token). - Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
- Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change. - Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change.
### Session memory search (experimental)
You can optionally index **session transcripts** and surface them via `memory_search`.
This is gated behind an experimental flag.
```json5
agents: {
defaults: {
memorySearch: {
experimental: { sessionMemory: true },
sources: ["memory", "sessions"]
}
}
}
```
Notes:
- Session indexing is **opt-in** (off by default).
- Session updates are debounced and indexed lazily on the next `memory_search` (or manual `clawdbot memory index`).
- Results still include snippets only; `memory_get` remains limited to memory files.
- Session indexing is isolated per agent (only that agents session logs are indexed).
### SQLite vector acceleration (sqlite-vec) ### SQLite vector acceleration (sqlite-vec)
When the sqlite-vec extension is available, Clawdbot stores embeddings in a When the sqlite-vec extension is available, Clawdbot stores embeddings in a

View File

@@ -99,4 +99,42 @@ describe("memory search config", () => {
headers: { "X-Default": "on" }, headers: { "X-Default": "on" },
}); });
}); });
it("gates session sources behind experimental flag", () => {
const cfg = {
agents: {
defaults: {
memorySearch: {
sources: ["memory", "sessions"],
},
},
list: [
{
id: "main",
default: true,
memorySearch: {
experimental: { sessionMemory: false },
},
},
],
},
};
const resolved = resolveMemorySearchConfig(cfg, "main");
expect(resolved?.sources).toEqual(["memory"]);
});
it("allows session sources when experimental flag is enabled", () => {
const cfg = {
agents: {
defaults: {
memorySearch: {
sources: ["memory", "sessions"],
experimental: { sessionMemory: true },
},
},
},
};
const resolved = resolveMemorySearchConfig(cfg, "main");
expect(resolved?.sources).toContain("sessions");
});
}); });

View File

@@ -8,12 +8,16 @@ import { resolveAgentConfig } from "./agent-scope.js";
export type ResolvedMemorySearchConfig = { export type ResolvedMemorySearchConfig = {
enabled: boolean; enabled: boolean;
sources: Array<"memory" | "sessions">;
provider: "openai" | "local"; provider: "openai" | "local";
remote?: { remote?: {
baseUrl?: string; baseUrl?: string;
apiKey?: string; apiKey?: string;
headers?: Record<string, string>; headers?: Record<string, string>;
}; };
experimental: {
sessionMemory: boolean;
};
fallback: "openai" | "none"; fallback: "openai" | "none";
model: string; model: string;
local: { local: {
@@ -51,6 +55,21 @@ const DEFAULT_CHUNK_OVERLAP = 80;
const DEFAULT_WATCH_DEBOUNCE_MS = 1500; const DEFAULT_WATCH_DEBOUNCE_MS = 1500;
const DEFAULT_MAX_RESULTS = 6; const DEFAULT_MAX_RESULTS = 6;
const DEFAULT_MIN_SCORE = 0.35; const DEFAULT_MIN_SCORE = 0.35;
const DEFAULT_SOURCES: Array<"memory" | "sessions"> = ["memory"];
function normalizeSources(
sources: Array<"memory" | "sessions"> | undefined,
sessionMemoryEnabled: boolean,
): Array<"memory" | "sessions"> {
const normalized = new Set<"memory" | "sessions">();
const input = sources?.length ? sources : DEFAULT_SOURCES;
for (const source of input) {
if (source === "memory") normalized.add("memory");
if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions");
}
if (normalized.size === 0) normalized.add("memory");
return Array.from(normalized);
}
function resolveStorePath(agentId: string, raw?: string): string { function resolveStorePath(agentId: string, raw?: string): string {
const stateDir = resolveStateDir(process.env, os.homedir); const stateDir = resolveStateDir(process.env, os.homedir);
@@ -66,6 +85,8 @@ function mergeConfig(
agentId: string, agentId: string,
): ResolvedMemorySearchConfig { ): ResolvedMemorySearchConfig {
const enabled = overrides?.enabled ?? defaults?.enabled ?? true; const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
const sessionMemory =
overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false;
const provider = overrides?.provider ?? defaults?.provider ?? "openai"; const provider = overrides?.provider ?? defaults?.provider ?? "openai";
const hasRemote = Boolean(defaults?.remote || overrides?.remote); const hasRemote = Boolean(defaults?.remote || overrides?.remote);
const remote = hasRemote const remote = hasRemote
@@ -81,6 +102,7 @@ function mergeConfig(
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath, modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir, modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
}; };
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
const vector = { const vector = {
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true, enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
extensionPath: extensionPath:
@@ -114,8 +136,12 @@ function mergeConfig(
const minScore = Math.max(0, Math.min(1, query.minScore)); const minScore = Math.max(0, Math.min(1, query.minScore));
return { return {
enabled, enabled,
sources,
provider, provider,
remote, remote,
experimental: {
sessionMemory,
},
fallback, fallback,
model, model,
local, local,

View File

@@ -2,6 +2,7 @@ import type { AgentMessage } from "@mariozechner/pi-agent-core";
import type { SessionManager } from "@mariozechner/pi-coding-agent"; import type { SessionManager } from "@mariozechner/pi-coding-agent";
import { makeMissingToolResult } from "./session-transcript-repair.js"; import { makeMissingToolResult } from "./session-transcript-repair.js";
import { emitSessionTranscriptUpdate } from "../sessions/transcript-events.js";
type ToolCall = { id: string; name?: string }; type ToolCall = { id: string; name?: string };
@@ -111,6 +112,12 @@ export function installSessionToolResultGuard(sessionManager: SessionManager): {
const result = originalAppend(sanitized as never); const result = originalAppend(sanitized as never);
const sessionFile = (sessionManager as { getSessionFile?: () => string | null })
.getSessionFile?.();
if (sessionFile) {
emitSessionTranscriptUpdate(sessionFile);
}
if (toolCalls.length > 0) { if (toolCalls.length > 0) {
for (const call of toolCalls) { for (const call of toolCalls) {
pending.set(call.id, call.name); pending.set(call.id, call.name);

View File

@@ -34,7 +34,7 @@ export function createMemorySearchTool(options: {
label: "Memory Search", label: "Memory Search",
name: "memory_search", name: "memory_search",
description: description:
"Mandatory recall step: semantically search MEMORY.md + memory/*.md before answering questions about prior work, decisions, dates, people, preferences, or todos; returns top snippets with path + lines.", "Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) before answering questions about prior work, decisions, dates, people, preferences, or todos; returns top snippets with path + lines.",
parameters: MemorySearchSchema, parameters: MemorySearchSchema,
execute: async (_toolCallId, params) => { execute: async (_toolCallId, params) => {
const query = readStringParam(params, "query", { required: true }); const query = readStringParam(params, "query", { required: true });

View File

@@ -51,6 +51,7 @@ export function registerMemoryCli(program: Command) {
`${chalk.bold.cyan("Memory Search")} (${agentId})`, `${chalk.bold.cyan("Memory Search")} (${agentId})`,
`Provider: ${status.provider} (requested: ${status.requestedProvider})`, `Provider: ${status.provider} (requested: ${status.requestedProvider})`,
status.fallback ? chalk.yellow(`Fallback: ${status.fallback.from}`) : null, status.fallback ? chalk.yellow(`Fallback: ${status.fallback.from}`) : null,
status.sources?.length ? `Sources: ${status.sources.join(", ")}` : null,
`Files: ${status.files}`, `Files: ${status.files}`,
`Chunks: ${status.chunks}`, `Chunks: ${status.chunks}`,
`Dirty: ${status.dirty ? "yes" : "no"}`, `Dirty: ${status.dirty ? "yes" : "no"}`,

View File

@@ -168,6 +168,9 @@ const FIELD_LABELS: Record<string, string> = {
"agents.defaults.bootstrapMaxChars": "Bootstrap Max Chars", "agents.defaults.bootstrapMaxChars": "Bootstrap Max Chars",
"agents.defaults.memorySearch": "Memory Search", "agents.defaults.memorySearch": "Memory Search",
"agents.defaults.memorySearch.enabled": "Enable Memory Search", "agents.defaults.memorySearch.enabled": "Enable Memory Search",
"agents.defaults.memorySearch.sources": "Memory Search Sources",
"agents.defaults.memorySearch.experimental.sessionMemory":
"Memory Search Session Index (Experimental)",
"agents.defaults.memorySearch.provider": "Memory Search Provider", "agents.defaults.memorySearch.provider": "Memory Search Provider",
"agents.defaults.memorySearch.remote.baseUrl": "Remote Embedding Base URL", "agents.defaults.memorySearch.remote.baseUrl": "Remote Embedding Base URL",
"agents.defaults.memorySearch.remote.apiKey": "Remote Embedding API Key", "agents.defaults.memorySearch.remote.apiKey": "Remote Embedding API Key",
@@ -353,6 +356,10 @@ const FIELD_HELP: Record<string, string> = {
"agents.defaults.models": "Configured model catalog (keys are full provider/model IDs).", "agents.defaults.models": "Configured model catalog (keys are full provider/model IDs).",
"agents.defaults.memorySearch": "agents.defaults.memorySearch":
"Vector search over MEMORY.md and memory/*.md (per-agent overrides supported).", "Vector search over MEMORY.md and memory/*.md (per-agent overrides supported).",
"agents.defaults.memorySearch.sources":
'Sources to index for memory search (default: ["memory"]; add "sessions" to include session transcripts).',
"agents.defaults.memorySearch.experimental.sessionMemory":
"Enable experimental session transcript indexing for memory search (default: false).",
"agents.defaults.memorySearch.provider": 'Embedding provider ("openai" or "local").', "agents.defaults.memorySearch.provider": 'Embedding provider ("openai" or "local").',
"agents.defaults.memorySearch.remote.baseUrl": "agents.defaults.memorySearch.remote.baseUrl":
"Custom OpenAI-compatible base URL (e.g. for Gemini/OpenRouter proxies).", "Custom OpenAI-compatible base URL (e.g. for Gemini/OpenRouter proxies).",

View File

@@ -6,6 +6,7 @@ import { CURRENT_SESSION_VERSION, SessionManager } from "@mariozechner/pi-coding
import type { SessionEntry } from "./types.js"; import type { SessionEntry } from "./types.js";
import { loadSessionStore, updateSessionStore } from "./store.js"; import { loadSessionStore, updateSessionStore } from "./store.js";
import { resolveDefaultSessionStorePath, resolveSessionTranscriptPath } from "./paths.js"; import { resolveDefaultSessionStorePath, resolveSessionTranscriptPath } from "./paths.js";
import { emitSessionTranscriptUpdate } from "../../sessions/transcript-events.js";
function stripQuery(value: string): string { function stripQuery(value: string): string {
const noHash = value.split("#")[0] ?? value; const noHash = value.split("#")[0] ?? value;
@@ -127,5 +128,6 @@ export async function appendAssistantMessageToSessionTranscript(params: {
}); });
} }
emitSessionTranscriptUpdate(sessionFile);
return { ok: true, sessionFile }; return { ok: true, sessionFile };
} }

View File

@@ -145,6 +145,13 @@ export type AgentToolsConfig = {
export type MemorySearchConfig = { export type MemorySearchConfig = {
/** Enable vector memory search (default: true). */ /** Enable vector memory search (default: true). */
enabled?: boolean; enabled?: boolean;
/** Sources to index and search (default: ["memory"]). */
sources?: Array<"memory" | "sessions">;
/** Experimental memory search settings. */
experimental?: {
/** Enable session transcript indexing (experimental, default: false). */
sessionMemory?: boolean;
};
/** Embedding provider mode. */ /** Embedding provider mode. */
provider?: "openai" | "local"; provider?: "openai" | "local";
remote?: { remote?: {

View File

@@ -194,6 +194,12 @@ export const AgentToolsSchema = z
export const MemorySearchSchema = z export const MemorySearchSchema = z
.object({ .object({
enabled: z.boolean().optional(), enabled: z.boolean().optional(),
sources: z.array(z.union([z.literal("memory"), z.literal("sessions")])).optional(),
experimental: z
.object({
sessionMemory: z.boolean().optional(),
})
.optional(),
provider: z.union([z.literal("openai"), z.literal("local")]).optional(), provider: z.union([z.literal("openai"), z.literal("local")]).optional(),
remote: z remote: z
.object({ .object({

View File

@@ -8,7 +8,9 @@ import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js"; import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
import { resolveMemorySearchConfig } from "../agents/memory-search.js"; import { resolveMemorySearchConfig } from "../agents/memory-search.js";
import type { ClawdbotConfig } from "../config/config.js"; import type { ClawdbotConfig } from "../config/config.js";
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
import { createSubsystemLogger } from "../logging.js"; import { createSubsystemLogger } from "../logging.js";
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
import { resolveUserPath, truncateUtf16Safe } from "../utils.js"; import { resolveUserPath, truncateUtf16Safe } from "../utils.js";
import { import {
createEmbeddingProvider, createEmbeddingProvider,
@@ -29,12 +31,15 @@ import {
} from "./internal.js"; } from "./internal.js";
import { requireNodeSqlite } from "./sqlite.js"; import { requireNodeSqlite } from "./sqlite.js";
type MemorySource = "memory" | "sessions";
export type MemorySearchResult = { export type MemorySearchResult = {
path: string; path: string;
startLine: number; startLine: number;
endLine: number; endLine: number;
score: number; score: number;
snippet: string; snippet: string;
source: MemorySource;
}; };
type MemoryIndexMeta = { type MemoryIndexMeta = {
@@ -45,9 +50,19 @@ type MemoryIndexMeta = {
vectorDims?: number; vectorDims?: number;
}; };
type SessionFileEntry = {
path: string;
absPath: string;
mtimeMs: number;
size: number;
hash: string;
content: string;
};
const META_KEY = "memory_index_meta_v1"; const META_KEY = "memory_index_meta_v1";
const SNIPPET_MAX_CHARS = 700; const SNIPPET_MAX_CHARS = 700;
const VECTOR_TABLE = "chunks_vec"; const VECTOR_TABLE = "chunks_vec";
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
const log = createSubsystemLogger("memory"); const log = createSubsystemLogger("memory");
@@ -66,6 +81,7 @@ export class MemoryIndexManager {
private readonly requestedProvider: "openai" | "local"; private readonly requestedProvider: "openai" | "local";
private readonly fallbackReason?: string; private readonly fallbackReason?: string;
private readonly db: DatabaseSync; private readonly db: DatabaseSync;
private readonly sources: Set<MemorySource>;
private readonly vector: { private readonly vector: {
enabled: boolean; enabled: boolean;
available: boolean | null; available: boolean | null;
@@ -76,9 +92,13 @@ export class MemoryIndexManager {
private vectorReady: Promise<boolean> | null = null; private vectorReady: Promise<boolean> | null = null;
private watcher: FSWatcher | null = null; private watcher: FSWatcher | null = null;
private watchTimer: NodeJS.Timeout | null = null; private watchTimer: NodeJS.Timeout | null = null;
private sessionWatchTimer: NodeJS.Timeout | null = null;
private sessionUnsubscribe: (() => void) | null = null;
private intervalTimer: NodeJS.Timeout | null = null; private intervalTimer: NodeJS.Timeout | null = null;
private closed = false; private closed = false;
private dirty = false; private dirty = false;
private sessionsDirty = false;
private sessionsDirtyFiles = new Set<string>();
private sessionWarm = new Set<string>(); private sessionWarm = new Set<string>();
private syncing: Promise<void> | null = null; private syncing: Promise<void> | null = null;
@@ -130,6 +150,7 @@ export class MemoryIndexManager {
this.provider = params.providerResult.provider; this.provider = params.providerResult.provider;
this.requestedProvider = params.providerResult.requestedProvider; this.requestedProvider = params.providerResult.requestedProvider;
this.fallbackReason = params.providerResult.fallbackReason; this.fallbackReason = params.providerResult.fallbackReason;
this.sources = new Set(params.settings.sources);
this.db = this.openDatabase(); this.db = this.openDatabase();
this.ensureSchema(); this.ensureSchema();
this.vector = { this.vector = {
@@ -142,8 +163,12 @@ export class MemoryIndexManager {
this.vector.dims = meta.vectorDims; this.vector.dims = meta.vectorDims;
} }
this.ensureWatcher(); this.ensureWatcher();
this.ensureSessionListener();
this.ensureIntervalSync(); this.ensureIntervalSync();
this.dirty = true; this.dirty = this.sources.has("memory");
if (this.sources.has("sessions")) {
this.sessionsDirty = true;
}
} }
async warmSession(sessionKey?: string): Promise<void> { async warmSession(sessionKey?: string): Promise<void> {
@@ -163,7 +188,7 @@ export class MemoryIndexManager {
}, },
): Promise<MemorySearchResult[]> { ): Promise<MemorySearchResult[]> {
await this.warmSession(opts?.sessionKey); await this.warmSession(opts?.sessionKey);
if (this.settings.sync.onSearch && this.dirty) { if (this.settings.sync.onSearch && (this.dirty || this.sessionsDirty)) {
await this.sync({ reason: "search" }); await this.sync({ reason: "search" });
} }
const cleaned = query.trim(); const cleaned = query.trim();
@@ -173,21 +198,29 @@ export class MemoryIndexManager {
const queryVec = await this.provider.embedQuery(cleaned); const queryVec = await this.provider.embedQuery(cleaned);
if (queryVec.length === 0) return []; if (queryVec.length === 0) return [];
if (await this.ensureVectorReady(queryVec.length)) { if (await this.ensureVectorReady(queryVec.length)) {
const sourceFilter = this.buildSourceFilter("c");
const rows = this.db const rows = this.db
.prepare( .prepare(
`SELECT c.path, c.start_line, c.end_line, c.text, `SELECT c.path, c.start_line, c.end_line, c.text,
c.source,
vec_distance_cosine(v.embedding, ?) AS dist vec_distance_cosine(v.embedding, ?) AS dist
FROM ${VECTOR_TABLE} v FROM ${VECTOR_TABLE} v
JOIN chunks c ON c.id = v.id JOIN chunks c ON c.id = v.id
WHERE c.model = ? WHERE c.model = ?${sourceFilter.sql}
ORDER BY dist ASC ORDER BY dist ASC
LIMIT ?`, LIMIT ?`,
) )
.all(vectorToBlob(queryVec), this.provider.model, maxResults) as Array<{ .all(
vectorToBlob(queryVec),
this.provider.model,
...sourceFilter.params,
maxResults,
) as Array<{
path: string; path: string;
start_line: number; start_line: number;
end_line: number; end_line: number;
text: string; text: string;
source: MemorySource;
dist: number; dist: number;
}>; }>;
return rows return rows
@@ -197,6 +230,7 @@ export class MemoryIndexManager {
endLine: row.end_line, endLine: row.end_line,
score: 1 - row.dist, score: 1 - row.dist,
snippet: truncateUtf16Safe(row.text, SNIPPET_MAX_CHARS), snippet: truncateUtf16Safe(row.text, SNIPPET_MAX_CHARS),
source: row.source,
})) }))
.filter((entry) => entry.score >= minScore); .filter((entry) => entry.score >= minScore);
} }
@@ -217,6 +251,7 @@ export class MemoryIndexManager {
endLine: entry.chunk.endLine, endLine: entry.chunk.endLine,
score: entry.score, score: entry.score,
snippet: truncateUtf16Safe(entry.chunk.text, SNIPPET_MAX_CHARS), snippet: truncateUtf16Safe(entry.chunk.text, SNIPPET_MAX_CHARS),
source: entry.chunk.source,
})); }));
} }
@@ -261,6 +296,7 @@ export class MemoryIndexManager {
provider: string; provider: string;
model: string; model: string;
requestedProvider: string; requestedProvider: string;
sources: MemorySource[];
fallback?: { from: string; reason?: string }; fallback?: { from: string; reason?: string };
vector?: { vector?: {
enabled: boolean; enabled: boolean;
@@ -270,10 +306,15 @@ export class MemoryIndexManager {
dims?: number; dims?: number;
}; };
} { } {
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as { const sourceFilter = this.buildSourceFilter();
const files = this.db
.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`)
.get(...sourceFilter.params) as {
c: number; c: number;
}; };
const chunks = this.db.prepare(`SELECT COUNT(*) as c FROM chunks`).get() as { const chunks = this.db
.prepare(`SELECT COUNT(*) as c FROM chunks WHERE 1=1${sourceFilter.sql}`)
.get(...sourceFilter.params) as {
c: number; c: number;
}; };
return { return {
@@ -285,6 +326,7 @@ export class MemoryIndexManager {
provider: this.provider.id, provider: this.provider.id,
model: this.provider.model, model: this.provider.model,
requestedProvider: this.requestedProvider, requestedProvider: this.requestedProvider,
sources: Array.from(this.sources),
fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined, fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined,
vector: { vector: {
enabled: this.vector.enabled, enabled: this.vector.enabled,
@@ -303,6 +345,10 @@ export class MemoryIndexManager {
clearTimeout(this.watchTimer); clearTimeout(this.watchTimer);
this.watchTimer = null; this.watchTimer = null;
} }
if (this.sessionWatchTimer) {
clearTimeout(this.sessionWatchTimer);
this.sessionWatchTimer = null;
}
if (this.intervalTimer) { if (this.intervalTimer) {
clearInterval(this.intervalTimer); clearInterval(this.intervalTimer);
this.intervalTimer = null; this.intervalTimer = null;
@@ -311,6 +357,10 @@ export class MemoryIndexManager {
await this.watcher.close(); await this.watcher.close();
this.watcher = null; this.watcher = null;
} }
if (this.sessionUnsubscribe) {
this.sessionUnsubscribe();
this.sessionUnsubscribe = null;
}
this.db.close(); this.db.close();
INDEX_CACHE.delete(this.cacheKey); INDEX_CACHE.delete(this.cacheKey);
} }
@@ -379,6 +429,22 @@ export class MemoryIndexManager {
} }
} }
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
const sources = Array.from(this.sources);
if (sources.length === 0) return { sql: "", params: [] };
const column = alias ? `${alias}.source` : "source";
const placeholders = sources.map(() => "?").join(", ");
return { sql: ` AND ${column} IN (${placeholders})`, params: sources };
}
private ensureColumn(table: "files" | "chunks", column: string, definition: string): void {
const rows = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{
name: string;
}>;
if (rows.some((row) => row.name === column)) return;
this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
}
private openDatabase(): DatabaseSync { private openDatabase(): DatabaseSync {
const dbPath = resolveUserPath(this.settings.store.path); const dbPath = resolveUserPath(this.settings.store.path);
const dir = path.dirname(dbPath); const dir = path.dirname(dbPath);
@@ -397,6 +463,7 @@ export class MemoryIndexManager {
this.db.exec(` this.db.exec(`
CREATE TABLE IF NOT EXISTS files ( CREATE TABLE IF NOT EXISTS files (
path TEXT PRIMARY KEY, path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL, hash TEXT NOT NULL,
mtime INTEGER NOT NULL, mtime INTEGER NOT NULL,
size INTEGER NOT NULL size INTEGER NOT NULL
@@ -406,6 +473,7 @@ export class MemoryIndexManager {
CREATE TABLE IF NOT EXISTS chunks ( CREATE TABLE IF NOT EXISTS chunks (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
path TEXT NOT NULL, path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL, start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL, end_line INTEGER NOT NULL,
hash TEXT NOT NULL, hash TEXT NOT NULL,
@@ -415,11 +483,14 @@ export class MemoryIndexManager {
updated_at INTEGER NOT NULL updated_at INTEGER NOT NULL
); );
`); `);
this.ensureColumn("files", "source", "TEXT NOT NULL DEFAULT 'memory'");
this.ensureColumn("chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`); this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
} }
private ensureWatcher() { private ensureWatcher() {
if (!this.settings.sync.watch || this.watcher) return; if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
const watchPaths = [ const watchPaths = [
path.join(this.workspaceDir, "MEMORY.md"), path.join(this.workspaceDir, "MEMORY.md"),
path.join(this.workspaceDir, "memory"), path.join(this.workspaceDir, "memory"),
@@ -440,6 +511,33 @@ export class MemoryIndexManager {
this.watcher.on("unlink", markDirty); this.watcher.on("unlink", markDirty);
} }
private ensureSessionListener() {
if (!this.sources.has("sessions") || this.sessionUnsubscribe) return;
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
if (this.closed) return;
const sessionFile = update.sessionFile;
if (!this.isSessionFileForAgent(sessionFile)) return;
this.scheduleSessionDirty(sessionFile);
});
}
private scheduleSessionDirty(sessionFile: string) {
this.sessionsDirtyFiles.add(sessionFile);
if (this.sessionWatchTimer) return;
this.sessionWatchTimer = setTimeout(() => {
this.sessionWatchTimer = null;
this.sessionsDirty = true;
}, SESSION_DIRTY_DEBOUNCE_MS);
}
private isSessionFileForAgent(sessionFile: string): boolean {
if (!sessionFile) return false;
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
const resolvedFile = path.resolve(sessionFile);
const resolvedDir = path.resolve(sessionsDir);
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
}
private ensureIntervalSync() { private ensureIntervalSync() {
const minutes = this.settings.sync.intervalMinutes; const minutes = this.settings.sync.intervalMinutes;
if (!minutes || minutes <= 0 || this.intervalTimer) return; if (!minutes || minutes <= 0 || this.intervalTimer) return;
@@ -452,7 +550,7 @@ export class MemoryIndexManager {
} }
private scheduleWatchSync() { private scheduleWatchSync() {
if (!this.settings.sync.watch) return; if (!this.sources.has("memory") || !this.settings.sync.watch) return;
if (this.watchTimer) clearTimeout(this.watchTimer); if (this.watchTimer) clearTimeout(this.watchTimer);
this.watchTimer = setTimeout(() => { this.watchTimer = setTimeout(() => {
this.watchTimer = null; this.watchTimer = null;
@@ -468,15 +566,22 @@ export class MemoryIndexManager {
endLine: number; endLine: number;
text: string; text: string;
embedding: number[]; embedding: number[];
source: MemorySource;
}> { }> {
const sourceFilter = this.buildSourceFilter();
const rows = this.db const rows = this.db
.prepare(`SELECT path, start_line, end_line, text, embedding FROM chunks WHERE model = ?`) .prepare(
.all(this.provider.model) as Array<{ `SELECT path, start_line, end_line, text, embedding, source
FROM chunks
WHERE model = ?${sourceFilter.sql}`,
)
.all(this.provider.model, ...sourceFilter.params) as Array<{
path: string; path: string;
start_line: number; start_line: number;
end_line: number; end_line: number;
text: string; text: string;
embedding: string; embedding: string;
source: MemorySource;
}>; }>;
return rows.map((row) => ({ return rows.map((row) => ({
path: row.path, path: row.path,
@@ -484,9 +589,77 @@ export class MemoryIndexManager {
endLine: row.end_line, endLine: row.end_line,
text: row.text, text: row.text,
embedding: parseEmbedding(row.embedding), embedding: parseEmbedding(row.embedding),
source: row.source,
})); }));
} }
private shouldSyncSessions(params?: { reason?: string; force?: boolean }, needsFullReindex = false) {
if (!this.sources.has("sessions")) return false;
if (params?.force) return true;
const reason = params?.reason;
if (reason === "session-start" || reason === "watch") return false;
return this.sessionsDirty || needsFullReindex;
}
private async syncMemoryFiles(params: { needsFullReindex: boolean }) {
const files = await listMemoryFiles(this.workspaceDir);
const fileEntries = await Promise.all(
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
);
const activePaths = new Set(fileEntries.map((entry) => entry.path));
for (const entry of fileEntries) {
const record = this.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "memory") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
continue;
}
await this.indexFile(entry, { source: "memory" });
}
const staleRows = this.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
}
}
private async syncSessionFiles(params: { needsFullReindex: boolean }) {
const files = await this.listSessionFiles();
const activePaths = new Set(files.map((file) => this.sessionPathForFile(file)));
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
for (const absPath of files) {
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) continue;
const entry = await this.buildSessionEntry(absPath);
if (!entry) continue;
const record = this.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "sessions") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
continue;
}
await this.indexFile(entry, { source: "sessions", content: entry.content });
}
const staleRows = this.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
this.db
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
this.db
.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
}
}
private async runSync(params?: { reason?: string; force?: boolean }) { private async runSync(params?: { reason?: string; force?: boolean }) {
const vectorReady = await this.ensureVectorReady(); const vectorReady = await this.ensureVectorReady();
const meta = this.readMeta(); const meta = this.readMeta();
@@ -502,29 +675,21 @@ export class MemoryIndexManager {
this.resetIndex(); this.resetIndex();
} }
const files = await listMemoryFiles(this.workspaceDir); const shouldSyncMemory =
const fileEntries = await Promise.all( this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty);
files.map(async (file) => buildFileEntry(file, this.workspaceDir)), const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
);
const activePaths = new Set(fileEntries.map((entry) => entry.path));
for (const entry of fileEntries) { if (shouldSyncMemory) {
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ?`).get(entry.path) as await this.syncMemoryFiles({ needsFullReindex });
| { hash: string } this.dirty = false;
| undefined;
if (!needsFullReindex && record?.hash === entry.hash) {
continue;
}
await this.indexFile(entry);
} }
const staleRows = this.db.prepare(`SELECT path FROM files`).all() as Array<{ if (shouldSyncSessions) {
path: string; await this.syncSessionFiles({ needsFullReindex });
}>; this.sessionsDirty = false;
for (const stale of staleRows) { this.sessionsDirtyFiles.clear();
if (activePaths.has(stale.path)) continue; } else if (needsFullReindex && this.sources.has("sessions")) {
this.db.prepare(`DELETE FROM files WHERE path = ?`).run(stale.path); this.sessionsDirty = true;
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path);
} }
const nextMeta: MemoryIndexMeta = { const nextMeta: MemoryIndexMeta = {
@@ -536,8 +701,9 @@ export class MemoryIndexManager {
if (this.vector.available && this.vector.dims) { if (this.vector.available && this.vector.dims) {
nextMeta.vectorDims = this.vector.dims; nextMeta.vectorDims = this.vector.dims;
} }
this.writeMeta(nextMeta); if (shouldSyncMemory || shouldSyncSessions || needsFullReindex) {
this.dirty = false; this.writeMeta(nextMeta);
}
} }
private resetIndex() { private resetIndex() {
@@ -545,6 +711,7 @@ export class MemoryIndexManager {
this.db.exec(`DELETE FROM chunks`); this.db.exec(`DELETE FROM chunks`);
this.dropVectorTable(); this.dropVectorTable();
this.vector.dims = undefined; this.vector.dims = undefined;
this.sessionsDirtyFiles.clear();
} }
private readMeta(): MemoryIndexMeta | null { private readMeta(): MemoryIndexMeta | null {
@@ -568,24 +735,115 @@ export class MemoryIndexManager {
.run(META_KEY, value); .run(META_KEY, value);
} }
private async indexFile(entry: MemoryFileEntry) { private async listSessionFiles(): Promise<string[]> {
const content = await fs.readFile(entry.absPath, "utf-8"); const dir = resolveSessionTranscriptsDirForAgent(this.agentId);
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
return entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".jsonl"))
.map((name) => path.join(dir, name));
} catch {
return [];
}
}
private sessionPathForFile(absPath: string): string {
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
}
private normalizeSessionText(value: string): string {
return value.replace(/\s*\n+\s*/g, " ").replace(/\s+/g, " ").trim();
}
private extractSessionText(content: unknown): string | null {
if (typeof content === "string") {
const normalized = this.normalizeSessionText(content);
return normalized ? normalized : null;
}
if (!Array.isArray(content)) return null;
const parts: string[] = [];
for (const block of content) {
if (!block || typeof block !== "object") continue;
const record = block as { type?: unknown; text?: unknown };
if (record.type !== "text" || typeof record.text !== "string") continue;
const normalized = this.normalizeSessionText(record.text);
if (normalized) parts.push(normalized);
}
if (parts.length === 0) return null;
return parts.join(" ");
}
private async buildSessionEntry(absPath: string): Promise<SessionFileEntry | null> {
try {
const stat = await fs.stat(absPath);
const raw = await fs.readFile(absPath, "utf-8");
const lines = raw.split("\n");
const collected: string[] = [];
for (const line of lines) {
if (!line.trim()) continue;
let record: unknown;
try {
record = JSON.parse(line);
} catch {
continue;
}
if (
!record ||
typeof record !== "object" ||
(record as { type?: unknown }).type !== "message"
) {
continue;
}
const message = (record as { message?: unknown }).message as
| { role?: unknown; content?: unknown }
| undefined;
if (!message || typeof message.role !== "string") continue;
if (message.role !== "user" && message.role !== "assistant") continue;
const text = this.extractSessionText(message.content);
if (!text) continue;
const label = message.role === "user" ? "User" : "Assistant";
collected.push(`${label}: ${text}`);
}
const content = collected.join("\n");
return {
path: this.sessionPathForFile(absPath),
absPath,
mtimeMs: stat.mtimeMs,
size: stat.size,
hash: hashText(content),
content,
};
} catch (err) {
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
return null;
}
}
private async indexFile(
entry: MemoryFileEntry | SessionFileEntry,
options: { source: MemorySource; content?: string },
) {
const content = options.content ?? (await fs.readFile(entry.absPath, "utf-8"));
const chunks = chunkMarkdown(content, this.settings.chunking); const chunks = chunkMarkdown(content, this.settings.chunking);
const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text)); const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text));
const sample = embeddings.find((embedding) => embedding.length > 0); const sample = embeddings.find((embedding) => embedding.length > 0);
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false; const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
const now = Date.now(); const now = Date.now();
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path); this.db
.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`)
.run(entry.path, options.source);
for (let i = 0; i < chunks.length; i++) { for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i]; const chunk = chunks[i];
const embedding = embeddings[i] ?? []; const embedding = embeddings[i] ?? [];
const id = hashText( const id = hashText(
`${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`, `${options.source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${this.provider.model}`,
); );
this.db this.db
.prepare( .prepare(
`INSERT INTO chunks (id, path, start_line, end_line, hash, model, text, embedding, updated_at) `INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET ON CONFLICT(id) DO UPDATE SET
hash=excluded.hash, hash=excluded.hash,
model=excluded.model, model=excluded.model,
@@ -596,6 +854,7 @@ export class MemoryIndexManager {
.run( .run(
id, id,
entry.path, entry.path,
options.source,
chunk.startLine, chunk.startLine,
chunk.endLine, chunk.endLine,
chunk.hash, chunk.hash,
@@ -612,9 +871,13 @@ export class MemoryIndexManager {
} }
this.db this.db
.prepare( .prepare(
`INSERT INTO files (path, hash, mtime, size) VALUES (?, ?, ?, ?) `INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET hash=excluded.hash, mtime=excluded.mtime, size=excluded.size`, ON CONFLICT(path) DO UPDATE SET
source=excluded.source,
hash=excluded.hash,
mtime=excluded.mtime,
size=excluded.size`,
) )
.run(entry.path, entry.hash, entry.mtimeMs, entry.size); .run(entry.path, options.source, entry.hash, entry.mtimeMs, entry.size);
} }
} }

View File

@@ -0,0 +1,23 @@
type SessionTranscriptUpdate = {
sessionFile: string;
};
type SessionTranscriptListener = (update: SessionTranscriptUpdate) => void;
const SESSION_TRANSCRIPT_LISTENERS = new Set<SessionTranscriptListener>();
export function onSessionTranscriptUpdate(listener: SessionTranscriptListener): () => void {
SESSION_TRANSCRIPT_LISTENERS.add(listener);
return () => {
SESSION_TRANSCRIPT_LISTENERS.delete(listener);
};
}
export function emitSessionTranscriptUpdate(sessionFile: string): void {
const trimmed = sessionFile.trim();
if (!trimmed) return;
const update = { sessionFile: trimmed };
for (const listener of SESSION_TRANSCRIPT_LISTENERS) {
listener(update);
}
}