refactor(memory): extract sync + status helpers

This commit is contained in:
Peter Steinberger
2026-01-18 07:02:14 +00:00
parent d3b15c6afa
commit c9c9516206
10 changed files with 532 additions and 321 deletions

View File

@@ -0,0 +1,16 @@
function normalizeHeaderName(name: string): string {
return name.trim().toLowerCase();
}
export function fingerprintHeaderNames(headers: Record<string, string> | undefined): string[] {
if (!headers) return [];
const out: string[] = [];
for (const key of Object.keys(headers)) {
const normalized = normalizeHeaderName(key);
if (!normalized) continue;
out.push(normalized);
}
out.sort((a, b) => a.localeCompare(b));
return out;
}

View File

@@ -0,0 +1,55 @@
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
import { hashText } from "./internal.js";
import { fingerprintHeaderNames } from "./headers-fingerprint.js";
export function computeMemoryManagerCacheKey(params: {
agentId: string;
workspaceDir: string;
settings: ResolvedMemorySearchConfig;
}): string {
const settings = params.settings;
const fingerprint = hashText(
JSON.stringify({
enabled: settings.enabled,
sources: [...settings.sources].sort((a, b) => a.localeCompare(b)),
provider: settings.provider,
model: settings.model,
fallback: settings.fallback,
local: {
modelPath: settings.local.modelPath,
modelCacheDir: settings.local.modelCacheDir,
},
remote: settings.remote
? {
baseUrl: settings.remote.baseUrl,
headerNames: fingerprintHeaderNames(settings.remote.headers),
batch: settings.remote.batch
? {
enabled: settings.remote.batch.enabled,
wait: settings.remote.batch.wait,
concurrency: settings.remote.batch.concurrency,
pollIntervalMs: settings.remote.batch.pollIntervalMs,
timeoutMinutes: settings.remote.batch.timeoutMinutes,
}
: undefined,
}
: undefined,
experimental: settings.experimental,
store: {
driver: settings.store.driver,
path: settings.store.path,
vector: {
enabled: settings.store.vector.enabled,
extensionPath: settings.store.vector.extensionPath,
},
},
chunking: settings.chunking,
sync: settings.sync,
query: settings.query,
cache: settings.cache,
}),
);
return `${params.agentId}:${params.workspaceDir}:${fingerprint}`;
}

View File

@@ -24,12 +24,10 @@ import {
runOpenAiEmbeddingBatches,
} from "./openai-batch.js";
import {
buildFileEntry,
chunkMarkdown,
ensureDir,
hashText,
isMemoryPath,
listMemoryFiles,
type MemoryChunk,
type MemoryFileEntry,
normalizeRelPath,
@@ -38,8 +36,13 @@ import {
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
import { searchKeyword, searchVector } from "./manager-search.js";
import { ensureMemoryIndexSchema } from "./memory-schema.js";
import { computeMemoryManagerCacheKey } from "./manager-cache-key.js";
import { computeEmbeddingProviderKey } from "./provider-key.js";
import { requireNodeSqlite } from "./sqlite.js";
import { loadSqliteVecExtension } from "./sqlite-vec.js";
import type { SessionFileEntry } from "./session-files.js";
import { syncMemoryFiles } from "./sync-memory-files.js";
import { syncSessionFiles } from "./sync-session-files.js";
type MemorySource = "memory" | "sessions";
@@ -61,15 +64,6 @@ type MemoryIndexMeta = {
vectorDims?: number;
};
type SessionFileEntry = {
path: string;
absPath: string;
mtimeMs: number;
size: number;
hash: string;
content: string;
};
type MemorySyncProgressUpdate = {
completed: number;
total: number;
@@ -157,7 +151,7 @@ export class MemoryIndexManager {
const settings = resolveMemorySearchConfig(cfg, agentId);
if (!settings) return null;
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
const key = computeMemoryManagerCacheKey({ agentId, workspaceDir, settings });
const existing = INDEX_CACHE.get(key);
if (existing) return existing;
const providerResult = await createEmbeddingProvider({
@@ -200,7 +194,13 @@ export class MemoryIndexManager {
this.openAi = params.providerResult.openAi;
this.sources = new Set(params.settings.sources);
this.db = this.openDatabase();
this.providerKey = this.computeProviderKey();
this.providerKey = computeEmbeddingProviderKey({
providerId: this.provider.id,
providerModel: this.provider.model,
openAi: this.openAi
? { baseUrl: this.openAi.baseUrl, model: this.openAi.model, headers: this.openAi.headers }
: undefined,
});
this.cache = {
enabled: params.settings.cache.enabled,
maxEntries: params.settings.cache.maxEntries,
@@ -714,170 +714,43 @@ export class MemoryIndexManager {
needsFullReindex: boolean;
progress?: MemorySyncProgressState;
}) {
const files = await listMemoryFiles(this.workspaceDir);
const fileEntries = await Promise.all(
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
);
log.debug("memory sync: indexing memory files", {
files: fileEntries.length,
await syncMemoryFiles({
workspaceDir: this.workspaceDir,
db: this.db,
needsFullReindex: params.needsFullReindex,
batch: this.batch.enabled,
progress: params.progress,
batchEnabled: this.batch.enabled,
concurrency: this.getIndexConcurrency(),
runWithConcurrency: this.runWithConcurrency.bind(this),
indexFile: async (entry) => await this.indexFile(entry, { source: "memory" }),
vectorTable: VECTOR_TABLE,
ftsTable: FTS_TABLE,
ftsEnabled: this.fts.enabled,
ftsAvailable: this.fts.available,
model: this.provider.model,
});
const activePaths = new Set(fileEntries.map((entry) => entry.path));
if (params.progress) {
params.progress.total += fileEntries.length;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
label: this.batch.enabled ? "Indexing memory files (batch)..." : "Indexing memory files…",
});
}
const tasks = fileEntries.map((entry) => async () => {
const record = this.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "memory") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
await this.indexFile(entry, { source: "memory" });
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
});
await this.runWithConcurrency(tasks, this.getIndexConcurrency());
const staleRows = this.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
try {
this.db
.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
)
.run(stale.path, "memory");
} catch {}
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
if (this.fts.enabled && this.fts.available) {
try {
this.db
.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`)
.run(stale.path, "memory", this.provider.model);
} catch {}
}
}
}
private async syncSessionFiles(params: {
needsFullReindex: boolean;
progress?: MemorySyncProgressState;
}) {
const files = await this.listSessionFiles();
const activePaths = new Set(files.map((file) => this.sessionPathForFile(file)));
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
log.debug("memory sync: indexing session files", {
files: files.length,
indexAll,
dirtyFiles: this.sessionsDirtyFiles.size,
batch: this.batch.enabled,
await syncSessionFiles({
agentId: this.agentId,
db: this.db,
needsFullReindex: params.needsFullReindex,
progress: params.progress,
batchEnabled: this.batch.enabled,
concurrency: this.getIndexConcurrency(),
runWithConcurrency: this.runWithConcurrency.bind(this),
indexFile: async (entry) => await this.indexFile(entry, { source: "sessions", content: entry.content }),
vectorTable: VECTOR_TABLE,
ftsTable: FTS_TABLE,
ftsEnabled: this.fts.enabled,
ftsAvailable: this.fts.available,
model: this.provider.model,
dirtyFiles: this.sessionsDirtyFiles,
});
if (params.progress) {
params.progress.total += files.length;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
label: this.batch.enabled ? "Indexing session files (batch)..." : "Indexing session files…",
});
}
const tasks = files.map((absPath) => async () => {
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
const entry = await this.buildSessionEntry(absPath);
if (!entry) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
const record = this.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "sessions") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
await this.indexFile(entry, { source: "sessions", content: entry.content });
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
});
await this.runWithConcurrency(tasks, this.getIndexConcurrency());
const staleRows = this.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
this.db
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
try {
this.db
.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
)
.run(stale.path, "sessions");
} catch {}
this.db
.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
if (this.fts.enabled && this.fts.available) {
try {
this.db
.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`)
.run(stale.path, "sessions", this.provider.model);
} catch {}
}
}
}
private createSyncProgress(
@@ -993,95 +866,6 @@ export class MemoryIndexManager {
.run(META_KEY, value);
}
private async listSessionFiles(): Promise<string[]> {
const dir = resolveSessionTranscriptsDirForAgent(this.agentId);
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
return entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".jsonl"))
.map((name) => path.join(dir, name));
} catch {
return [];
}
}
private sessionPathForFile(absPath: string): string {
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
}
private normalizeSessionText(value: string): string {
return value
.replace(/\s*\n+\s*/g, " ")
.replace(/\s+/g, " ")
.trim();
}
private extractSessionText(content: unknown): string | null {
if (typeof content === "string") {
const normalized = this.normalizeSessionText(content);
return normalized ? normalized : null;
}
if (!Array.isArray(content)) return null;
const parts: string[] = [];
for (const block of content) {
if (!block || typeof block !== "object") continue;
const record = block as { type?: unknown; text?: unknown };
if (record.type !== "text" || typeof record.text !== "string") continue;
const normalized = this.normalizeSessionText(record.text);
if (normalized) parts.push(normalized);
}
if (parts.length === 0) return null;
return parts.join(" ");
}
private async buildSessionEntry(absPath: string): Promise<SessionFileEntry | null> {
try {
const stat = await fs.stat(absPath);
const raw = await fs.readFile(absPath, "utf-8");
const lines = raw.split("\n");
const collected: string[] = [];
for (const line of lines) {
if (!line.trim()) continue;
let record: unknown;
try {
record = JSON.parse(line);
} catch {
continue;
}
if (
!record ||
typeof record !== "object" ||
(record as { type?: unknown }).type !== "message"
) {
continue;
}
const message = (record as { message?: unknown }).message as
| { role?: unknown; content?: unknown }
| undefined;
if (!message || typeof message.role !== "string") continue;
if (message.role !== "user" && message.role !== "assistant") continue;
const text = this.extractSessionText(message.content);
if (!text) continue;
const label = message.role === "user" ? "User" : "Assistant";
collected.push(`${label}: ${text}`);
}
const content = collected.join("\n");
return {
path: this.sessionPathForFile(absPath),
absPath,
mtimeMs: stat.mtimeMs,
size: stat.size,
hash: hashText(content),
content,
};
} catch (err) {
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
return null;
}
}
private estimateEmbeddingTokens(text: string): number {
if (!text) return 0;
return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
@@ -1233,24 +1017,6 @@ export class MemoryIndexManager {
return embeddings;
}
private computeProviderKey(): string {
if (this.provider.id === "openai" && this.openAi) {
const entries = Object.entries(this.openAi.headers)
.filter(([key]) => key.toLowerCase() !== "authorization")
.sort(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => [key, value]);
return hashText(
JSON.stringify({
provider: "openai",
baseUrl: this.openAi.baseUrl,
model: this.openAi.model,
headers: entries,
}),
);
}
return hashText(JSON.stringify({ provider: this.provider.id, model: this.provider.model }));
}
private async embedChunksWithBatch(
chunks: MemoryChunk[],
entry: MemoryFileEntry | SessionFileEntry,

View File

@@ -0,0 +1,22 @@
import { hashText } from "./internal.js";
import { fingerprintHeaderNames } from "./headers-fingerprint.js";
export function computeEmbeddingProviderKey(params: {
providerId: string;
providerModel: string;
openAi?: { baseUrl: string; model: string; headers: Record<string, string> };
}): string {
if (params.openAi) {
const headerNames = fingerprintHeaderNames(params.openAi.headers);
return hashText(
JSON.stringify({
provider: "openai",
baseUrl: params.openAi.baseUrl,
model: params.openAi.model,
headerNames,
}),
);
}
return hashText(JSON.stringify({ provider: params.providerId, model: params.providerModel }));
}

103
src/memory/session-files.ts Normal file
View File

@@ -0,0 +1,103 @@
import fs from "node:fs/promises";
import path from "node:path";
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
import { createSubsystemLogger } from "../logging.js";
import { hashText } from "./internal.js";
const log = createSubsystemLogger("memory");
export type SessionFileEntry = {
path: string;
absPath: string;
mtimeMs: number;
size: number;
hash: string;
content: string;
};
export async function listSessionFilesForAgent(agentId: string): Promise<string[]> {
const dir = resolveSessionTranscriptsDirForAgent(agentId);
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
return entries
.filter((entry) => entry.isFile())
.map((entry) => entry.name)
.filter((name) => name.endsWith(".jsonl"))
.map((name) => path.join(dir, name));
} catch {
return [];
}
}
export function sessionPathForFile(absPath: string): string {
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
}
function normalizeSessionText(value: string): string {
return value
.replace(/\s*\n+\s*/g, " ")
.replace(/\s+/g, " ")
.trim();
}
export function extractSessionText(content: unknown): string | null {
if (typeof content === "string") {
const normalized = normalizeSessionText(content);
return normalized ? normalized : null;
}
if (!Array.isArray(content)) return null;
const parts: string[] = [];
for (const block of content) {
if (!block || typeof block !== "object") continue;
const record = block as { type?: unknown; text?: unknown };
if (record.type !== "text" || typeof record.text !== "string") continue;
const normalized = normalizeSessionText(record.text);
if (normalized) parts.push(normalized);
}
if (parts.length === 0) return null;
return parts.join(" ");
}
export async function buildSessionEntry(absPath: string): Promise<SessionFileEntry | null> {
try {
const stat = await fs.stat(absPath);
const raw = await fs.readFile(absPath, "utf-8");
const lines = raw.split("\n");
const collected: string[] = [];
for (const line of lines) {
if (!line.trim()) continue;
let record: unknown;
try {
record = JSON.parse(line);
} catch {
continue;
}
if (!record || typeof record !== "object" || (record as { type?: unknown }).type !== "message") {
continue;
}
const message = (record as { message?: unknown }).message as
| { role?: unknown; content?: unknown }
| undefined;
if (!message || typeof message.role !== "string") continue;
if (message.role !== "user" && message.role !== "assistant") continue;
const text = extractSessionText(message.content);
if (!text) continue;
const label = message.role === "user" ? "User" : "Assistant";
collected.push(`${label}: ${text}`);
}
const content = collected.join("\n");
return {
path: sessionPathForFile(absPath),
absPath,
mtimeMs: stat.mtimeMs,
size: stat.size,
hash: hashText(content),
content,
};
} catch (err) {
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
return null;
}
}

View File

@@ -0,0 +1,34 @@
export type Tone = "ok" | "warn" | "muted";
export function resolveMemoryVectorState(vector: {
enabled: boolean;
available?: boolean;
}): { tone: Tone; state: "ready" | "unavailable" | "disabled" | "unknown" } {
if (vector.enabled === false) return { tone: "muted", state: "disabled" };
if (vector.available === true) return { tone: "ok", state: "ready" };
if (vector.available === false) return { tone: "warn", state: "unavailable" };
return { tone: "muted", state: "unknown" };
}
export function resolveMemoryFtsState(fts: {
enabled: boolean;
available: boolean;
}): { tone: Tone; state: "ready" | "unavailable" | "disabled" } {
if (fts.enabled === false) return { tone: "muted", state: "disabled" };
return fts.available ? { tone: "ok", state: "ready" } : { tone: "warn", state: "unavailable" };
}
export function resolveMemoryCacheSummary(cache: {
enabled: boolean;
entries?: number;
}): { tone: Tone; text: string } {
if (!cache.enabled) return { tone: "muted", text: "cache off" };
const suffix = typeof cache.entries === "number" ? ` (${cache.entries})` : "";
return { tone: "ok", text: `cache on${suffix}` };
}
export function resolveMemoryCacheState(cache: {
enabled: boolean;
}): { tone: Tone; state: "enabled" | "disabled" } {
return cache.enabled ? { tone: "ok", state: "enabled" } : { tone: "muted", state: "disabled" };
}

View File

@@ -0,0 +1,102 @@
import type { DatabaseSync } from "node:sqlite";
import { createSubsystemLogger } from "../logging.js";
import {
buildFileEntry,
listMemoryFiles,
type MemoryFileEntry,
} from "./internal.js";
const log = createSubsystemLogger("memory");
type ProgressState = {
completed: number;
total: number;
label?: string;
report: (update: { completed: number; total: number; label?: string }) => void;
};
export async function syncMemoryFiles(params: {
workspaceDir: string;
db: DatabaseSync;
needsFullReindex: boolean;
progress?: ProgressState;
batchEnabled: boolean;
concurrency: number;
runWithConcurrency: <T>(tasks: Array<() => Promise<T>>, concurrency: number) => Promise<T[]>;
indexFile: (entry: MemoryFileEntry) => Promise<void>;
vectorTable: string;
ftsTable: string;
ftsEnabled: boolean;
ftsAvailable: boolean;
model: string;
}) {
const files = await listMemoryFiles(params.workspaceDir);
const fileEntries = await Promise.all(files.map(async (file) => buildFileEntry(file, params.workspaceDir)));
log.debug("memory sync: indexing memory files", {
files: fileEntries.length,
needsFullReindex: params.needsFullReindex,
batch: params.batchEnabled,
concurrency: params.concurrency,
});
const activePaths = new Set(fileEntries.map((entry) => entry.path));
if (params.progress) {
params.progress.total += fileEntries.length;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
label: params.batchEnabled ? "Indexing memory files (batch)..." : "Indexing memory files…",
});
}
const tasks = fileEntries.map((entry) => async () => {
const record = params.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "memory") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
await params.indexFile(entry);
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
});
await params.runWithConcurrency(tasks, params.concurrency);
const staleRows = params.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
params.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
try {
params.db
.prepare(
`DELETE FROM ${params.vectorTable} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
)
.run(stale.path, "memory");
} catch {}
params.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
if (params.ftsEnabled && params.ftsAvailable) {
try {
params.db
.prepare(`DELETE FROM ${params.ftsTable} WHERE path = ? AND source = ? AND model = ?`)
.run(stale.path, "memory", params.model);
} catch {}
}
}
}

View File

@@ -0,0 +1,126 @@
import type { DatabaseSync } from "node:sqlite";
import { createSubsystemLogger } from "../logging.js";
import type { SessionFileEntry } from "./session-files.js";
import { buildSessionEntry, listSessionFilesForAgent, sessionPathForFile } from "./session-files.js";
const log = createSubsystemLogger("memory");
type ProgressState = {
completed: number;
total: number;
label?: string;
report: (update: { completed: number; total: number; label?: string }) => void;
};
export async function syncSessionFiles(params: {
agentId: string;
db: DatabaseSync;
needsFullReindex: boolean;
progress?: ProgressState;
batchEnabled: boolean;
concurrency: number;
runWithConcurrency: <T>(tasks: Array<() => Promise<T>>, concurrency: number) => Promise<T[]>;
indexFile: (entry: SessionFileEntry) => Promise<void>;
vectorTable: string;
ftsTable: string;
ftsEnabled: boolean;
ftsAvailable: boolean;
model: string;
dirtyFiles: Set<string>;
}) {
const files = await listSessionFilesForAgent(params.agentId);
const activePaths = new Set(files.map((file) => sessionPathForFile(file)));
const indexAll = params.needsFullReindex || params.dirtyFiles.size === 0;
log.debug("memory sync: indexing session files", {
files: files.length,
indexAll,
dirtyFiles: params.dirtyFiles.size,
batch: params.batchEnabled,
concurrency: params.concurrency,
});
if (params.progress) {
params.progress.total += files.length;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
label: params.batchEnabled ? "Indexing session files (batch)..." : "Indexing session files…",
});
}
const tasks = files.map((absPath) => async () => {
if (!indexAll && !params.dirtyFiles.has(absPath)) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
const entry = await buildSessionEntry(absPath);
if (!entry) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
const record = params.db
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
.get(entry.path, "sessions") as { hash: string } | undefined;
if (!params.needsFullReindex && record?.hash === entry.hash) {
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
return;
}
await params.indexFile(entry);
if (params.progress) {
params.progress.completed += 1;
params.progress.report({
completed: params.progress.completed,
total: params.progress.total,
});
}
});
await params.runWithConcurrency(tasks, params.concurrency);
const staleRows = params.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>;
for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue;
params.db
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
try {
params.db
.prepare(
`DELETE FROM ${params.vectorTable} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
)
.run(stale.path, "sessions");
} catch {}
params.db
.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`)
.run(stale.path, "sessions");
if (params.ftsEnabled && params.ftsAvailable) {
try {
params.db
.prepare(`DELETE FROM ${params.ftsTable} WHERE path = ? AND source = ? AND model = ?`)
.run(stale.path, "sessions", params.model);
} catch {}
}
}
}