fix: report memory index progress
This commit is contained in:
@@ -167,7 +167,9 @@ describe("memory cli", () => {
|
|||||||
registerMemoryCli(program);
|
registerMemoryCli(program);
|
||||||
await program.parseAsync(["memory", "status", "--index"], { from: "user" });
|
await program.parseAsync(["memory", "status", "--index"], { from: "user" });
|
||||||
|
|
||||||
expect(sync).toHaveBeenCalledWith({ reason: "cli" });
|
expect(sync).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ reason: "cli", progress: expect.any(Function) }),
|
||||||
|
);
|
||||||
expect(probeEmbeddingAvailability).toHaveBeenCalled();
|
expect(probeEmbeddingAvailability).toHaveBeenCalled();
|
||||||
expect(close).toHaveBeenCalled();
|
expect(close).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import type { Command } from "commander";
|
|||||||
|
|
||||||
import { resolveDefaultAgentId } from "../agents/agent-scope.js";
|
import { resolveDefaultAgentId } from "../agents/agent-scope.js";
|
||||||
import { loadConfig } from "../config/config.js";
|
import { loadConfig } from "../config/config.js";
|
||||||
import { withProgress } from "./progress.js";
|
import { withProgress, withProgressTotals } from "./progress.js";
|
||||||
import { getMemorySearchManager } from "../memory/index.js";
|
import { getMemorySearchManager } from "../memory/index.js";
|
||||||
import { defaultRuntime } from "../runtime.js";
|
import { defaultRuntime } from "../runtime.js";
|
||||||
import { formatDocsLink } from "../terminal/links.js";
|
import { formatDocsLink } from "../terminal/links.js";
|
||||||
@@ -51,26 +51,38 @@ export function registerMemoryCli(program: Command) {
|
|||||||
let embeddingProbe: Awaited<ReturnType<typeof manager.probeEmbeddingAvailability>> | undefined;
|
let embeddingProbe: Awaited<ReturnType<typeof manager.probeEmbeddingAvailability>> | undefined;
|
||||||
let indexError: string | undefined;
|
let indexError: string | undefined;
|
||||||
if (deep) {
|
if (deep) {
|
||||||
const total = opts.index ? 3 : 2;
|
await withProgress({ label: "Checking memory…", total: 2 }, async (progress) => {
|
||||||
await withProgress({ label: "Checking memory…", total }, async (progress) => {
|
|
||||||
progress.setLabel("Probing vector…");
|
progress.setLabel("Probing vector…");
|
||||||
await manager.probeVectorAvailability();
|
await manager.probeVectorAvailability();
|
||||||
progress.tick();
|
progress.tick();
|
||||||
progress.setLabel("Probing embeddings…");
|
progress.setLabel("Probing embeddings…");
|
||||||
embeddingProbe = await manager.probeEmbeddingAvailability();
|
embeddingProbe = await manager.probeEmbeddingAvailability();
|
||||||
progress.tick();
|
progress.tick();
|
||||||
if (opts.index) {
|
|
||||||
progress.setLabel("Indexing memory…");
|
|
||||||
try {
|
|
||||||
await manager.sync({ reason: "cli" });
|
|
||||||
} catch (err) {
|
|
||||||
indexError = err instanceof Error ? err.message : String(err);
|
|
||||||
defaultRuntime.error(`Memory index failed: ${indexError}`);
|
|
||||||
process.exitCode = 1;
|
|
||||||
}
|
|
||||||
progress.tick();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
if (opts.index) {
|
||||||
|
await withProgressTotals(
|
||||||
|
{ label: "Indexing memory…", total: 0 },
|
||||||
|
async (update, progress) => {
|
||||||
|
try {
|
||||||
|
await manager.sync({
|
||||||
|
reason: "cli",
|
||||||
|
progress: (syncUpdate) => {
|
||||||
|
update({
|
||||||
|
completed: syncUpdate.completed,
|
||||||
|
total: syncUpdate.total,
|
||||||
|
label: syncUpdate.label,
|
||||||
|
});
|
||||||
|
if (syncUpdate.label) progress.setLabel(syncUpdate.label);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
indexError = err instanceof Error ? err.message : String(err);
|
||||||
|
defaultRuntime.error(`Memory index failed: ${indexError}`);
|
||||||
|
process.exitCode = 1;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
await manager.probeVectorAvailability();
|
await manager.probeVectorAvailability();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -109,4 +109,44 @@ describe("memory embedding batches", () => {
|
|||||||
|
|
||||||
expect(embedBatch.mock.calls.length).toBe(1);
|
expect(embedBatch.mock.calls.length).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("reports sync progress totals", async () => {
|
||||||
|
const line = "c".repeat(120);
|
||||||
|
const content = Array.from({ length: 20 }, () => line).join("\n");
|
||||||
|
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-05.md"), content);
|
||||||
|
|
||||||
|
const cfg = {
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
workspace: workspaceDir,
|
||||||
|
memorySearch: {
|
||||||
|
provider: "openai",
|
||||||
|
model: "mock-embed",
|
||||||
|
store: { path: indexPath },
|
||||||
|
chunking: { tokens: 200, overlap: 0 },
|
||||||
|
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||||
|
query: { minScore: 0 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
list: [{ id: "main", default: true }],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||||
|
expect(result.manager).not.toBeNull();
|
||||||
|
if (!result.manager) throw new Error("manager missing");
|
||||||
|
manager = result.manager;
|
||||||
|
const updates: Array<{ completed: number; total: number; label?: string }> = [];
|
||||||
|
await manager.sync({
|
||||||
|
force: true,
|
||||||
|
progress: (update) => {
|
||||||
|
updates.push(update);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(updates.length).toBeGreaterThan(0);
|
||||||
|
const last = updates[updates.length - 1];
|
||||||
|
expect(last?.total).toBeGreaterThan(0);
|
||||||
|
expect(last?.completed).toBe(last?.total);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -60,12 +60,24 @@ type SessionFileEntry = {
|
|||||||
content: string;
|
content: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type MemorySyncProgressUpdate = {
|
||||||
|
completed: number;
|
||||||
|
total: number;
|
||||||
|
label?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type MemorySyncProgressState = {
|
||||||
|
completed: number;
|
||||||
|
total: number;
|
||||||
|
report: (update: MemorySyncProgressUpdate) => void;
|
||||||
|
};
|
||||||
|
|
||||||
const META_KEY = "memory_index_meta_v1";
|
const META_KEY = "memory_index_meta_v1";
|
||||||
const SNIPPET_MAX_CHARS = 700;
|
const SNIPPET_MAX_CHARS = 700;
|
||||||
const VECTOR_TABLE = "chunks_vec";
|
const VECTOR_TABLE = "chunks_vec";
|
||||||
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
|
const SESSION_DIRTY_DEBOUNCE_MS = 5000;
|
||||||
const EMBEDDING_BATCH_MAX_TOKENS = 8000;
|
const EMBEDDING_BATCH_MAX_TOKENS = 8000;
|
||||||
const EMBEDDING_APPROX_CHARS_PER_TOKEN = 2;
|
const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1;
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
|
|
||||||
@@ -258,7 +270,11 @@ export class MemoryIndexManager {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async sync(params?: { reason?: string; force?: boolean }): Promise<void> {
|
async sync(params?: {
|
||||||
|
reason?: string;
|
||||||
|
force?: boolean;
|
||||||
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
|
}): Promise<void> {
|
||||||
if (this.syncing) return this.syncing;
|
if (this.syncing) return this.syncing;
|
||||||
this.syncing = this.runSync(params).finally(() => {
|
this.syncing = this.runSync(params).finally(() => {
|
||||||
this.syncing = null;
|
this.syncing = null;
|
||||||
@@ -650,21 +666,46 @@ export class MemoryIndexManager {
|
|||||||
return this.sessionsDirty || needsFullReindex;
|
return this.sessionsDirty || needsFullReindex;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async syncMemoryFiles(params: { needsFullReindex: boolean }) {
|
private async syncMemoryFiles(params: {
|
||||||
|
needsFullReindex: boolean;
|
||||||
|
progress?: MemorySyncProgressState;
|
||||||
|
}) {
|
||||||
const files = await listMemoryFiles(this.workspaceDir);
|
const files = await listMemoryFiles(this.workspaceDir);
|
||||||
const fileEntries = await Promise.all(
|
const fileEntries = await Promise.all(
|
||||||
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
||||||
);
|
);
|
||||||
const activePaths = new Set(fileEntries.map((entry) => entry.path));
|
const activePaths = new Set(fileEntries.map((entry) => entry.path));
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.total += fileEntries.length;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
label: "Indexing memory files…",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
for (const entry of fileEntries) {
|
for (const entry of fileEntries) {
|
||||||
const record = this.db
|
const record = this.db
|
||||||
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
.get(entry.path, "memory") as { hash: string } | undefined;
|
.get(entry.path, "memory") as { hash: string } | undefined;
|
||||||
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
await this.indexFile(entry, { source: "memory" });
|
await this.indexFile(entry, { source: "memory" });
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const staleRows = this.db
|
const staleRows = this.db
|
||||||
@@ -677,22 +718,65 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async syncSessionFiles(params: { needsFullReindex: boolean }) {
|
private async syncSessionFiles(params: {
|
||||||
|
needsFullReindex: boolean;
|
||||||
|
progress?: MemorySyncProgressState;
|
||||||
|
}) {
|
||||||
const files = await this.listSessionFiles();
|
const files = await this.listSessionFiles();
|
||||||
const activePaths = new Set(files.map((file) => this.sessionPathForFile(file)));
|
const activePaths = new Set(files.map((file) => this.sessionPathForFile(file)));
|
||||||
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.total += files.length;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
label: "Indexing session files…",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
for (const absPath of files) {
|
for (const absPath of files) {
|
||||||
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) continue;
|
if (!indexAll && !this.sessionsDirtyFiles.has(absPath)) {
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const entry = await this.buildSessionEntry(absPath);
|
const entry = await this.buildSessionEntry(absPath);
|
||||||
if (!entry) continue;
|
if (!entry) {
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const record = this.db
|
const record = this.db
|
||||||
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`)
|
||||||
.get(entry.path, "sessions") as { hash: string } | undefined;
|
.get(entry.path, "sessions") as { hash: string } | undefined;
|
||||||
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
if (!params.needsFullReindex && record?.hash === entry.hash) {
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
await this.indexFile(entry, { source: "sessions", content: entry.content });
|
await this.indexFile(entry, { source: "sessions", content: entry.content });
|
||||||
|
if (params.progress) {
|
||||||
|
params.progress.completed += 1;
|
||||||
|
params.progress.report({
|
||||||
|
completed: params.progress.completed,
|
||||||
|
total: params.progress.total,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const staleRows = this.db
|
const staleRows = this.db
|
||||||
@@ -709,7 +793,14 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async runSync(params?: { reason?: string; force?: boolean }) {
|
private async runSync(params?: {
|
||||||
|
reason?: string;
|
||||||
|
force?: boolean;
|
||||||
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
|
}) {
|
||||||
|
const progress: MemorySyncProgressState | null = params?.progress
|
||||||
|
? { completed: 0, total: 0, report: params.progress }
|
||||||
|
: null;
|
||||||
const vectorReady = await this.ensureVectorReady();
|
const vectorReady = await this.ensureVectorReady();
|
||||||
const meta = this.readMeta();
|
const meta = this.readMeta();
|
||||||
const needsFullReindex =
|
const needsFullReindex =
|
||||||
@@ -729,12 +820,12 @@ export class MemoryIndexManager {
|
|||||||
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
|
||||||
|
|
||||||
if (shouldSyncMemory) {
|
if (shouldSyncMemory) {
|
||||||
await this.syncMemoryFiles({ needsFullReindex });
|
await this.syncMemoryFiles({ needsFullReindex, progress: progress ?? undefined });
|
||||||
this.dirty = false;
|
this.dirty = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (shouldSyncSessions) {
|
if (shouldSyncSessions) {
|
||||||
await this.syncSessionFiles({ needsFullReindex });
|
await this.syncSessionFiles({ needsFullReindex, progress: progress ?? undefined });
|
||||||
this.sessionsDirty = false;
|
this.sessionsDirty = false;
|
||||||
this.sessionsDirtyFiles.clear();
|
this.sessionsDirtyFiles.clear();
|
||||||
} else if (needsFullReindex && this.sources.has("sessions")) {
|
} else if (needsFullReindex && this.sources.has("sessions")) {
|
||||||
|
|||||||
Reference in New Issue
Block a user