let5see/clawdbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add sqlite-vec memory search acceleration

Browse Source
This commit is contained in:
Peter Steinberger
2026-01-17 18:02:25 +00:00
parent 252dfbcd40
commit 5a08471dcd
13 changed files with 432 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/agents/memory-search.test.ts
View File

@@ -29,6 +29,12 @@ describe("memory search config", () => {
memorySearch: {
provider: "openai",
model: "text-embedding-3-small",
store: {
vector: {
enabled: false,
extensionPath: "/opt/sqlite-vec.dylib",
},
},
chunking: { tokens: 500, overlap: 100 },
query: { maxResults: 4, minScore: 0.2 },
},
@@ -40,6 +46,11 @@ describe("memory search config", () => {
memorySearch: {
chunking: { tokens: 320 },
query: { maxResults: 8 },
store: {
vector: {
enabled: true,
},
},
},
},
],
@@ -52,6 +63,8 @@ describe("memory search config", () => {
expect(resolved?.chunking.overlap).toBe(100);
expect(resolved?.query.maxResults).toBe(8);
expect(resolved?.query.minScore).toBe(0.2);
expect(resolved?.store.vector.enabled).toBe(true);
expect(resolved?.store.vector.extensionPath).toBe("/opt/sqlite-vec.dylib");
});
it("merges remote defaults with agent overrides", () => {

10
src/agents/memory-search.ts
View File

@@ -23,6 +23,10 @@ export type ResolvedMemorySearchConfig = {
store: {
driver: "sqlite";
path: string;
vector: {
enabled: boolean;
extensionPath?: string;
};
};
chunking: {
tokens: number;
@@ -77,9 +81,15 @@ function mergeConfig(
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
};
const vector = {
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
extensionPath:
overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath,
};
const store = {
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
vector,
};
const chunking = {
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,

95
src/cli/memory-cli.test.ts Normal file
View File

@@ -0,0 +1,95 @@
import { Command } from "commander";
import { afterEach, describe, expect, it, vi } from "vitest";
const getMemorySearchManager = vi.fn();
const loadConfig = vi.fn(() => ({}));
const resolveDefaultAgentId = vi.fn(() => "main");
vi.mock("../memory/index.js", () => ({
getMemorySearchManager,
}));
vi.mock("../config/config.js", () => ({
loadConfig,
}));
vi.mock("../agents/agent-scope.js", () => ({
resolveDefaultAgentId,
}));
afterEach(() => {
vi.restoreAllMocks();
getMemorySearchManager.mockReset();
});
describe("memory cli", () => {
it("prints vector status when available", async () => {
const { registerMemoryCli } = await import("./memory-cli.js");
const { defaultRuntime } = await import("../runtime.js");
getMemorySearchManager.mockResolvedValueOnce({
manager: {
status: () => ({
files: 2,
chunks: 5,
dirty: false,
workspaceDir: "/tmp/clawd",
dbPath: "/tmp/memory.sqlite",
provider: "openai",
model: "text-embedding-3-small",
requestedProvider: "openai",
vector: {
enabled: true,
available: true,
extensionPath: "/opt/sqlite-vec.dylib",
dims: 1024,
},
}),
},
});
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
const program = new Command();
program.name("test");
registerMemoryCli(program);
await program.parseAsync(["memory", "status"], { from: "user" });
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: ready"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector dims: 1024"));
expect(log).toHaveBeenCalledWith(
expect.stringContaining("Vector path: /opt/sqlite-vec.dylib"),
);
});
it("prints vector error when unavailable", async () => {
const { registerMemoryCli } = await import("./memory-cli.js");
const { defaultRuntime } = await import("../runtime.js");
getMemorySearchManager.mockResolvedValueOnce({
manager: {
status: () => ({
files: 0,
chunks: 0,
dirty: true,
workspaceDir: "/tmp/clawd",
dbPath: "/tmp/memory.sqlite",
provider: "openai",
model: "text-embedding-3-small",
requestedProvider: "openai",
vector: {
enabled: true,
available: false,
loadError: "load failed",
},
}),
},
});
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
const program = new Command();
program.name("test");
registerMemoryCli(program);
await program.parseAsync(["memory", "status", "--agent", "main"], { from: "user" });
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: unavailable"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector error: load failed"));
});
});

17
src/cli/memory-cli.ts
View File

@@ -56,6 +56,23 @@ export function registerMemoryCli(program: Command) {
`Dirty: ${status.dirty ? "yes" : "no"}`,
`Index: ${status.dbPath}`,
].filter(Boolean) as string[];
if (status.vector) {
const vectorState = status.vector.enabled
? status.vector.available
? "ready"
: "unavailable"
: "disabled";
lines.push(`Vector: ${vectorState}`);
if (status.vector.dims) {
lines.push(`Vector dims: ${status.vector.dims}`);
}
if (status.vector.extensionPath) {
lines.push(`Vector path: ${status.vector.extensionPath}`);
}
if (status.vector.loadError) {
lines.push(chalk.yellow(`Vector error: ${status.vector.loadError}`));
}
}
if (status.fallback?.reason) {
lines.push(chalk.gray(status.fallback.reason));
}

9
src/config/schema.ts
View File

@@ -176,6 +176,9 @@ const FIELD_LABELS: Record<string, string> = {
"agents.defaults.memorySearch.fallback": "Memory Search Fallback",
"agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path",
"agents.defaults.memorySearch.store.path": "Memory Search Index Path",
"agents.defaults.memorySearch.store.vector.enabled": "Memory Search Vector Index",
"agents.defaults.memorySearch.store.vector.extensionPath":
"Memory Search Vector Extension Path",
"agents.defaults.memorySearch.chunking.tokens": "Memory Chunk Tokens",
"agents.defaults.memorySearch.chunking.overlap": "Memory Chunk Overlap Tokens",
"agents.defaults.memorySearch.sync.onSessionStart": "Index on Session Start",
@@ -362,7 +365,11 @@ const FIELD_HELP: Record<string, string> = {
"agents.defaults.memorySearch.fallback":
'Fallback to OpenAI when local embeddings fail ("openai" or "none").',
"agents.defaults.memorySearch.store.path":
"SQLite index path (default: ~/.clawdbot/memory/{agentId}.sqlite).",
"SQLite index path (default: ~/.clawdbot/state/memory/{agentId}.sqlite).",
"agents.defaults.memorySearch.store.vector.enabled":
"Enable sqlite-vec extension for vector search (default: true).",
"agents.defaults.memorySearch.store.vector.extensionPath":
"Optional override path to sqlite-vec extension library (.dylib/.so/.dll).",
"agents.defaults.memorySearch.sync.onSearch":
"Lazy sync: reindex on first search after a change.",
"agents.defaults.memorySearch.sync.watch": "Watch memory files for changes (chokidar).",

6
src/config/types.tools.ts
View File

@@ -167,6 +167,12 @@ export type MemorySearchConfig = {
store?: {
driver?: "sqlite";
path?: string;
vector?: {
/** Enable sqlite-vec extension for vector search (default: true). */
enabled?: boolean;
/** Optional override path to sqlite-vec extension (.dylib/.so/.dll). */
extensionPath?: string;
};
};
/** Chunking configuration. */
chunking?: {

6
src/config/zod-schema.agent-runtime.ts
View File

@@ -214,6 +214,12 @@ export const MemorySearchSchema = z
.object({
driver: z.literal("sqlite").optional(),
path: z.string().optional(),
vector: z
.object({
enabled: z.boolean().optional(),
extensionPath: z.string().optional(),
})
.optional(),
})
.optional(),
chunking: z

155
src/memory/manager.ts
View File

@@ -42,15 +42,20 @@ type MemoryIndexMeta = {
provider: string;
chunkTokens: number;
chunkOverlap: number;
vectorDims?: number;
};
const META_KEY = "memory_index_meta_v1";
const SNIPPET_MAX_CHARS = 700;
const VECTOR_TABLE = "chunks_vec";
const log = createSubsystemLogger("memory");
const INDEX_CACHE = new Map<string, MemoryIndexManager>();
const vectorToBlob = (embedding: number[]): Buffer =>
Buffer.from(new Float32Array(embedding).buffer);
export class MemoryIndexManager {
private readonly cacheKey: string;
private readonly cfg: ClawdbotConfig;
@@ -61,6 +66,14 @@ export class MemoryIndexManager {
private readonly requestedProvider: "openai" | "local";
private readonly fallbackReason?: string;
private readonly db: DatabaseSync;
private readonly vector: {
enabled: boolean;
available: boolean | null;
extensionPath?: string;
loadError?: string;
dims?: number;
};
private vectorReady: Promise<boolean> | null = null;
private watcher: FSWatcher | null = null;
private watchTimer: NodeJS.Timeout | null = null;
private intervalTimer: NodeJS.Timeout | null = null;
@@ -119,6 +132,15 @@ export class MemoryIndexManager {
this.fallbackReason = params.providerResult.fallbackReason;
this.db = this.openDatabase();
this.ensureSchema();
this.vector = {
enabled: params.settings.store.vector.enabled,
available: null,
extensionPath: params.settings.store.vector.extensionPath,
};
const meta = this.readMeta();
if (meta?.vectorDims) {
this.vector.dims = meta.vectorDims;
}
this.ensureWatcher();
this.ensureIntervalSync();
this.dirty = true;
@@ -146,8 +168,38 @@ export class MemoryIndexManager {
}
const cleaned = query.trim();
if (!cleaned) return [];
const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
const queryVec = await this.provider.embedQuery(cleaned);
if (queryVec.length === 0) return [];
if (await this.ensureVectorReady(queryVec.length)) {
const rows = this.db
.prepare(
`SELECT c.path, c.start_line, c.end_line, c.text,
vec_distance_cosine(v.embedding, ?) AS dist
FROM ${VECTOR_TABLE} v
JOIN chunks c ON c.id = v.id
WHERE c.model = ?
ORDER BY dist ASC
LIMIT ?`,
)
.all(vectorToBlob(queryVec), this.provider.model, maxResults) as Array<{
path: string;
start_line: number;
end_line: number;
text: string;
dist: number;
}>;
return rows
.map((row) => ({
path: row.path,
startLine: row.start_line,
endLine: row.end_line,
score: 1 - row.dist,
snippet: truncateUtf16Safe(row.text, SNIPPET_MAX_CHARS),
}))
.filter((entry) => entry.score >= minScore);
}
const candidates = this.listChunks();
const scored = candidates
.map((chunk) => ({
@@ -155,8 +207,6 @@ export class MemoryIndexManager {
score: cosineSimilarity(queryVec, chunk.embedding),
}))
.filter((entry) => Number.isFinite(entry.score));
const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
return scored
.filter((entry) => entry.score >= minScore)
.sort((a, b) => b.score - a.score)
@@ -212,6 +262,13 @@ export class MemoryIndexManager {
model: string;
requestedProvider: string;
fallback?: { from: string; reason?: string };
vector?: {
enabled: boolean;
available?: boolean;
extensionPath?: string;
loadError?: string;
dims?: number;
};
} {
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as {
c: number;
@@ -229,6 +286,13 @@ export class MemoryIndexManager {
model: this.provider.model,
requestedProvider: this.requestedProvider,
fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined,
vector: {
enabled: this.vector.enabled,
available: this.vector.available ?? undefined,
extensionPath: this.vector.extensionPath,
loadError: this.vector.loadError,
dims: this.vector.dims,
},
};
}
@@ -251,12 +315,76 @@ export class MemoryIndexManager {
INDEX_CACHE.delete(this.cacheKey);
}
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
if (!this.vector.enabled) return false;
if (!this.vectorReady) {
this.vectorReady = this.loadVectorExtension();
}
const ready = await this.vectorReady;
if (ready && typeof dimensions === "number" && dimensions > 0) {
this.ensureVectorTable(dimensions);
}
return ready;
}
private async loadVectorExtension(): Promise<boolean> {
if (this.vector.available !== null) return this.vector.available;
if (!this.vector.enabled) {
this.vector.available = false;
return false;
}
try {
const sqliteVec = await import("sqlite-vec");
const extensionPath = this.vector.extensionPath?.trim()
? resolveUserPath(this.vector.extensionPath)
: sqliteVec.getLoadablePath();
this.db.enableLoadExtension(true);
if (this.vector.extensionPath?.trim()) {
this.db.loadExtension(extensionPath);
} else {
sqliteVec.load(this.db);
}
this.vector.extensionPath = extensionPath;
this.vector.available = true;
return true;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.vector.available = false;
this.vector.loadError = message;
log.warn(`sqlite-vec unavailable: ${message}`);
return false;
}
}
private ensureVectorTable(dimensions: number): void {
if (this.vector.dims === dimensions) return;
if (this.vector.dims && this.vector.dims !== dimensions) {
this.dropVectorTable();
}
this.db.exec(
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
` id TEXT PRIMARY KEY,\n` +
` embedding FLOAT[${dimensions}]\n` +
`)`,
);
this.vector.dims = dimensions;
}
private dropVectorTable(): void {
try {
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
}
}
private openDatabase(): DatabaseSync {
const dbPath = resolveUserPath(this.settings.store.path);
const dir = path.dirname(dbPath);
ensureDir(dir);
const { DatabaseSync } = requireNodeSqlite();
return new DatabaseSync(dbPath);
return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled });
}
private ensureSchema() {
@@ -360,6 +488,7 @@ export class MemoryIndexManager {
}
private async runSync(params?: { reason?: string; force?: boolean }) {
const vectorReady = await this.ensureVectorReady();
const meta = this.readMeta();
const needsFullReindex =
params?.force ||
@@ -367,7 +496,8 @@ export class MemoryIndexManager {
meta.model !== this.provider.model ||
meta.provider !== this.provider.id ||
meta.chunkTokens !== this.settings.chunking.tokens ||
meta.chunkOverlap !== this.settings.chunking.overlap;
meta.chunkOverlap !== this.settings.chunking.overlap ||
(vectorReady && !meta?.vectorDims);
if (needsFullReindex) {
this.resetIndex();
}
@@ -397,18 +527,24 @@ export class MemoryIndexManager {
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path);
}
this.writeMeta({
const nextMeta: MemoryIndexMeta = {
model: this.provider.model,
provider: this.provider.id,
chunkTokens: this.settings.chunking.tokens,
chunkOverlap: this.settings.chunking.overlap,
});
};
if (this.vector.available && this.vector.dims) {
nextMeta.vectorDims = this.vector.dims;
}
this.writeMeta(nextMeta);
this.dirty = false;
}
private resetIndex() {
this.db.exec(`DELETE FROM files`);
this.db.exec(`DELETE FROM chunks`);
this.dropVectorTable();
this.vector.dims = undefined;
}
private readMeta(): MemoryIndexMeta | null {
@@ -436,6 +572,8 @@ export class MemoryIndexManager {
const content = await fs.readFile(entry.absPath, "utf-8");
const chunks = chunkMarkdown(content, this.settings.chunking);
const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text));
const sample = embeddings.find((embedding) => embedding.length > 0);
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
const now = Date.now();
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path);
for (let i = 0; i < chunks.length; i++) {
@@ -466,6 +604,11 @@ export class MemoryIndexManager {
JSON.stringify(embedding),
now,
);
if (vectorReady && embedding.length > 0) {
this.db
.prepare(`INSERT OR REPLACE INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`)
.run(id, vectorToBlob(embedding));
}
}
this.db
.prepare(