feat: add sqlite-vec memory search acceleration

This commit is contained in:
Peter Steinberger
2026-01-17 18:02:25 +00:00
parent 252dfbcd40
commit 5a08471dcd
13 changed files with 432 additions and 7 deletions

View File

@@ -49,6 +49,7 @@ Docs: https://docs.clawd.bot
- Status: trim `/status` to current-provider usage only and drop the OAuth/token block. - Status: trim `/status` to current-provider usage only and drop the OAuth/token block.
- Directory: unify `clawdbot directory` across channels and plugin channels. - Directory: unify `clawdbot directory` across channels and plugin channels.
- UI: allow deleting sessions from the Control UI. - UI: allow deleting sessions from the Control UI.
- Memory: add sqlite-vec vector acceleration with CLI status details.
- Skills: add user-invocable skill commands and expanded skill command registration. - Skills: add user-invocable skill commands and expanded skill command registration.
- Telegram: default reaction level to minimal and enable reaction notifications by default. - Telegram: default reaction level to minimal and enable reaction notifications by default.
- Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2. - Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2.

View File

@@ -78,6 +78,7 @@ Defaults:
- Watches memory files for changes (debounced). - Watches memory files for changes (debounced).
- Uses remote embeddings (OpenAI) unless configured for local. - Uses remote embeddings (OpenAI) unless configured for local.
- Local mode uses node-llama-cpp and may require `pnpm approve-builds`. - Local mode uses node-llama-cpp and may require `pnpm approve-builds`.
- Uses sqlite-vec (when available) to accelerate vector search inside SQLite.
Remote embeddings **require** an API key for the embedding provider. By default Remote embeddings **require** an API key for the embedding provider. By default
this is OpenAI (`OPENAI_API_KEY` or `models.providers.openai.apiKey`). Codex this is OpenAI (`OPENAI_API_KEY` or `models.providers.openai.apiKey`). Codex
@@ -143,6 +144,37 @@ Local mode:
- Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token). - Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
- Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change. - Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change.
### SQLite vector acceleration (sqlite-vec)
When the sqlite-vec extension is available, Clawdbot stores embeddings in a
SQLite virtual table (`vec0`) and performs vector distance queries in the
database. This keeps search fast without loading every embedding into JS.
Configuration (optional):
```json5
agents: {
defaults: {
memorySearch: {
store: {
vector: {
enabled: true,
extensionPath: "/path/to/sqlite-vec"
}
}
}
}
}
```
Notes:
- `enabled` defaults to true; when disabled, search falls back to in-process
cosine similarity over stored embeddings.
- If the sqlite-vec extension is missing or fails to load, Clawdbot logs the
error and continues with the JS fallback (no vector table).
- `extensionPath` overrides the bundled sqlite-vec path (useful for custom builds
or non-standard install locations).
### Local embedding auto-download ### Local embedding auto-download
- Default local embedding model: `hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf` (~0.6 GB). - Default local embedding model: `hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf` (~0.6 GB).

View File

@@ -174,6 +174,7 @@
"proper-lockfile": "^4.1.2", "proper-lockfile": "^4.1.2",
"qrcode-terminal": "^0.12.0", "qrcode-terminal": "^0.12.0",
"sharp": "^0.34.5", "sharp": "^0.34.5",
"sqlite-vec": "0.1.7-alpha.2",
"tar": "^7.5.3", "tar": "^7.5.3",
"tslog": "^4.10.2", "tslog": "^4.10.2",
"undici": "^7.18.2", "undici": "^7.18.2",

54
pnpm-lock.yaml generated
View File

@@ -133,6 +133,9 @@ importers:
sharp: sharp:
specifier: ^0.34.5 specifier: ^0.34.5
version: 0.34.5 version: 0.34.5
sqlite-vec:
specifier: 0.1.7-alpha.2
version: 0.1.7-alpha.2
tar: tar:
specifier: 7.5.3 specifier: 7.5.3
version: 7.5.3 version: 7.5.3
@@ -3910,6 +3913,34 @@ packages:
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
engines: {node: '>= 10.x'} engines: {node: '>= 10.x'}
sqlite-vec-darwin-arm64@0.1.7-alpha.2:
resolution: {integrity: sha512-raIATOqFYkeCHhb/t3r7W7Cf2lVYdf4J3ogJ6GFc8PQEgHCPEsi+bYnm2JT84MzLfTlSTIdxr4/NKv+zF7oLPw==}
cpu: [arm64]
os: [darwin]
sqlite-vec-darwin-x64@0.1.7-alpha.2:
resolution: {integrity: sha512-jeZEELsQjjRsVojsvU5iKxOvkaVuE+JYC8Y4Ma8U45aAERrDYmqZoHvgSG7cg1PXL3bMlumFTAmHynf1y4pOzA==}
cpu: [x64]
os: [darwin]
sqlite-vec-linux-arm64@0.1.7-alpha.2:
resolution: {integrity: sha512-6Spj4Nfi7tG13jsUG+W7jnT0bCTWbyPImu2M8nWp20fNrd1SZ4g3CSlDAK8GBdavX7wRlbBHCZ+BDa++rbDewA==}
cpu: [arm64]
os: [linux]
sqlite-vec-linux-x64@0.1.7-alpha.2:
resolution: {integrity: sha512-IcgrbHaDccTVhXDf8Orwdc2+hgDLAFORl6OBUhcvlmwswwBP1hqBTSEhovClG4NItwTOBNgpwOoQ7Qp3VDPWLg==}
cpu: [x64]
os: [linux]
sqlite-vec-windows-x64@0.1.7-alpha.2:
resolution: {integrity: sha512-TRP6hTjAcwvQ6xpCZvjP00pdlda8J38ArFy1lMYhtQWXiIBmWnhMaMbq4kaeCYwvTTddfidatRS+TJrwIKB/oQ==}
cpu: [x64]
os: [win32]
sqlite-vec@0.1.7-alpha.2:
resolution: {integrity: sha512-rNgRCv+4V4Ed3yc33Qr+nNmjhtrMnnHzXfLVPeGb28Dx5mmDL3Ngw/Wk8vhCGjj76+oC6gnkmMG8y73BZWGBwQ==}
stackback@0.0.2: stackback@0.0.2:
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
@@ -8563,6 +8594,29 @@ snapshots:
split2@4.2.0: {} split2@4.2.0: {}
sqlite-vec-darwin-arm64@0.1.7-alpha.2:
optional: true
sqlite-vec-darwin-x64@0.1.7-alpha.2:
optional: true
sqlite-vec-linux-arm64@0.1.7-alpha.2:
optional: true
sqlite-vec-linux-x64@0.1.7-alpha.2:
optional: true
sqlite-vec-windows-x64@0.1.7-alpha.2:
optional: true
sqlite-vec@0.1.7-alpha.2:
optionalDependencies:
sqlite-vec-darwin-arm64: 0.1.7-alpha.2
sqlite-vec-darwin-x64: 0.1.7-alpha.2
sqlite-vec-linux-arm64: 0.1.7-alpha.2
sqlite-vec-linux-x64: 0.1.7-alpha.2
sqlite-vec-windows-x64: 0.1.7-alpha.2
stackback@0.0.2: {} stackback@0.0.2: {}
statuses@2.0.2: {} statuses@2.0.2: {}

View File

@@ -0,0 +1,40 @@
import { DatabaseSync } from "node:sqlite";
import { load, getLoadablePath } from "sqlite-vec";
function vec(values) {
return Buffer.from(new Float32Array(values).buffer);
}
const db = new DatabaseSync(":memory:", { allowExtension: true });
try {
load(db);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
console.error("sqlite-vec load failed:");
console.error(message);
console.error("expected extension path:", getLoadablePath());
process.exit(1);
}
db.exec(`
CREATE VIRTUAL TABLE v USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[4]
);
`);
const insert = db.prepare("INSERT INTO v (id, embedding) VALUES (?, ?)");
insert.run("a", vec([1, 0, 0, 0]));
insert.run("b", vec([0, 1, 0, 0]));
insert.run("c", vec([0.2, 0.2, 0, 0]));
const query = vec([1, 0, 0, 0]);
const rows = db
.prepare(
"SELECT id, vec_distance_cosine(embedding, ?) AS dist FROM v ORDER BY dist ASC"
)
.all(query);
console.log("sqlite-vec ok");
console.log(rows);

View File

@@ -29,6 +29,12 @@ describe("memory search config", () => {
memorySearch: { memorySearch: {
provider: "openai", provider: "openai",
model: "text-embedding-3-small", model: "text-embedding-3-small",
store: {
vector: {
enabled: false,
extensionPath: "/opt/sqlite-vec.dylib",
},
},
chunking: { tokens: 500, overlap: 100 }, chunking: { tokens: 500, overlap: 100 },
query: { maxResults: 4, minScore: 0.2 }, query: { maxResults: 4, minScore: 0.2 },
}, },
@@ -40,6 +46,11 @@ describe("memory search config", () => {
memorySearch: { memorySearch: {
chunking: { tokens: 320 }, chunking: { tokens: 320 },
query: { maxResults: 8 }, query: { maxResults: 8 },
store: {
vector: {
enabled: true,
},
},
}, },
}, },
], ],
@@ -52,6 +63,8 @@ describe("memory search config", () => {
expect(resolved?.chunking.overlap).toBe(100); expect(resolved?.chunking.overlap).toBe(100);
expect(resolved?.query.maxResults).toBe(8); expect(resolved?.query.maxResults).toBe(8);
expect(resolved?.query.minScore).toBe(0.2); expect(resolved?.query.minScore).toBe(0.2);
expect(resolved?.store.vector.enabled).toBe(true);
expect(resolved?.store.vector.extensionPath).toBe("/opt/sqlite-vec.dylib");
}); });
it("merges remote defaults with agent overrides", () => { it("merges remote defaults with agent overrides", () => {

View File

@@ -23,6 +23,10 @@ export type ResolvedMemorySearchConfig = {
store: { store: {
driver: "sqlite"; driver: "sqlite";
path: string; path: string;
vector: {
enabled: boolean;
extensionPath?: string;
};
}; };
chunking: { chunking: {
tokens: number; tokens: number;
@@ -77,9 +81,15 @@ function mergeConfig(
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath, modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir, modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
}; };
const vector = {
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
extensionPath:
overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath,
};
const store = { const store = {
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite", driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path), path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
vector,
}; };
const chunking = { const chunking = {
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS, tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,

View File

@@ -0,0 +1,95 @@
import { Command } from "commander";
import { afterEach, describe, expect, it, vi } from "vitest";
const getMemorySearchManager = vi.fn();
const loadConfig = vi.fn(() => ({}));
const resolveDefaultAgentId = vi.fn(() => "main");
vi.mock("../memory/index.js", () => ({
getMemorySearchManager,
}));
vi.mock("../config/config.js", () => ({
loadConfig,
}));
vi.mock("../agents/agent-scope.js", () => ({
resolveDefaultAgentId,
}));
afterEach(() => {
vi.restoreAllMocks();
getMemorySearchManager.mockReset();
});
describe("memory cli", () => {
it("prints vector status when available", async () => {
const { registerMemoryCli } = await import("./memory-cli.js");
const { defaultRuntime } = await import("../runtime.js");
getMemorySearchManager.mockResolvedValueOnce({
manager: {
status: () => ({
files: 2,
chunks: 5,
dirty: false,
workspaceDir: "/tmp/clawd",
dbPath: "/tmp/memory.sqlite",
provider: "openai",
model: "text-embedding-3-small",
requestedProvider: "openai",
vector: {
enabled: true,
available: true,
extensionPath: "/opt/sqlite-vec.dylib",
dims: 1024,
},
}),
},
});
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
const program = new Command();
program.name("test");
registerMemoryCli(program);
await program.parseAsync(["memory", "status"], { from: "user" });
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: ready"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector dims: 1024"));
expect(log).toHaveBeenCalledWith(
expect.stringContaining("Vector path: /opt/sqlite-vec.dylib"),
);
});
it("prints vector error when unavailable", async () => {
const { registerMemoryCli } = await import("./memory-cli.js");
const { defaultRuntime } = await import("../runtime.js");
getMemorySearchManager.mockResolvedValueOnce({
manager: {
status: () => ({
files: 0,
chunks: 0,
dirty: true,
workspaceDir: "/tmp/clawd",
dbPath: "/tmp/memory.sqlite",
provider: "openai",
model: "text-embedding-3-small",
requestedProvider: "openai",
vector: {
enabled: true,
available: false,
loadError: "load failed",
},
}),
},
});
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
const program = new Command();
program.name("test");
registerMemoryCli(program);
await program.parseAsync(["memory", "status", "--agent", "main"], { from: "user" });
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: unavailable"));
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector error: load failed"));
});
});

View File

@@ -56,6 +56,23 @@ export function registerMemoryCli(program: Command) {
`Dirty: ${status.dirty ? "yes" : "no"}`, `Dirty: ${status.dirty ? "yes" : "no"}`,
`Index: ${status.dbPath}`, `Index: ${status.dbPath}`,
].filter(Boolean) as string[]; ].filter(Boolean) as string[];
if (status.vector) {
const vectorState = status.vector.enabled
? status.vector.available
? "ready"
: "unavailable"
: "disabled";
lines.push(`Vector: ${vectorState}`);
if (status.vector.dims) {
lines.push(`Vector dims: ${status.vector.dims}`);
}
if (status.vector.extensionPath) {
lines.push(`Vector path: ${status.vector.extensionPath}`);
}
if (status.vector.loadError) {
lines.push(chalk.yellow(`Vector error: ${status.vector.loadError}`));
}
}
if (status.fallback?.reason) { if (status.fallback?.reason) {
lines.push(chalk.gray(status.fallback.reason)); lines.push(chalk.gray(status.fallback.reason));
} }

View File

@@ -176,6 +176,9 @@ const FIELD_LABELS: Record<string, string> = {
"agents.defaults.memorySearch.fallback": "Memory Search Fallback", "agents.defaults.memorySearch.fallback": "Memory Search Fallback",
"agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path", "agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path",
"agents.defaults.memorySearch.store.path": "Memory Search Index Path", "agents.defaults.memorySearch.store.path": "Memory Search Index Path",
"agents.defaults.memorySearch.store.vector.enabled": "Memory Search Vector Index",
"agents.defaults.memorySearch.store.vector.extensionPath":
"Memory Search Vector Extension Path",
"agents.defaults.memorySearch.chunking.tokens": "Memory Chunk Tokens", "agents.defaults.memorySearch.chunking.tokens": "Memory Chunk Tokens",
"agents.defaults.memorySearch.chunking.overlap": "Memory Chunk Overlap Tokens", "agents.defaults.memorySearch.chunking.overlap": "Memory Chunk Overlap Tokens",
"agents.defaults.memorySearch.sync.onSessionStart": "Index on Session Start", "agents.defaults.memorySearch.sync.onSessionStart": "Index on Session Start",
@@ -362,7 +365,11 @@ const FIELD_HELP: Record<string, string> = {
"agents.defaults.memorySearch.fallback": "agents.defaults.memorySearch.fallback":
'Fallback to OpenAI when local embeddings fail ("openai" or "none").', 'Fallback to OpenAI when local embeddings fail ("openai" or "none").',
"agents.defaults.memorySearch.store.path": "agents.defaults.memorySearch.store.path":
"SQLite index path (default: ~/.clawdbot/memory/{agentId}.sqlite).", "SQLite index path (default: ~/.clawdbot/state/memory/{agentId}.sqlite).",
"agents.defaults.memorySearch.store.vector.enabled":
"Enable sqlite-vec extension for vector search (default: true).",
"agents.defaults.memorySearch.store.vector.extensionPath":
"Optional override path to sqlite-vec extension library (.dylib/.so/.dll).",
"agents.defaults.memorySearch.sync.onSearch": "agents.defaults.memorySearch.sync.onSearch":
"Lazy sync: reindex on first search after a change.", "Lazy sync: reindex on first search after a change.",
"agents.defaults.memorySearch.sync.watch": "Watch memory files for changes (chokidar).", "agents.defaults.memorySearch.sync.watch": "Watch memory files for changes (chokidar).",

View File

@@ -167,6 +167,12 @@ export type MemorySearchConfig = {
store?: { store?: {
driver?: "sqlite"; driver?: "sqlite";
path?: string; path?: string;
vector?: {
/** Enable sqlite-vec extension for vector search (default: true). */
enabled?: boolean;
/** Optional override path to sqlite-vec extension (.dylib/.so/.dll). */
extensionPath?: string;
};
}; };
/** Chunking configuration. */ /** Chunking configuration. */
chunking?: { chunking?: {

View File

@@ -214,6 +214,12 @@ export const MemorySearchSchema = z
.object({ .object({
driver: z.literal("sqlite").optional(), driver: z.literal("sqlite").optional(),
path: z.string().optional(), path: z.string().optional(),
vector: z
.object({
enabled: z.boolean().optional(),
extensionPath: z.string().optional(),
})
.optional(),
}) })
.optional(), .optional(),
chunking: z chunking: z

View File

@@ -42,15 +42,20 @@ type MemoryIndexMeta = {
provider: string; provider: string;
chunkTokens: number; chunkTokens: number;
chunkOverlap: number; chunkOverlap: number;
vectorDims?: number;
}; };
const META_KEY = "memory_index_meta_v1"; const META_KEY = "memory_index_meta_v1";
const SNIPPET_MAX_CHARS = 700; const SNIPPET_MAX_CHARS = 700;
const VECTOR_TABLE = "chunks_vec";
const log = createSubsystemLogger("memory"); const log = createSubsystemLogger("memory");
const INDEX_CACHE = new Map<string, MemoryIndexManager>(); const INDEX_CACHE = new Map<string, MemoryIndexManager>();
const vectorToBlob = (embedding: number[]): Buffer =>
Buffer.from(new Float32Array(embedding).buffer);
export class MemoryIndexManager { export class MemoryIndexManager {
private readonly cacheKey: string; private readonly cacheKey: string;
private readonly cfg: ClawdbotConfig; private readonly cfg: ClawdbotConfig;
@@ -61,6 +66,14 @@ export class MemoryIndexManager {
private readonly requestedProvider: "openai" | "local"; private readonly requestedProvider: "openai" | "local";
private readonly fallbackReason?: string; private readonly fallbackReason?: string;
private readonly db: DatabaseSync; private readonly db: DatabaseSync;
private readonly vector: {
enabled: boolean;
available: boolean | null;
extensionPath?: string;
loadError?: string;
dims?: number;
};
private vectorReady: Promise<boolean> | null = null;
private watcher: FSWatcher | null = null; private watcher: FSWatcher | null = null;
private watchTimer: NodeJS.Timeout | null = null; private watchTimer: NodeJS.Timeout | null = null;
private intervalTimer: NodeJS.Timeout | null = null; private intervalTimer: NodeJS.Timeout | null = null;
@@ -119,6 +132,15 @@ export class MemoryIndexManager {
this.fallbackReason = params.providerResult.fallbackReason; this.fallbackReason = params.providerResult.fallbackReason;
this.db = this.openDatabase(); this.db = this.openDatabase();
this.ensureSchema(); this.ensureSchema();
this.vector = {
enabled: params.settings.store.vector.enabled,
available: null,
extensionPath: params.settings.store.vector.extensionPath,
};
const meta = this.readMeta();
if (meta?.vectorDims) {
this.vector.dims = meta.vectorDims;
}
this.ensureWatcher(); this.ensureWatcher();
this.ensureIntervalSync(); this.ensureIntervalSync();
this.dirty = true; this.dirty = true;
@@ -146,8 +168,38 @@ export class MemoryIndexManager {
} }
const cleaned = query.trim(); const cleaned = query.trim();
if (!cleaned) return []; if (!cleaned) return [];
const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
const queryVec = await this.provider.embedQuery(cleaned); const queryVec = await this.provider.embedQuery(cleaned);
if (queryVec.length === 0) return []; if (queryVec.length === 0) return [];
if (await this.ensureVectorReady(queryVec.length)) {
const rows = this.db
.prepare(
`SELECT c.path, c.start_line, c.end_line, c.text,
vec_distance_cosine(v.embedding, ?) AS dist
FROM ${VECTOR_TABLE} v
JOIN chunks c ON c.id = v.id
WHERE c.model = ?
ORDER BY dist ASC
LIMIT ?`,
)
.all(vectorToBlob(queryVec), this.provider.model, maxResults) as Array<{
path: string;
start_line: number;
end_line: number;
text: string;
dist: number;
}>;
return rows
.map((row) => ({
path: row.path,
startLine: row.start_line,
endLine: row.end_line,
score: 1 - row.dist,
snippet: truncateUtf16Safe(row.text, SNIPPET_MAX_CHARS),
}))
.filter((entry) => entry.score >= minScore);
}
const candidates = this.listChunks(); const candidates = this.listChunks();
const scored = candidates const scored = candidates
.map((chunk) => ({ .map((chunk) => ({
@@ -155,8 +207,6 @@ export class MemoryIndexManager {
score: cosineSimilarity(queryVec, chunk.embedding), score: cosineSimilarity(queryVec, chunk.embedding),
})) }))
.filter((entry) => Number.isFinite(entry.score)); .filter((entry) => Number.isFinite(entry.score));
const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
return scored return scored
.filter((entry) => entry.score >= minScore) .filter((entry) => entry.score >= minScore)
.sort((a, b) => b.score - a.score) .sort((a, b) => b.score - a.score)
@@ -212,6 +262,13 @@ export class MemoryIndexManager {
model: string; model: string;
requestedProvider: string; requestedProvider: string;
fallback?: { from: string; reason?: string }; fallback?: { from: string; reason?: string };
vector?: {
enabled: boolean;
available?: boolean;
extensionPath?: string;
loadError?: string;
dims?: number;
};
} { } {
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as { const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as {
c: number; c: number;
@@ -229,6 +286,13 @@ export class MemoryIndexManager {
model: this.provider.model, model: this.provider.model,
requestedProvider: this.requestedProvider, requestedProvider: this.requestedProvider,
fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined, fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined,
vector: {
enabled: this.vector.enabled,
available: this.vector.available ?? undefined,
extensionPath: this.vector.extensionPath,
loadError: this.vector.loadError,
dims: this.vector.dims,
},
}; };
} }
@@ -251,12 +315,76 @@ export class MemoryIndexManager {
INDEX_CACHE.delete(this.cacheKey); INDEX_CACHE.delete(this.cacheKey);
} }
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
if (!this.vector.enabled) return false;
if (!this.vectorReady) {
this.vectorReady = this.loadVectorExtension();
}
const ready = await this.vectorReady;
if (ready && typeof dimensions === "number" && dimensions > 0) {
this.ensureVectorTable(dimensions);
}
return ready;
}
private async loadVectorExtension(): Promise<boolean> {
if (this.vector.available !== null) return this.vector.available;
if (!this.vector.enabled) {
this.vector.available = false;
return false;
}
try {
const sqliteVec = await import("sqlite-vec");
const extensionPath = this.vector.extensionPath?.trim()
? resolveUserPath(this.vector.extensionPath)
: sqliteVec.getLoadablePath();
this.db.enableLoadExtension(true);
if (this.vector.extensionPath?.trim()) {
this.db.loadExtension(extensionPath);
} else {
sqliteVec.load(this.db);
}
this.vector.extensionPath = extensionPath;
this.vector.available = true;
return true;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
this.vector.available = false;
this.vector.loadError = message;
log.warn(`sqlite-vec unavailable: ${message}`);
return false;
}
}
private ensureVectorTable(dimensions: number): void {
if (this.vector.dims === dimensions) return;
if (this.vector.dims && this.vector.dims !== dimensions) {
this.dropVectorTable();
}
this.db.exec(
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
` id TEXT PRIMARY KEY,\n` +
` embedding FLOAT[${dimensions}]\n` +
`)`,
);
this.vector.dims = dimensions;
}
private dropVectorTable(): void {
try {
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
}
}
private openDatabase(): DatabaseSync { private openDatabase(): DatabaseSync {
const dbPath = resolveUserPath(this.settings.store.path); const dbPath = resolveUserPath(this.settings.store.path);
const dir = path.dirname(dbPath); const dir = path.dirname(dbPath);
ensureDir(dir); ensureDir(dir);
const { DatabaseSync } = requireNodeSqlite(); const { DatabaseSync } = requireNodeSqlite();
return new DatabaseSync(dbPath); return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled });
} }
private ensureSchema() { private ensureSchema() {
@@ -360,6 +488,7 @@ export class MemoryIndexManager {
} }
private async runSync(params?: { reason?: string; force?: boolean }) { private async runSync(params?: { reason?: string; force?: boolean }) {
const vectorReady = await this.ensureVectorReady();
const meta = this.readMeta(); const meta = this.readMeta();
const needsFullReindex = const needsFullReindex =
params?.force || params?.force ||
@@ -367,7 +496,8 @@ export class MemoryIndexManager {
meta.model !== this.provider.model || meta.model !== this.provider.model ||
meta.provider !== this.provider.id || meta.provider !== this.provider.id ||
meta.chunkTokens !== this.settings.chunking.tokens || meta.chunkTokens !== this.settings.chunking.tokens ||
meta.chunkOverlap !== this.settings.chunking.overlap; meta.chunkOverlap !== this.settings.chunking.overlap ||
(vectorReady && !meta?.vectorDims);
if (needsFullReindex) { if (needsFullReindex) {
this.resetIndex(); this.resetIndex();
} }
@@ -397,18 +527,24 @@ export class MemoryIndexManager {
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path); this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path);
} }
this.writeMeta({ const nextMeta: MemoryIndexMeta = {
model: this.provider.model, model: this.provider.model,
provider: this.provider.id, provider: this.provider.id,
chunkTokens: this.settings.chunking.tokens, chunkTokens: this.settings.chunking.tokens,
chunkOverlap: this.settings.chunking.overlap, chunkOverlap: this.settings.chunking.overlap,
}); };
if (this.vector.available && this.vector.dims) {
nextMeta.vectorDims = this.vector.dims;
}
this.writeMeta(nextMeta);
this.dirty = false; this.dirty = false;
} }
private resetIndex() { private resetIndex() {
this.db.exec(`DELETE FROM files`); this.db.exec(`DELETE FROM files`);
this.db.exec(`DELETE FROM chunks`); this.db.exec(`DELETE FROM chunks`);
this.dropVectorTable();
this.vector.dims = undefined;
} }
private readMeta(): MemoryIndexMeta | null { private readMeta(): MemoryIndexMeta | null {
@@ -436,6 +572,8 @@ export class MemoryIndexManager {
const content = await fs.readFile(entry.absPath, "utf-8"); const content = await fs.readFile(entry.absPath, "utf-8");
const chunks = chunkMarkdown(content, this.settings.chunking); const chunks = chunkMarkdown(content, this.settings.chunking);
const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text)); const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text));
const sample = embeddings.find((embedding) => embedding.length > 0);
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
const now = Date.now(); const now = Date.now();
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path); this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path);
for (let i = 0; i < chunks.length; i++) { for (let i = 0; i < chunks.length; i++) {
@@ -466,6 +604,11 @@ export class MemoryIndexManager {
JSON.stringify(embedding), JSON.stringify(embedding),
now, now,
); );
if (vectorReady && embedding.length > 0) {
this.db
.prepare(`INSERT OR REPLACE INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`)
.run(id, vectorToBlob(embedding));
}
} }
this.db this.db
.prepare( .prepare(