feat: add sqlite-vec memory search acceleration
This commit is contained in:
@@ -49,6 +49,7 @@ Docs: https://docs.clawd.bot
|
|||||||
- Status: trim `/status` to current-provider usage only and drop the OAuth/token block.
|
- Status: trim `/status` to current-provider usage only and drop the OAuth/token block.
|
||||||
- Directory: unify `clawdbot directory` across channels and plugin channels.
|
- Directory: unify `clawdbot directory` across channels and plugin channels.
|
||||||
- UI: allow deleting sessions from the Control UI.
|
- UI: allow deleting sessions from the Control UI.
|
||||||
|
- Memory: add sqlite-vec vector acceleration with CLI status details.
|
||||||
- Skills: add user-invocable skill commands and expanded skill command registration.
|
- Skills: add user-invocable skill commands and expanded skill command registration.
|
||||||
- Telegram: default reaction level to minimal and enable reaction notifications by default.
|
- Telegram: default reaction level to minimal and enable reaction notifications by default.
|
||||||
- Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2.
|
- Telegram: allow reply-chain messages to bypass mention gating in groups. (#1038) — thanks @adityashaw2.
|
||||||
|
|||||||
@@ -78,6 +78,7 @@ Defaults:
|
|||||||
- Watches memory files for changes (debounced).
|
- Watches memory files for changes (debounced).
|
||||||
- Uses remote embeddings (OpenAI) unless configured for local.
|
- Uses remote embeddings (OpenAI) unless configured for local.
|
||||||
- Local mode uses node-llama-cpp and may require `pnpm approve-builds`.
|
- Local mode uses node-llama-cpp and may require `pnpm approve-builds`.
|
||||||
|
- Uses sqlite-vec (when available) to accelerate vector search inside SQLite.
|
||||||
|
|
||||||
Remote embeddings **require** an API key for the embedding provider. By default
|
Remote embeddings **require** an API key for the embedding provider. By default
|
||||||
this is OpenAI (`OPENAI_API_KEY` or `models.providers.openai.apiKey`). Codex
|
this is OpenAI (`OPENAI_API_KEY` or `models.providers.openai.apiKey`). Codex
|
||||||
@@ -143,6 +144,37 @@ Local mode:
|
|||||||
- Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
|
- Index storage: per-agent SQLite at `~/.clawdbot/state/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
|
||||||
- Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change.
|
- Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync runs on session start, on first search when dirty, and optionally on an interval. Reindex triggers when embedding model/provider or chunk sizes change.
|
||||||
|
|
||||||
|
### SQLite vector acceleration (sqlite-vec)
|
||||||
|
|
||||||
|
When the sqlite-vec extension is available, Clawdbot stores embeddings in a
|
||||||
|
SQLite virtual table (`vec0`) and performs vector distance queries in the
|
||||||
|
database. This keeps search fast without loading every embedding into JS.
|
||||||
|
|
||||||
|
Configuration (optional):
|
||||||
|
|
||||||
|
```json5
|
||||||
|
agents: {
|
||||||
|
defaults: {
|
||||||
|
memorySearch: {
|
||||||
|
store: {
|
||||||
|
vector: {
|
||||||
|
enabled: true,
|
||||||
|
extensionPath: "/path/to/sqlite-vec"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- `enabled` defaults to true; when disabled, search falls back to in-process
|
||||||
|
cosine similarity over stored embeddings.
|
||||||
|
- If the sqlite-vec extension is missing or fails to load, Clawdbot logs the
|
||||||
|
error and continues with the JS fallback (no vector table).
|
||||||
|
- `extensionPath` overrides the bundled sqlite-vec path (useful for custom builds
|
||||||
|
or non-standard install locations).
|
||||||
|
|
||||||
### Local embedding auto-download
|
### Local embedding auto-download
|
||||||
|
|
||||||
- Default local embedding model: `hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf` (~0.6 GB).
|
- Default local embedding model: `hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf` (~0.6 GB).
|
||||||
|
|||||||
@@ -174,6 +174,7 @@
|
|||||||
"proper-lockfile": "^4.1.2",
|
"proper-lockfile": "^4.1.2",
|
||||||
"qrcode-terminal": "^0.12.0",
|
"qrcode-terminal": "^0.12.0",
|
||||||
"sharp": "^0.34.5",
|
"sharp": "^0.34.5",
|
||||||
|
"sqlite-vec": "0.1.7-alpha.2",
|
||||||
"tar": "^7.5.3",
|
"tar": "^7.5.3",
|
||||||
"tslog": "^4.10.2",
|
"tslog": "^4.10.2",
|
||||||
"undici": "^7.18.2",
|
"undici": "^7.18.2",
|
||||||
|
|||||||
54
pnpm-lock.yaml
generated
54
pnpm-lock.yaml
generated
@@ -133,6 +133,9 @@ importers:
|
|||||||
sharp:
|
sharp:
|
||||||
specifier: ^0.34.5
|
specifier: ^0.34.5
|
||||||
version: 0.34.5
|
version: 0.34.5
|
||||||
|
sqlite-vec:
|
||||||
|
specifier: 0.1.7-alpha.2
|
||||||
|
version: 0.1.7-alpha.2
|
||||||
tar:
|
tar:
|
||||||
specifier: 7.5.3
|
specifier: 7.5.3
|
||||||
version: 7.5.3
|
version: 7.5.3
|
||||||
@@ -3910,6 +3913,34 @@ packages:
|
|||||||
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
|
resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
|
||||||
engines: {node: '>= 10.x'}
|
engines: {node: '>= 10.x'}
|
||||||
|
|
||||||
|
sqlite-vec-darwin-arm64@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-raIATOqFYkeCHhb/t3r7W7Cf2lVYdf4J3ogJ6GFc8PQEgHCPEsi+bYnm2JT84MzLfTlSTIdxr4/NKv+zF7oLPw==}
|
||||||
|
cpu: [arm64]
|
||||||
|
os: [darwin]
|
||||||
|
|
||||||
|
sqlite-vec-darwin-x64@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-jeZEELsQjjRsVojsvU5iKxOvkaVuE+JYC8Y4Ma8U45aAERrDYmqZoHvgSG7cg1PXL3bMlumFTAmHynf1y4pOzA==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [darwin]
|
||||||
|
|
||||||
|
sqlite-vec-linux-arm64@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-6Spj4Nfi7tG13jsUG+W7jnT0bCTWbyPImu2M8nWp20fNrd1SZ4g3CSlDAK8GBdavX7wRlbBHCZ+BDa++rbDewA==}
|
||||||
|
cpu: [arm64]
|
||||||
|
os: [linux]
|
||||||
|
|
||||||
|
sqlite-vec-linux-x64@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-IcgrbHaDccTVhXDf8Orwdc2+hgDLAFORl6OBUhcvlmwswwBP1hqBTSEhovClG4NItwTOBNgpwOoQ7Qp3VDPWLg==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [linux]
|
||||||
|
|
||||||
|
sqlite-vec-windows-x64@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-TRP6hTjAcwvQ6xpCZvjP00pdlda8J38ArFy1lMYhtQWXiIBmWnhMaMbq4kaeCYwvTTddfidatRS+TJrwIKB/oQ==}
|
||||||
|
cpu: [x64]
|
||||||
|
os: [win32]
|
||||||
|
|
||||||
|
sqlite-vec@0.1.7-alpha.2:
|
||||||
|
resolution: {integrity: sha512-rNgRCv+4V4Ed3yc33Qr+nNmjhtrMnnHzXfLVPeGb28Dx5mmDL3Ngw/Wk8vhCGjj76+oC6gnkmMG8y73BZWGBwQ==}
|
||||||
|
|
||||||
stackback@0.0.2:
|
stackback@0.0.2:
|
||||||
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
|
resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==}
|
||||||
|
|
||||||
@@ -8563,6 +8594,29 @@ snapshots:
|
|||||||
|
|
||||||
split2@4.2.0: {}
|
split2@4.2.0: {}
|
||||||
|
|
||||||
|
sqlite-vec-darwin-arm64@0.1.7-alpha.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
sqlite-vec-darwin-x64@0.1.7-alpha.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
sqlite-vec-linux-arm64@0.1.7-alpha.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
sqlite-vec-linux-x64@0.1.7-alpha.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
sqlite-vec-windows-x64@0.1.7-alpha.2:
|
||||||
|
optional: true
|
||||||
|
|
||||||
|
sqlite-vec@0.1.7-alpha.2:
|
||||||
|
optionalDependencies:
|
||||||
|
sqlite-vec-darwin-arm64: 0.1.7-alpha.2
|
||||||
|
sqlite-vec-darwin-x64: 0.1.7-alpha.2
|
||||||
|
sqlite-vec-linux-arm64: 0.1.7-alpha.2
|
||||||
|
sqlite-vec-linux-x64: 0.1.7-alpha.2
|
||||||
|
sqlite-vec-windows-x64: 0.1.7-alpha.2
|
||||||
|
|
||||||
stackback@0.0.2: {}
|
stackback@0.0.2: {}
|
||||||
|
|
||||||
statuses@2.0.2: {}
|
statuses@2.0.2: {}
|
||||||
|
|||||||
40
scripts/sqlite-vec-smoke.mjs
Normal file
40
scripts/sqlite-vec-smoke.mjs
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
import { DatabaseSync } from "node:sqlite";
|
||||||
|
import { load, getLoadablePath } from "sqlite-vec";
|
||||||
|
|
||||||
|
function vec(values) {
|
||||||
|
return Buffer.from(new Float32Array(values).buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
const db = new DatabaseSync(":memory:", { allowExtension: true });
|
||||||
|
|
||||||
|
try {
|
||||||
|
load(db);
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
console.error("sqlite-vec load failed:");
|
||||||
|
console.error(message);
|
||||||
|
console.error("expected extension path:", getLoadablePath());
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
db.exec(`
|
||||||
|
CREATE VIRTUAL TABLE v USING vec0(
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
embedding FLOAT[4]
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
const insert = db.prepare("INSERT INTO v (id, embedding) VALUES (?, ?)");
|
||||||
|
insert.run("a", vec([1, 0, 0, 0]));
|
||||||
|
insert.run("b", vec([0, 1, 0, 0]));
|
||||||
|
insert.run("c", vec([0.2, 0.2, 0, 0]));
|
||||||
|
|
||||||
|
const query = vec([1, 0, 0, 0]);
|
||||||
|
const rows = db
|
||||||
|
.prepare(
|
||||||
|
"SELECT id, vec_distance_cosine(embedding, ?) AS dist FROM v ORDER BY dist ASC"
|
||||||
|
)
|
||||||
|
.all(query);
|
||||||
|
|
||||||
|
console.log("sqlite-vec ok");
|
||||||
|
console.log(rows);
|
||||||
@@ -29,6 +29,12 @@ describe("memory search config", () => {
|
|||||||
memorySearch: {
|
memorySearch: {
|
||||||
provider: "openai",
|
provider: "openai",
|
||||||
model: "text-embedding-3-small",
|
model: "text-embedding-3-small",
|
||||||
|
store: {
|
||||||
|
vector: {
|
||||||
|
enabled: false,
|
||||||
|
extensionPath: "/opt/sqlite-vec.dylib",
|
||||||
|
},
|
||||||
|
},
|
||||||
chunking: { tokens: 500, overlap: 100 },
|
chunking: { tokens: 500, overlap: 100 },
|
||||||
query: { maxResults: 4, minScore: 0.2 },
|
query: { maxResults: 4, minScore: 0.2 },
|
||||||
},
|
},
|
||||||
@@ -40,6 +46,11 @@ describe("memory search config", () => {
|
|||||||
memorySearch: {
|
memorySearch: {
|
||||||
chunking: { tokens: 320 },
|
chunking: { tokens: 320 },
|
||||||
query: { maxResults: 8 },
|
query: { maxResults: 8 },
|
||||||
|
store: {
|
||||||
|
vector: {
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -52,6 +63,8 @@ describe("memory search config", () => {
|
|||||||
expect(resolved?.chunking.overlap).toBe(100);
|
expect(resolved?.chunking.overlap).toBe(100);
|
||||||
expect(resolved?.query.maxResults).toBe(8);
|
expect(resolved?.query.maxResults).toBe(8);
|
||||||
expect(resolved?.query.minScore).toBe(0.2);
|
expect(resolved?.query.minScore).toBe(0.2);
|
||||||
|
expect(resolved?.store.vector.enabled).toBe(true);
|
||||||
|
expect(resolved?.store.vector.extensionPath).toBe("/opt/sqlite-vec.dylib");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("merges remote defaults with agent overrides", () => {
|
it("merges remote defaults with agent overrides", () => {
|
||||||
|
|||||||
@@ -23,6 +23,10 @@ export type ResolvedMemorySearchConfig = {
|
|||||||
store: {
|
store: {
|
||||||
driver: "sqlite";
|
driver: "sqlite";
|
||||||
path: string;
|
path: string;
|
||||||
|
vector: {
|
||||||
|
enabled: boolean;
|
||||||
|
extensionPath?: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
chunking: {
|
chunking: {
|
||||||
tokens: number;
|
tokens: number;
|
||||||
@@ -77,9 +81,15 @@ function mergeConfig(
|
|||||||
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
|
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
|
||||||
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
|
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
|
||||||
};
|
};
|
||||||
|
const vector = {
|
||||||
|
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
|
||||||
|
extensionPath:
|
||||||
|
overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath,
|
||||||
|
};
|
||||||
const store = {
|
const store = {
|
||||||
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
|
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
|
||||||
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
|
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
|
||||||
|
vector,
|
||||||
};
|
};
|
||||||
const chunking = {
|
const chunking = {
|
||||||
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,
|
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,
|
||||||
|
|||||||
95
src/cli/memory-cli.test.ts
Normal file
95
src/cli/memory-cli.test.ts
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
import { Command } from "commander";
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const getMemorySearchManager = vi.fn();
|
||||||
|
const loadConfig = vi.fn(() => ({}));
|
||||||
|
const resolveDefaultAgentId = vi.fn(() => "main");
|
||||||
|
|
||||||
|
vi.mock("../memory/index.js", () => ({
|
||||||
|
getMemorySearchManager,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../config/config.js", () => ({
|
||||||
|
loadConfig,
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock("../agents/agent-scope.js", () => ({
|
||||||
|
resolveDefaultAgentId,
|
||||||
|
}));
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
getMemorySearchManager.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("memory cli", () => {
|
||||||
|
it("prints vector status when available", async () => {
|
||||||
|
const { registerMemoryCli } = await import("./memory-cli.js");
|
||||||
|
const { defaultRuntime } = await import("../runtime.js");
|
||||||
|
getMemorySearchManager.mockResolvedValueOnce({
|
||||||
|
manager: {
|
||||||
|
status: () => ({
|
||||||
|
files: 2,
|
||||||
|
chunks: 5,
|
||||||
|
dirty: false,
|
||||||
|
workspaceDir: "/tmp/clawd",
|
||||||
|
dbPath: "/tmp/memory.sqlite",
|
||||||
|
provider: "openai",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
requestedProvider: "openai",
|
||||||
|
vector: {
|
||||||
|
enabled: true,
|
||||||
|
available: true,
|
||||||
|
extensionPath: "/opt/sqlite-vec.dylib",
|
||||||
|
dims: 1024,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
|
||||||
|
const program = new Command();
|
||||||
|
program.name("test");
|
||||||
|
registerMemoryCli(program);
|
||||||
|
await program.parseAsync(["memory", "status"], { from: "user" });
|
||||||
|
|
||||||
|
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: ready"));
|
||||||
|
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector dims: 1024"));
|
||||||
|
expect(log).toHaveBeenCalledWith(
|
||||||
|
expect.stringContaining("Vector path: /opt/sqlite-vec.dylib"),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prints vector error when unavailable", async () => {
|
||||||
|
const { registerMemoryCli } = await import("./memory-cli.js");
|
||||||
|
const { defaultRuntime } = await import("../runtime.js");
|
||||||
|
getMemorySearchManager.mockResolvedValueOnce({
|
||||||
|
manager: {
|
||||||
|
status: () => ({
|
||||||
|
files: 0,
|
||||||
|
chunks: 0,
|
||||||
|
dirty: true,
|
||||||
|
workspaceDir: "/tmp/clawd",
|
||||||
|
dbPath: "/tmp/memory.sqlite",
|
||||||
|
provider: "openai",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
requestedProvider: "openai",
|
||||||
|
vector: {
|
||||||
|
enabled: true,
|
||||||
|
available: false,
|
||||||
|
loadError: "load failed",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const log = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
|
||||||
|
const program = new Command();
|
||||||
|
program.name("test");
|
||||||
|
registerMemoryCli(program);
|
||||||
|
await program.parseAsync(["memory", "status", "--agent", "main"], { from: "user" });
|
||||||
|
|
||||||
|
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector: unavailable"));
|
||||||
|
expect(log).toHaveBeenCalledWith(expect.stringContaining("Vector error: load failed"));
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -56,6 +56,23 @@ export function registerMemoryCli(program: Command) {
|
|||||||
`Dirty: ${status.dirty ? "yes" : "no"}`,
|
`Dirty: ${status.dirty ? "yes" : "no"}`,
|
||||||
`Index: ${status.dbPath}`,
|
`Index: ${status.dbPath}`,
|
||||||
].filter(Boolean) as string[];
|
].filter(Boolean) as string[];
|
||||||
|
if (status.vector) {
|
||||||
|
const vectorState = status.vector.enabled
|
||||||
|
? status.vector.available
|
||||||
|
? "ready"
|
||||||
|
: "unavailable"
|
||||||
|
: "disabled";
|
||||||
|
lines.push(`Vector: ${vectorState}`);
|
||||||
|
if (status.vector.dims) {
|
||||||
|
lines.push(`Vector dims: ${status.vector.dims}`);
|
||||||
|
}
|
||||||
|
if (status.vector.extensionPath) {
|
||||||
|
lines.push(`Vector path: ${status.vector.extensionPath}`);
|
||||||
|
}
|
||||||
|
if (status.vector.loadError) {
|
||||||
|
lines.push(chalk.yellow(`Vector error: ${status.vector.loadError}`));
|
||||||
|
}
|
||||||
|
}
|
||||||
if (status.fallback?.reason) {
|
if (status.fallback?.reason) {
|
||||||
lines.push(chalk.gray(status.fallback.reason));
|
lines.push(chalk.gray(status.fallback.reason));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -176,6 +176,9 @@ const FIELD_LABELS: Record<string, string> = {
|
|||||||
"agents.defaults.memorySearch.fallback": "Memory Search Fallback",
|
"agents.defaults.memorySearch.fallback": "Memory Search Fallback",
|
||||||
"agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path",
|
"agents.defaults.memorySearch.local.modelPath": "Local Embedding Model Path",
|
||||||
"agents.defaults.memorySearch.store.path": "Memory Search Index Path",
|
"agents.defaults.memorySearch.store.path": "Memory Search Index Path",
|
||||||
|
"agents.defaults.memorySearch.store.vector.enabled": "Memory Search Vector Index",
|
||||||
|
"agents.defaults.memorySearch.store.vector.extensionPath":
|
||||||
|
"Memory Search Vector Extension Path",
|
||||||
"agents.defaults.memorySearch.chunking.tokens": "Memory Chunk Tokens",
|
"agents.defaults.memorySearch.chunking.tokens": "Memory Chunk Tokens",
|
||||||
"agents.defaults.memorySearch.chunking.overlap": "Memory Chunk Overlap Tokens",
|
"agents.defaults.memorySearch.chunking.overlap": "Memory Chunk Overlap Tokens",
|
||||||
"agents.defaults.memorySearch.sync.onSessionStart": "Index on Session Start",
|
"agents.defaults.memorySearch.sync.onSessionStart": "Index on Session Start",
|
||||||
@@ -362,7 +365,11 @@ const FIELD_HELP: Record<string, string> = {
|
|||||||
"agents.defaults.memorySearch.fallback":
|
"agents.defaults.memorySearch.fallback":
|
||||||
'Fallback to OpenAI when local embeddings fail ("openai" or "none").',
|
'Fallback to OpenAI when local embeddings fail ("openai" or "none").',
|
||||||
"agents.defaults.memorySearch.store.path":
|
"agents.defaults.memorySearch.store.path":
|
||||||
"SQLite index path (default: ~/.clawdbot/memory/{agentId}.sqlite).",
|
"SQLite index path (default: ~/.clawdbot/state/memory/{agentId}.sqlite).",
|
||||||
|
"agents.defaults.memorySearch.store.vector.enabled":
|
||||||
|
"Enable sqlite-vec extension for vector search (default: true).",
|
||||||
|
"agents.defaults.memorySearch.store.vector.extensionPath":
|
||||||
|
"Optional override path to sqlite-vec extension library (.dylib/.so/.dll).",
|
||||||
"agents.defaults.memorySearch.sync.onSearch":
|
"agents.defaults.memorySearch.sync.onSearch":
|
||||||
"Lazy sync: reindex on first search after a change.",
|
"Lazy sync: reindex on first search after a change.",
|
||||||
"agents.defaults.memorySearch.sync.watch": "Watch memory files for changes (chokidar).",
|
"agents.defaults.memorySearch.sync.watch": "Watch memory files for changes (chokidar).",
|
||||||
|
|||||||
@@ -167,6 +167,12 @@ export type MemorySearchConfig = {
|
|||||||
store?: {
|
store?: {
|
||||||
driver?: "sqlite";
|
driver?: "sqlite";
|
||||||
path?: string;
|
path?: string;
|
||||||
|
vector?: {
|
||||||
|
/** Enable sqlite-vec extension for vector search (default: true). */
|
||||||
|
enabled?: boolean;
|
||||||
|
/** Optional override path to sqlite-vec extension (.dylib/.so/.dll). */
|
||||||
|
extensionPath?: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
/** Chunking configuration. */
|
/** Chunking configuration. */
|
||||||
chunking?: {
|
chunking?: {
|
||||||
|
|||||||
@@ -214,6 +214,12 @@ export const MemorySearchSchema = z
|
|||||||
.object({
|
.object({
|
||||||
driver: z.literal("sqlite").optional(),
|
driver: z.literal("sqlite").optional(),
|
||||||
path: z.string().optional(),
|
path: z.string().optional(),
|
||||||
|
vector: z
|
||||||
|
.object({
|
||||||
|
enabled: z.boolean().optional(),
|
||||||
|
extensionPath: z.string().optional(),
|
||||||
|
})
|
||||||
|
.optional(),
|
||||||
})
|
})
|
||||||
.optional(),
|
.optional(),
|
||||||
chunking: z
|
chunking: z
|
||||||
|
|||||||
@@ -42,15 +42,20 @@ type MemoryIndexMeta = {
|
|||||||
provider: string;
|
provider: string;
|
||||||
chunkTokens: number;
|
chunkTokens: number;
|
||||||
chunkOverlap: number;
|
chunkOverlap: number;
|
||||||
|
vectorDims?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
const META_KEY = "memory_index_meta_v1";
|
const META_KEY = "memory_index_meta_v1";
|
||||||
const SNIPPET_MAX_CHARS = 700;
|
const SNIPPET_MAX_CHARS = 700;
|
||||||
|
const VECTOR_TABLE = "chunks_vec";
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
|
|
||||||
const INDEX_CACHE = new Map<string, MemoryIndexManager>();
|
const INDEX_CACHE = new Map<string, MemoryIndexManager>();
|
||||||
|
|
||||||
|
const vectorToBlob = (embedding: number[]): Buffer =>
|
||||||
|
Buffer.from(new Float32Array(embedding).buffer);
|
||||||
|
|
||||||
export class MemoryIndexManager {
|
export class MemoryIndexManager {
|
||||||
private readonly cacheKey: string;
|
private readonly cacheKey: string;
|
||||||
private readonly cfg: ClawdbotConfig;
|
private readonly cfg: ClawdbotConfig;
|
||||||
@@ -61,6 +66,14 @@ export class MemoryIndexManager {
|
|||||||
private readonly requestedProvider: "openai" | "local";
|
private readonly requestedProvider: "openai" | "local";
|
||||||
private readonly fallbackReason?: string;
|
private readonly fallbackReason?: string;
|
||||||
private readonly db: DatabaseSync;
|
private readonly db: DatabaseSync;
|
||||||
|
private readonly vector: {
|
||||||
|
enabled: boolean;
|
||||||
|
available: boolean | null;
|
||||||
|
extensionPath?: string;
|
||||||
|
loadError?: string;
|
||||||
|
dims?: number;
|
||||||
|
};
|
||||||
|
private vectorReady: Promise<boolean> | null = null;
|
||||||
private watcher: FSWatcher | null = null;
|
private watcher: FSWatcher | null = null;
|
||||||
private watchTimer: NodeJS.Timeout | null = null;
|
private watchTimer: NodeJS.Timeout | null = null;
|
||||||
private intervalTimer: NodeJS.Timeout | null = null;
|
private intervalTimer: NodeJS.Timeout | null = null;
|
||||||
@@ -119,6 +132,15 @@ export class MemoryIndexManager {
|
|||||||
this.fallbackReason = params.providerResult.fallbackReason;
|
this.fallbackReason = params.providerResult.fallbackReason;
|
||||||
this.db = this.openDatabase();
|
this.db = this.openDatabase();
|
||||||
this.ensureSchema();
|
this.ensureSchema();
|
||||||
|
this.vector = {
|
||||||
|
enabled: params.settings.store.vector.enabled,
|
||||||
|
available: null,
|
||||||
|
extensionPath: params.settings.store.vector.extensionPath,
|
||||||
|
};
|
||||||
|
const meta = this.readMeta();
|
||||||
|
if (meta?.vectorDims) {
|
||||||
|
this.vector.dims = meta.vectorDims;
|
||||||
|
}
|
||||||
this.ensureWatcher();
|
this.ensureWatcher();
|
||||||
this.ensureIntervalSync();
|
this.ensureIntervalSync();
|
||||||
this.dirty = true;
|
this.dirty = true;
|
||||||
@@ -146,8 +168,38 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
const cleaned = query.trim();
|
const cleaned = query.trim();
|
||||||
if (!cleaned) return [];
|
if (!cleaned) return [];
|
||||||
|
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
||||||
|
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
||||||
const queryVec = await this.provider.embedQuery(cleaned);
|
const queryVec = await this.provider.embedQuery(cleaned);
|
||||||
if (queryVec.length === 0) return [];
|
if (queryVec.length === 0) return [];
|
||||||
|
if (await this.ensureVectorReady(queryVec.length)) {
|
||||||
|
const rows = this.db
|
||||||
|
.prepare(
|
||||||
|
`SELECT c.path, c.start_line, c.end_line, c.text,
|
||||||
|
vec_distance_cosine(v.embedding, ?) AS dist
|
||||||
|
FROM ${VECTOR_TABLE} v
|
||||||
|
JOIN chunks c ON c.id = v.id
|
||||||
|
WHERE c.model = ?
|
||||||
|
ORDER BY dist ASC
|
||||||
|
LIMIT ?`,
|
||||||
|
)
|
||||||
|
.all(vectorToBlob(queryVec), this.provider.model, maxResults) as Array<{
|
||||||
|
path: string;
|
||||||
|
start_line: number;
|
||||||
|
end_line: number;
|
||||||
|
text: string;
|
||||||
|
dist: number;
|
||||||
|
}>;
|
||||||
|
return rows
|
||||||
|
.map((row) => ({
|
||||||
|
path: row.path,
|
||||||
|
startLine: row.start_line,
|
||||||
|
endLine: row.end_line,
|
||||||
|
score: 1 - row.dist,
|
||||||
|
snippet: truncateUtf16Safe(row.text, SNIPPET_MAX_CHARS),
|
||||||
|
}))
|
||||||
|
.filter((entry) => entry.score >= minScore);
|
||||||
|
}
|
||||||
const candidates = this.listChunks();
|
const candidates = this.listChunks();
|
||||||
const scored = candidates
|
const scored = candidates
|
||||||
.map((chunk) => ({
|
.map((chunk) => ({
|
||||||
@@ -155,8 +207,6 @@ export class MemoryIndexManager {
|
|||||||
score: cosineSimilarity(queryVec, chunk.embedding),
|
score: cosineSimilarity(queryVec, chunk.embedding),
|
||||||
}))
|
}))
|
||||||
.filter((entry) => Number.isFinite(entry.score));
|
.filter((entry) => Number.isFinite(entry.score));
|
||||||
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
|
||||||
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
|
||||||
return scored
|
return scored
|
||||||
.filter((entry) => entry.score >= minScore)
|
.filter((entry) => entry.score >= minScore)
|
||||||
.sort((a, b) => b.score - a.score)
|
.sort((a, b) => b.score - a.score)
|
||||||
@@ -212,6 +262,13 @@ export class MemoryIndexManager {
|
|||||||
model: string;
|
model: string;
|
||||||
requestedProvider: string;
|
requestedProvider: string;
|
||||||
fallback?: { from: string; reason?: string };
|
fallback?: { from: string; reason?: string };
|
||||||
|
vector?: {
|
||||||
|
enabled: boolean;
|
||||||
|
available?: boolean;
|
||||||
|
extensionPath?: string;
|
||||||
|
loadError?: string;
|
||||||
|
dims?: number;
|
||||||
|
};
|
||||||
} {
|
} {
|
||||||
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as {
|
const files = this.db.prepare(`SELECT COUNT(*) as c FROM files`).get() as {
|
||||||
c: number;
|
c: number;
|
||||||
@@ -229,6 +286,13 @@ export class MemoryIndexManager {
|
|||||||
model: this.provider.model,
|
model: this.provider.model,
|
||||||
requestedProvider: this.requestedProvider,
|
requestedProvider: this.requestedProvider,
|
||||||
fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined,
|
fallback: this.fallbackReason ? { from: "local", reason: this.fallbackReason } : undefined,
|
||||||
|
vector: {
|
||||||
|
enabled: this.vector.enabled,
|
||||||
|
available: this.vector.available ?? undefined,
|
||||||
|
extensionPath: this.vector.extensionPath,
|
||||||
|
loadError: this.vector.loadError,
|
||||||
|
dims: this.vector.dims,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,12 +315,76 @@ export class MemoryIndexManager {
|
|||||||
INDEX_CACHE.delete(this.cacheKey);
|
INDEX_CACHE.delete(this.cacheKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
||||||
|
if (!this.vector.enabled) return false;
|
||||||
|
if (!this.vectorReady) {
|
||||||
|
this.vectorReady = this.loadVectorExtension();
|
||||||
|
}
|
||||||
|
const ready = await this.vectorReady;
|
||||||
|
if (ready && typeof dimensions === "number" && dimensions > 0) {
|
||||||
|
this.ensureVectorTable(dimensions);
|
||||||
|
}
|
||||||
|
return ready;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async loadVectorExtension(): Promise<boolean> {
|
||||||
|
if (this.vector.available !== null) return this.vector.available;
|
||||||
|
if (!this.vector.enabled) {
|
||||||
|
this.vector.available = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const sqliteVec = await import("sqlite-vec");
|
||||||
|
const extensionPath = this.vector.extensionPath?.trim()
|
||||||
|
? resolveUserPath(this.vector.extensionPath)
|
||||||
|
: sqliteVec.getLoadablePath();
|
||||||
|
this.db.enableLoadExtension(true);
|
||||||
|
if (this.vector.extensionPath?.trim()) {
|
||||||
|
this.db.loadExtension(extensionPath);
|
||||||
|
} else {
|
||||||
|
sqliteVec.load(this.db);
|
||||||
|
}
|
||||||
|
this.vector.extensionPath = extensionPath;
|
||||||
|
this.vector.available = true;
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
this.vector.available = false;
|
||||||
|
this.vector.loadError = message;
|
||||||
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureVectorTable(dimensions: number): void {
|
||||||
|
if (this.vector.dims === dimensions) return;
|
||||||
|
if (this.vector.dims && this.vector.dims !== dimensions) {
|
||||||
|
this.dropVectorTable();
|
||||||
|
}
|
||||||
|
this.db.exec(
|
||||||
|
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
|
||||||
|
` id TEXT PRIMARY KEY,\n` +
|
||||||
|
` embedding FLOAT[${dimensions}]\n` +
|
||||||
|
`)`,
|
||||||
|
);
|
||||||
|
this.vector.dims = dimensions;
|
||||||
|
}
|
||||||
|
|
||||||
|
private dropVectorTable(): void {
|
||||||
|
try {
|
||||||
|
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private openDatabase(): DatabaseSync {
|
private openDatabase(): DatabaseSync {
|
||||||
const dbPath = resolveUserPath(this.settings.store.path);
|
const dbPath = resolveUserPath(this.settings.store.path);
|
||||||
const dir = path.dirname(dbPath);
|
const dir = path.dirname(dbPath);
|
||||||
ensureDir(dir);
|
ensureDir(dir);
|
||||||
const { DatabaseSync } = requireNodeSqlite();
|
const { DatabaseSync } = requireNodeSqlite();
|
||||||
return new DatabaseSync(dbPath);
|
return new DatabaseSync(dbPath, { allowExtension: this.settings.store.vector.enabled });
|
||||||
}
|
}
|
||||||
|
|
||||||
private ensureSchema() {
|
private ensureSchema() {
|
||||||
@@ -360,6 +488,7 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async runSync(params?: { reason?: string; force?: boolean }) {
|
private async runSync(params?: { reason?: string; force?: boolean }) {
|
||||||
|
const vectorReady = await this.ensureVectorReady();
|
||||||
const meta = this.readMeta();
|
const meta = this.readMeta();
|
||||||
const needsFullReindex =
|
const needsFullReindex =
|
||||||
params?.force ||
|
params?.force ||
|
||||||
@@ -367,7 +496,8 @@ export class MemoryIndexManager {
|
|||||||
meta.model !== this.provider.model ||
|
meta.model !== this.provider.model ||
|
||||||
meta.provider !== this.provider.id ||
|
meta.provider !== this.provider.id ||
|
||||||
meta.chunkTokens !== this.settings.chunking.tokens ||
|
meta.chunkTokens !== this.settings.chunking.tokens ||
|
||||||
meta.chunkOverlap !== this.settings.chunking.overlap;
|
meta.chunkOverlap !== this.settings.chunking.overlap ||
|
||||||
|
(vectorReady && !meta?.vectorDims);
|
||||||
if (needsFullReindex) {
|
if (needsFullReindex) {
|
||||||
this.resetIndex();
|
this.resetIndex();
|
||||||
}
|
}
|
||||||
@@ -397,18 +527,24 @@ export class MemoryIndexManager {
|
|||||||
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path);
|
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(stale.path);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.writeMeta({
|
const nextMeta: MemoryIndexMeta = {
|
||||||
model: this.provider.model,
|
model: this.provider.model,
|
||||||
provider: this.provider.id,
|
provider: this.provider.id,
|
||||||
chunkTokens: this.settings.chunking.tokens,
|
chunkTokens: this.settings.chunking.tokens,
|
||||||
chunkOverlap: this.settings.chunking.overlap,
|
chunkOverlap: this.settings.chunking.overlap,
|
||||||
});
|
};
|
||||||
|
if (this.vector.available && this.vector.dims) {
|
||||||
|
nextMeta.vectorDims = this.vector.dims;
|
||||||
|
}
|
||||||
|
this.writeMeta(nextMeta);
|
||||||
this.dirty = false;
|
this.dirty = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private resetIndex() {
|
private resetIndex() {
|
||||||
this.db.exec(`DELETE FROM files`);
|
this.db.exec(`DELETE FROM files`);
|
||||||
this.db.exec(`DELETE FROM chunks`);
|
this.db.exec(`DELETE FROM chunks`);
|
||||||
|
this.dropVectorTable();
|
||||||
|
this.vector.dims = undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
private readMeta(): MemoryIndexMeta | null {
|
private readMeta(): MemoryIndexMeta | null {
|
||||||
@@ -436,6 +572,8 @@ export class MemoryIndexManager {
|
|||||||
const content = await fs.readFile(entry.absPath, "utf-8");
|
const content = await fs.readFile(entry.absPath, "utf-8");
|
||||||
const chunks = chunkMarkdown(content, this.settings.chunking);
|
const chunks = chunkMarkdown(content, this.settings.chunking);
|
||||||
const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text));
|
const embeddings = await this.provider.embedBatch(chunks.map((chunk) => chunk.text));
|
||||||
|
const sample = embeddings.find((embedding) => embedding.length > 0);
|
||||||
|
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path);
|
this.db.prepare(`DELETE FROM chunks WHERE path = ?`).run(entry.path);
|
||||||
for (let i = 0; i < chunks.length; i++) {
|
for (let i = 0; i < chunks.length; i++) {
|
||||||
@@ -466,6 +604,11 @@ export class MemoryIndexManager {
|
|||||||
JSON.stringify(embedding),
|
JSON.stringify(embedding),
|
||||||
now,
|
now,
|
||||||
);
|
);
|
||||||
|
if (vectorReady && embedding.length > 0) {
|
||||||
|
this.db
|
||||||
|
.prepare(`INSERT OR REPLACE INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`)
|
||||||
|
.run(id, vectorToBlob(embedding));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
this.db
|
this.db
|
||||||
.prepare(
|
.prepare(
|
||||||
|
|||||||
Reference in New Issue
Block a user