fix: prevent memory CLI hangs
This commit is contained in:
@@ -6,6 +6,7 @@ Docs: https://docs.clawd.bot
|
|||||||
|
|
||||||
### Changes
|
### Changes
|
||||||
- Highlight: Lobster optional plugin tool for typed workflows + approval gates. https://docs.clawd.bot/tools/lobster
|
- Highlight: Lobster optional plugin tool for typed workflows + approval gates. https://docs.clawd.bot/tools/lobster
|
||||||
|
- Memory: prevent CLI hangs by deferring vector probes, adding sqlite-vec/embedding timeouts, and showing sync progress early.
|
||||||
- Docs: add troubleshooting entry for gateway.mode blocking gateway start. https://docs.clawd.bot/gateway/troubleshooting
|
- Docs: add troubleshooting entry for gateway.mode blocking gateway start. https://docs.clawd.bot/gateway/troubleshooting
|
||||||
- Onboarding: remove the run setup-token auth option (paste setup-token or reuse CLI creds instead).
|
- Onboarding: remove the run setup-token auth option (paste setup-token or reuse CLI creds instead).
|
||||||
- Signal: add typing indicators and DM read receipts via signal-cli.
|
- Signal: add typing indicators and DM read receipts via signal-cli.
|
||||||
|
|||||||
@@ -323,9 +323,11 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
|||||||
}
|
}
|
||||||
if (status.vector) {
|
if (status.vector) {
|
||||||
const vectorState = status.vector.enabled
|
const vectorState = status.vector.enabled
|
||||||
? status.vector.available
|
? status.vector.available === undefined
|
||||||
? "ready"
|
? "unknown"
|
||||||
: "unavailable"
|
: status.vector.available
|
||||||
|
? "ready"
|
||||||
|
: "unavailable"
|
||||||
: "disabled";
|
: "disabled";
|
||||||
const vectorColor =
|
const vectorColor =
|
||||||
vectorState === "ready"
|
vectorState === "ready"
|
||||||
|
|||||||
@@ -589,16 +589,20 @@ export async function updateCommand(opts: UpdateCommandOptions): Promise<void> {
|
|||||||
let tag = explicitTag ?? channelToNpmTag(channel);
|
let tag = explicitTag ?? channelToNpmTag(channel);
|
||||||
if (updateInstallKind !== "git") {
|
if (updateInstallKind !== "git") {
|
||||||
const currentVersion = switchToPackage ? null : await readPackageVersion(root);
|
const currentVersion = switchToPackage ? null : await readPackageVersion(root);
|
||||||
|
let fallbackToLatest = false;
|
||||||
const targetVersion = explicitTag
|
const targetVersion = explicitTag
|
||||||
? await resolveTargetVersion(tag, timeoutMs)
|
? await resolveTargetVersion(tag, timeoutMs)
|
||||||
: await resolveNpmChannelTag({ channel, timeoutMs }).then((resolved) => {
|
: await resolveNpmChannelTag({ channel, timeoutMs }).then((resolved) => {
|
||||||
tag = resolved.tag;
|
tag = resolved.tag;
|
||||||
|
fallbackToLatest = channel === "beta" && resolved.tag === "latest";
|
||||||
return resolved.version;
|
return resolved.version;
|
||||||
});
|
});
|
||||||
const cmp =
|
const cmp =
|
||||||
currentVersion && targetVersion ? compareSemverStrings(currentVersion, targetVersion) : null;
|
currentVersion && targetVersion ? compareSemverStrings(currentVersion, targetVersion) : null;
|
||||||
const needsConfirm =
|
const needsConfirm =
|
||||||
currentVersion != null && (targetVersion == null || (cmp != null && cmp > 0));
|
!fallbackToLatest &&
|
||||||
|
currentVersion != null &&
|
||||||
|
(targetVersion == null || (cmp != null && cmp > 0));
|
||||||
|
|
||||||
if (needsConfirm && !opts.yes) {
|
if (needsConfirm && !opts.yes) {
|
||||||
if (!process.stdin.isTTY || opts.json) {
|
if (!process.stdin.isTTY || opts.json) {
|
||||||
|
|||||||
@@ -102,6 +102,11 @@ const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|||||||
const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
|
const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
|
||||||
const BATCH_FAILURE_LIMIT = 2;
|
const BATCH_FAILURE_LIMIT = 2;
|
||||||
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
||||||
|
const VECTOR_LOAD_TIMEOUT_MS = 30_000;
|
||||||
|
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
|
||||||
|
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
|
||||||
|
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 60_000;
|
||||||
|
const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 60_000;
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
|
|
||||||
@@ -281,7 +286,7 @@ export class MemoryIndexManager {
|
|||||||
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
const queryVec = await this.provider.embedQuery(cleaned);
|
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
||||||
const hasVector = queryVec.some((v) => v !== 0);
|
const hasVector = queryVec.some((v) => v !== 0);
|
||||||
const vectorResults = hasVector
|
const vectorResults = hasVector
|
||||||
? await this.searchVector(queryVec, candidates).catch(() => [])
|
? await this.searchVector(queryVec, candidates).catch(() => [])
|
||||||
@@ -580,9 +585,23 @@ export class MemoryIndexManager {
|
|||||||
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
||||||
if (!this.vector.enabled) return false;
|
if (!this.vector.enabled) return false;
|
||||||
if (!this.vectorReady) {
|
if (!this.vectorReady) {
|
||||||
this.vectorReady = this.loadVectorExtension();
|
this.vectorReady = this.withTimeout(
|
||||||
|
this.loadVectorExtension(),
|
||||||
|
VECTOR_LOAD_TIMEOUT_MS,
|
||||||
|
`sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1000)}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let ready = false;
|
||||||
|
try {
|
||||||
|
ready = await this.vectorReady;
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
this.vector.available = false;
|
||||||
|
this.vector.loadError = message;
|
||||||
|
this.vectorReady = null;
|
||||||
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
const ready = await this.vectorReady;
|
|
||||||
if (ready && typeof dimensions === "number" && dimensions > 0) {
|
if (ready && typeof dimensions === "number" && dimensions > 0) {
|
||||||
this.ensureVectorTable(dimensions);
|
this.ensureVectorTable(dimensions);
|
||||||
}
|
}
|
||||||
@@ -1153,6 +1172,13 @@ export class MemoryIndexManager {
|
|||||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||||
}) {
|
}) {
|
||||||
const progress = params?.progress ? this.createSyncProgress(params.progress) : undefined;
|
const progress = params?.progress ? this.createSyncProgress(params.progress) : undefined;
|
||||||
|
if (progress) {
|
||||||
|
progress.report({
|
||||||
|
completed: progress.completed,
|
||||||
|
total: progress.total,
|
||||||
|
label: "Loading vector extension…",
|
||||||
|
});
|
||||||
|
}
|
||||||
const vectorReady = await this.ensureVectorReady();
|
const vectorReady = await this.ensureVectorReady();
|
||||||
const meta = this.readMeta();
|
const meta = this.readMeta();
|
||||||
const needsFullReindex =
|
const needsFullReindex =
|
||||||
@@ -1844,7 +1870,17 @@ export class MemoryIndexManager {
|
|||||||
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
|
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
return await this.provider.embedBatch(texts);
|
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||||
|
log.debug("memory embeddings: batch start", {
|
||||||
|
provider: this.provider.id,
|
||||||
|
items: texts.length,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
return await this.withTimeout(
|
||||||
|
this.provider.embedBatch(texts),
|
||||||
|
timeoutMs,
|
||||||
|
`memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||||
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
if (!this.isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) {
|
if (!this.isRetryableEmbeddingError(message) || attempt >= EMBEDDING_RETRY_MAX_ATTEMPTS) {
|
||||||
@@ -1868,6 +1904,41 @@ export class MemoryIndexManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private resolveEmbeddingTimeout(kind: "query" | "batch"): number {
|
||||||
|
const isLocal = this.provider.id === "local";
|
||||||
|
if (kind === "query") {
|
||||||
|
return isLocal ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
||||||
|
}
|
||||||
|
return isLocal ? EMBEDDING_BATCH_TIMEOUT_LOCAL_MS : EMBEDDING_BATCH_TIMEOUT_REMOTE_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async embedQueryWithTimeout(text: string): Promise<number[]> {
|
||||||
|
const timeoutMs = this.resolveEmbeddingTimeout("query");
|
||||||
|
log.debug("memory embeddings: query start", { provider: this.provider.id, timeoutMs });
|
||||||
|
return await this.withTimeout(
|
||||||
|
this.provider.embedQuery(text),
|
||||||
|
timeoutMs,
|
||||||
|
`memory embeddings query timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async withTimeout<T>(
|
||||||
|
promise: Promise<T>,
|
||||||
|
timeoutMs: number,
|
||||||
|
message: string,
|
||||||
|
): Promise<T> {
|
||||||
|
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise;
|
||||||
|
let timer: NodeJS.Timeout | null = null;
|
||||||
|
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||||
|
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
return (await Promise.race([promise, timeoutPromise])) as T;
|
||||||
|
} finally {
|
||||||
|
if (timer) clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
|
private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
|
||||||
if (tasks.length === 0) return [];
|
if (tasks.length === 0) return [];
|
||||||
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
||||||
|
|||||||
Reference in New Issue
Block a user