fix(whatsapp): reconnect on crypto unhandled rejection
This commit is contained in:
@@ -27,6 +27,7 @@ import {
|
|||||||
PortInUseError,
|
PortInUseError,
|
||||||
} from "./infra/ports.js";
|
} from "./infra/ports.js";
|
||||||
import { assertSupportedRuntime } from "./infra/runtime-guard.js";
|
import { assertSupportedRuntime } from "./infra/runtime-guard.js";
|
||||||
|
import { isUnhandledRejectionHandled } from "./infra/unhandled-rejections.js";
|
||||||
import { enableConsoleCapture } from "./logging.js";
|
import { enableConsoleCapture } from "./logging.js";
|
||||||
import { runCommandWithTimeout, runExec } from "./process/exec.js";
|
import { runCommandWithTimeout, runExec } from "./process/exec.js";
|
||||||
import { monitorWebProvider } from "./provider-web.js";
|
import { monitorWebProvider } from "./provider-web.js";
|
||||||
@@ -79,6 +80,7 @@ if (isMain) {
|
|||||||
// Global error handlers to prevent silent crashes from unhandled rejections/exceptions.
|
// Global error handlers to prevent silent crashes from unhandled rejections/exceptions.
|
||||||
// These log the error and exit gracefully instead of crashing without trace.
|
// These log the error and exit gracefully instead of crashing without trace.
|
||||||
process.on("unhandledRejection", (reason, _promise) => {
|
process.on("unhandledRejection", (reason, _promise) => {
|
||||||
|
if (isUnhandledRejectionHandled(reason)) return;
|
||||||
console.error(
|
console.error(
|
||||||
"[clawdbot] Unhandled promise rejection:",
|
"[clawdbot] Unhandled promise rejection:",
|
||||||
reason instanceof Error ? (reason.stack ?? reason.message) : reason,
|
reason instanceof Error ? (reason.stack ?? reason.message) : reason,
|
||||||
|
|||||||
26
src/infra/unhandled-rejections.ts
Normal file
26
src/infra/unhandled-rejections.ts
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
type UnhandledRejectionHandler = (reason: unknown) => boolean;
|
||||||
|
|
||||||
|
const handlers = new Set<UnhandledRejectionHandler>();
|
||||||
|
|
||||||
|
export function registerUnhandledRejectionHandler(
|
||||||
|
handler: UnhandledRejectionHandler,
|
||||||
|
): () => void {
|
||||||
|
handlers.add(handler);
|
||||||
|
return () => {
|
||||||
|
handlers.delete(handler);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isUnhandledRejectionHandled(reason: unknown): boolean {
|
||||||
|
for (const handler of handlers) {
|
||||||
|
try {
|
||||||
|
if (handler(reason)) return true;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(
|
||||||
|
"[clawdbot] Unhandled rejection handler failed:",
|
||||||
|
err instanceof Error ? (err.stack ?? err.message) : err,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
@@ -32,13 +32,6 @@ async function main() {
|
|||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (process.env.CLAWDBOT_SMOKE_QR === "1") {
|
|
||||||
const { renderQrPngBase64 } = await import("../web/qr-image.js");
|
|
||||||
await renderQrPngBase64("clawdbot-smoke");
|
|
||||||
console.log("smoke: qr ok");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await patchBunLongForProtobuf();
|
await patchBunLongForProtobuf();
|
||||||
|
|
||||||
const { loadDotEnv } = await import("../infra/dotenv.js");
|
const { loadDotEnv } = await import("../infra/dotenv.js");
|
||||||
@@ -52,11 +45,15 @@ async function main() {
|
|||||||
|
|
||||||
const { assertSupportedRuntime } = await import("../infra/runtime-guard.js");
|
const { assertSupportedRuntime } = await import("../infra/runtime-guard.js");
|
||||||
assertSupportedRuntime();
|
assertSupportedRuntime();
|
||||||
|
const { isUnhandledRejectionHandled } = await import(
|
||||||
|
"../infra/unhandled-rejections.js"
|
||||||
|
);
|
||||||
|
|
||||||
const { buildProgram } = await import("../cli/program.js");
|
const { buildProgram } = await import("../cli/program.js");
|
||||||
const program = buildProgram();
|
const program = buildProgram();
|
||||||
|
|
||||||
process.on("unhandledRejection", (reason, _promise) => {
|
process.on("unhandledRejection", (reason, _promise) => {
|
||||||
|
if (isUnhandledRejectionHandled(reason)) return;
|
||||||
console.error(
|
console.error(
|
||||||
"[clawdbot] Unhandled promise rejection:",
|
"[clawdbot] Unhandled promise rejection:",
|
||||||
reason instanceof Error ? (reason.stack ?? reason.message) : reason,
|
reason instanceof Error ? (reason.stack ?? reason.message) : reason,
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import { chunkText, resolveTextChunkLimit } from "../auto-reply/chunk.js";
|
import { chunkText, resolveTextChunkLimit } from "../auto-reply/chunk.js";
|
||||||
import { hasControlCommand } from "../auto-reply/command-detection.js";
|
|
||||||
import { formatAgentEnvelope } from "../auto-reply/envelope.js";
|
import { formatAgentEnvelope } from "../auto-reply/envelope.js";
|
||||||
import {
|
import {
|
||||||
normalizeGroupActivation,
|
normalizeGroupActivation,
|
||||||
@@ -26,6 +25,7 @@ import {
|
|||||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||||
import { emitHeartbeatEvent } from "../infra/heartbeat-events.js";
|
import { emitHeartbeatEvent } from "../infra/heartbeat-events.js";
|
||||||
import { enqueueSystemEvent } from "../infra/system-events.js";
|
import { enqueueSystemEvent } from "../infra/system-events.js";
|
||||||
|
import { registerUnhandledRejectionHandler } from "../infra/unhandled-rejections.js";
|
||||||
import { createSubsystemLogger, getChildLogger } from "../logging.js";
|
import { createSubsystemLogger, getChildLogger } from "../logging.js";
|
||||||
import { defaultRuntime, type RuntimeEnv } from "../runtime.js";
|
import { defaultRuntime, type RuntimeEnv } from "../runtime.js";
|
||||||
import { isSelfChatMode, jidToE164, normalizeE164 } from "../utils.js";
|
import { isSelfChatMode, jidToE164, normalizeE164 } from "../utils.js";
|
||||||
@@ -49,6 +49,45 @@ const whatsappInboundLog = whatsappLog.child("inbound");
|
|||||||
const whatsappOutboundLog = whatsappLog.child("outbound");
|
const whatsappOutboundLog = whatsappLog.child("outbound");
|
||||||
const whatsappHeartbeatLog = whatsappLog.child("heartbeat");
|
const whatsappHeartbeatLog = whatsappLog.child("heartbeat");
|
||||||
|
|
||||||
|
const isLikelyWhatsAppCryptoError = (reason: unknown) => {
|
||||||
|
const formatReason = (value: unknown): string => {
|
||||||
|
if (value == null) return "";
|
||||||
|
if (typeof value === "string") return value;
|
||||||
|
if (value instanceof Error) {
|
||||||
|
return `${value.message}\n${value.stack ?? ""}`;
|
||||||
|
}
|
||||||
|
if (typeof value === "object") {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(value);
|
||||||
|
} catch {
|
||||||
|
return Object.prototype.toString.call(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof value === "number") return String(value);
|
||||||
|
if (typeof value === "boolean") return String(value);
|
||||||
|
if (typeof value === "bigint") return String(value);
|
||||||
|
if (typeof value === "symbol") return value.description ?? value.toString();
|
||||||
|
if (typeof value === "function")
|
||||||
|
return value.name ? `[function ${value.name}]` : "[function]";
|
||||||
|
return Object.prototype.toString.call(value);
|
||||||
|
};
|
||||||
|
const raw =
|
||||||
|
reason instanceof Error
|
||||||
|
? `${reason.message}\n${reason.stack ?? ""}`
|
||||||
|
: formatReason(reason);
|
||||||
|
const haystack = raw.toLowerCase();
|
||||||
|
const hasAuthError =
|
||||||
|
haystack.includes("unsupported state or unable to authenticate data") ||
|
||||||
|
haystack.includes("bad mac");
|
||||||
|
if (!hasAuthError) return false;
|
||||||
|
return (
|
||||||
|
haystack.includes("@whiskeysockets/baileys") ||
|
||||||
|
haystack.includes("baileys") ||
|
||||||
|
haystack.includes("noise-handler") ||
|
||||||
|
haystack.includes("aesdecryptgcm")
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
// Send via the active gateway-backed listener. The monitor already owns the single
|
// Send via the active gateway-backed listener. The monitor already owns the single
|
||||||
// Baileys session, so use its send API directly.
|
// Baileys session, so use its send API directly.
|
||||||
async function sendWithIpcFallback(
|
async function sendWithIpcFallback(
|
||||||
@@ -849,23 +888,35 @@ export async function monitorWebProvider(
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
const resolveCommandAllowFrom = () => {
|
const resolveOwnerList = (selfE164?: string | null) => {
|
||||||
const allowFrom = mentionConfig.allowFrom;
|
const allowFrom = mentionConfig.allowFrom;
|
||||||
const raw =
|
const raw =
|
||||||
Array.isArray(allowFrom) && allowFrom.length > 0 ? allowFrom : [];
|
Array.isArray(allowFrom) && allowFrom.length > 0
|
||||||
|
? allowFrom
|
||||||
|
: selfE164
|
||||||
|
? [selfE164]
|
||||||
|
: [];
|
||||||
return raw
|
return raw
|
||||||
.filter((entry): entry is string => Boolean(entry && entry !== "*"))
|
.filter((entry): entry is string => Boolean(entry && entry !== "*"))
|
||||||
.map((entry) => normalizeE164(entry))
|
.map((entry) => normalizeE164(entry))
|
||||||
.filter((entry): entry is string => Boolean(entry));
|
.filter((entry): entry is string => Boolean(entry));
|
||||||
};
|
};
|
||||||
|
|
||||||
const isCommandAuthorized = (msg: WebInboundMsg) => {
|
const isOwnerSender = (msg: WebInboundMsg) => {
|
||||||
const allowFrom = resolveCommandAllowFrom();
|
|
||||||
if (allowFrom.length === 0) return true;
|
|
||||||
if (mentionConfig.allowFrom?.includes("*")) return true;
|
|
||||||
const sender = normalizeE164(msg.senderE164 ?? "");
|
const sender = normalizeE164(msg.senderE164 ?? "");
|
||||||
if (!sender) return false;
|
if (!sender) return false;
|
||||||
return allowFrom.includes(sender);
|
const owners = resolveOwnerList(msg.selfE164 ?? undefined);
|
||||||
|
return owners.includes(sender);
|
||||||
|
};
|
||||||
|
|
||||||
|
const isStatusCommand = (body: string) => {
|
||||||
|
const trimmed = body.trim().toLowerCase();
|
||||||
|
if (!trimmed) return false;
|
||||||
|
return (
|
||||||
|
trimmed === "/status" ||
|
||||||
|
trimmed === "status" ||
|
||||||
|
trimmed.startsWith("/status ")
|
||||||
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
const stripMentionsForCommand = (text: string, selfE164?: string | null) => {
|
const stripMentionsForCommand = (text: string, selfE164?: string | null) => {
|
||||||
@@ -912,6 +963,7 @@ export async function monitorWebProvider(
|
|||||||
let lastMessageAt: number | null = null;
|
let lastMessageAt: number | null = null;
|
||||||
let handledMessages = 0;
|
let handledMessages = 0;
|
||||||
let _lastInboundMsg: WebInboundMsg | null = null;
|
let _lastInboundMsg: WebInboundMsg | null = null;
|
||||||
|
let unregisterUnhandled: (() => void) | null = null;
|
||||||
|
|
||||||
// Watchdog to detect stuck message processing (e.g., event emitter died)
|
// Watchdog to detect stuck message processing (e.g., event emitter died)
|
||||||
// Should be significantly longer than the reply heartbeat interval to avoid false positives
|
// Should be significantly longer than the reply heartbeat interval to avoid false positives
|
||||||
@@ -1182,7 +1234,6 @@ export async function monitorWebProvider(
|
|||||||
SenderName: msg.senderName,
|
SenderName: msg.senderName,
|
||||||
SenderE164: msg.senderE164,
|
SenderE164: msg.senderE164,
|
||||||
WasMentioned: msg.wasMentioned,
|
WasMentioned: msg.wasMentioned,
|
||||||
CommandAuthorized: isCommandAuthorized(msg),
|
|
||||||
Surface: "whatsapp",
|
Surface: "whatsapp",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1323,15 +1374,12 @@ export async function monitorWebProvider(
|
|||||||
noteGroupMember(conversationId, msg.senderE164, msg.senderName);
|
noteGroupMember(conversationId, msg.senderE164, msg.senderName);
|
||||||
const commandBody = stripMentionsForCommand(msg.body, msg.selfE164);
|
const commandBody = stripMentionsForCommand(msg.body, msg.selfE164);
|
||||||
const activationCommand = parseActivationCommand(commandBody);
|
const activationCommand = parseActivationCommand(commandBody);
|
||||||
const commandAuthorized = isCommandAuthorized(msg);
|
const isOwner = isOwnerSender(msg);
|
||||||
const statusCommand = hasControlCommand(commandBody);
|
const statusCommand = isStatusCommand(commandBody);
|
||||||
const hasAnyMention = (msg.mentionedJids?.length ?? 0) > 0;
|
|
||||||
const shouldBypassMention =
|
const shouldBypassMention =
|
||||||
commandAuthorized &&
|
isOwner && (activationCommand.hasCommand || statusCommand);
|
||||||
(activationCommand.hasCommand || statusCommand) &&
|
|
||||||
!hasAnyMention;
|
|
||||||
|
|
||||||
if (activationCommand.hasCommand && !commandAuthorized) {
|
if (activationCommand.hasCommand && !isOwner) {
|
||||||
logVerbose(
|
logVerbose(
|
||||||
`Ignoring /activation from non-owner in group ${conversationId}`,
|
`Ignoring /activation from non-owner in group ${conversationId}`,
|
||||||
);
|
);
|
||||||
@@ -1393,9 +1441,27 @@ export async function monitorWebProvider(
|
|||||||
);
|
);
|
||||||
|
|
||||||
setActiveWebListener(listener);
|
setActiveWebListener(listener);
|
||||||
|
unregisterUnhandled = registerUnhandledRejectionHandler((reason) => {
|
||||||
|
if (!isLikelyWhatsAppCryptoError(reason)) return false;
|
||||||
|
const errorStr = formatError(reason);
|
||||||
|
reconnectLogger.warn(
|
||||||
|
{ connectionId, error: errorStr },
|
||||||
|
"web reconnect: unhandled rejection from WhatsApp socket; forcing reconnect",
|
||||||
|
);
|
||||||
|
listener.signalClose?.({
|
||||||
|
status: 499,
|
||||||
|
isLoggedOut: false,
|
||||||
|
error: reason,
|
||||||
|
});
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
|
||||||
const closeListener = async () => {
|
const closeListener = async () => {
|
||||||
setActiveWebListener(null);
|
setActiveWebListener(null);
|
||||||
|
if (unregisterUnhandled) {
|
||||||
|
unregisterUnhandled();
|
||||||
|
unregisterUnhandled = null;
|
||||||
|
}
|
||||||
if (heartbeat) clearInterval(heartbeat);
|
if (heartbeat) clearInterval(heartbeat);
|
||||||
if (watchdogTimer) clearInterval(watchdogTimer);
|
if (watchdogTimer) clearInterval(watchdogTimer);
|
||||||
if (backgroundTasks.size > 0) {
|
if (backgroundTasks.size > 0) {
|
||||||
|
|||||||
Reference in New Issue
Block a user