fix: dedupe inbound messages across providers

This commit is contained in:
Peter Steinberger
2026-01-11 00:12:17 +01:00
parent bd2002010c
commit 7c76561569
18 changed files with 353 additions and 53 deletions

View File

@@ -17,6 +17,7 @@ vi.mock("../agents/pi-embedded.js", () => ({
}));
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js";
import { getReplyFromConfig } from "../auto-reply/reply.js";
import type { ClawdbotConfig } from "../config/config.js";
import { resetLogger, setLoggerOverride } from "../logging.js";
@@ -57,6 +58,7 @@ const rmDirWithRetries = async (dir: string): Promise<void> => {
};
beforeEach(async () => {
resetInboundDedupe();
previousHome = process.env.HOME;
tempHome = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-web-home-"));
process.env.HOME = tempHome;

View File

@@ -106,11 +106,12 @@ vi.mock("./session.js", () => {
};
});
import { monitorWebInbox } from "./inbound.js";
import { monitorWebInbox, resetWebInboundDedupe } from "./inbound.js";
describe("web inbound media saves with extension", () => {
beforeEach(() => {
saveMediaBufferSpy.mockClear();
resetWebInboundDedupe();
});
beforeAll(async () => {

View File

@@ -14,6 +14,7 @@ import {
import { loadConfig } from "../config/config.js";
import { logVerbose, shouldLogVerbose } from "../globals.js";
import { createDedupeCache } from "../infra/dedupe.js";
import { recordProviderActivity } from "../infra/provider-activity.js";
import { createSubsystemLogger, getChildLogger } from "../logging.js";
import { saveMediaBuffer } from "../media/store.js";
@@ -48,6 +49,17 @@ export type WebListenerCloseReason = {
error?: unknown;
};
const RECENT_WEB_MESSAGE_TTL_MS = 20 * 60_000;
const RECENT_WEB_MESSAGE_MAX = 5000;
const recentInboundMessages = createDedupeCache({
ttlMs: RECENT_WEB_MESSAGE_TTL_MS,
maxSize: RECENT_WEB_MESSAGE_MAX,
});
export function resetWebInboundDedupe(): void {
recentInboundMessages.clear();
}
export type WebInboundMessage = {
id?: string;
from: string; // conversation id: E.164 for direct chats, group JID for groups
@@ -117,7 +129,6 @@ export async function monitorWebInbox(options: {
}
const selfJid = sock.user?.id;
const selfE164 = selfJid ? jidToE164(selfJid) : null;
const seen = new Set<string>();
const groupMetaCache = new Map<
string,
{ subject?: string; participants?: string[]; expires: number }
@@ -169,9 +180,6 @@ export async function monitorWebInbox(options: {
direction: "inbound",
});
const id = msg.key?.id ?? undefined;
// De-dupe on message id; Baileys can emit retries.
if (id && seen.has(id)) continue;
if (id) seen.add(id);
// Note: not filtering fromMe here - echo detection happens in auto-reply layer
const remoteJid = msg.key?.remoteJid;
if (!remoteJid) continue;
@@ -179,6 +187,10 @@ export async function monitorWebInbox(options: {
if (remoteJid.endsWith("@status") || remoteJid.endsWith("@broadcast"))
continue;
const group = isJidGroup(remoteJid);
if (id) {
const dedupeKey = `${options.accountId}:${remoteJid}:${id}`;
if (recentInboundMessages.check(dedupeKey)) continue;
}
const participantJid = msg.key?.participant ?? undefined;
const from = group ? remoteJid : await resolveInboundJid(remoteJid);
// Skip if we still can't resolve an id to key conversation

View File

@@ -76,7 +76,7 @@ import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { resetLogger, setLoggerOverride } from "../logging.js";
import { monitorWebInbox } from "./inbound.js";
import { monitorWebInbox, resetWebInboundDedupe } from "./inbound.js";
const ACCOUNT_ID = "default";
let authDir: string;
@@ -89,6 +89,7 @@ describe("web monitor inbox", () => {
code: "PAIRCODE",
created: true,
});
resetWebInboundDedupe();
authDir = fsSync.mkdtempSync(path.join(os.tmpdir(), "clawdbot-auth-"));
});
@@ -151,6 +152,39 @@ describe("web monitor inbox", () => {
await listener.close();
});
it("deduplicates redelivered messages by id", async () => {
const onMessage = vi.fn(async () => {
return;
});
const listener = await monitorWebInbox({
verbose: false,
onMessage,
accountId: ACCOUNT_ID,
authDir,
});
const sock = await createWaSocket();
const upsert = {
type: "notify",
messages: [
{
key: { id: "abc", fromMe: false, remoteJid: "999@s.whatsapp.net" },
message: { conversation: "ping" },
messageTimestamp: 1_700_000_000,
pushName: "Tester",
},
],
};
sock.ev.emit("messages.upsert", upsert);
sock.ev.emit("messages.upsert", upsert);
await new Promise((resolve) => setImmediate(resolve));
expect(onMessage).toHaveBeenCalledTimes(1);
await listener.close();
});
it("resolves LID JIDs using Baileys LID mapping store", async () => {
const onMessage = vi.fn(async () => {
return;