fix: dedupe inbound messages across providers
This commit is contained in:
@@ -17,6 +17,7 @@ vi.mock("../agents/pi-embedded.js", () => ({
|
||||
}));
|
||||
|
||||
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
|
||||
import { resetInboundDedupe } from "../auto-reply/reply/inbound-dedupe.js";
|
||||
import { getReplyFromConfig } from "../auto-reply/reply.js";
|
||||
import type { ClawdbotConfig } from "../config/config.js";
|
||||
import { resetLogger, setLoggerOverride } from "../logging.js";
|
||||
@@ -57,6 +58,7 @@ const rmDirWithRetries = async (dir: string): Promise<void> => {
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
resetInboundDedupe();
|
||||
previousHome = process.env.HOME;
|
||||
tempHome = await fs.mkdtemp(path.join(os.tmpdir(), "clawdbot-web-home-"));
|
||||
process.env.HOME = tempHome;
|
||||
|
||||
@@ -106,11 +106,12 @@ vi.mock("./session.js", () => {
|
||||
};
|
||||
});
|
||||
|
||||
import { monitorWebInbox } from "./inbound.js";
|
||||
import { monitorWebInbox, resetWebInboundDedupe } from "./inbound.js";
|
||||
|
||||
describe("web inbound media saves with extension", () => {
|
||||
beforeEach(() => {
|
||||
saveMediaBufferSpy.mockClear();
|
||||
resetWebInboundDedupe();
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
|
||||
import { loadConfig } from "../config/config.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import { createDedupeCache } from "../infra/dedupe.js";
|
||||
import { recordProviderActivity } from "../infra/provider-activity.js";
|
||||
import { createSubsystemLogger, getChildLogger } from "../logging.js";
|
||||
import { saveMediaBuffer } from "../media/store.js";
|
||||
@@ -48,6 +49,17 @@ export type WebListenerCloseReason = {
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
const RECENT_WEB_MESSAGE_TTL_MS = 20 * 60_000;
|
||||
const RECENT_WEB_MESSAGE_MAX = 5000;
|
||||
const recentInboundMessages = createDedupeCache({
|
||||
ttlMs: RECENT_WEB_MESSAGE_TTL_MS,
|
||||
maxSize: RECENT_WEB_MESSAGE_MAX,
|
||||
});
|
||||
|
||||
export function resetWebInboundDedupe(): void {
|
||||
recentInboundMessages.clear();
|
||||
}
|
||||
|
||||
export type WebInboundMessage = {
|
||||
id?: string;
|
||||
from: string; // conversation id: E.164 for direct chats, group JID for groups
|
||||
@@ -117,7 +129,6 @@ export async function monitorWebInbox(options: {
|
||||
}
|
||||
const selfJid = sock.user?.id;
|
||||
const selfE164 = selfJid ? jidToE164(selfJid) : null;
|
||||
const seen = new Set<string>();
|
||||
const groupMetaCache = new Map<
|
||||
string,
|
||||
{ subject?: string; participants?: string[]; expires: number }
|
||||
@@ -169,9 +180,6 @@ export async function monitorWebInbox(options: {
|
||||
direction: "inbound",
|
||||
});
|
||||
const id = msg.key?.id ?? undefined;
|
||||
// De-dupe on message id; Baileys can emit retries.
|
||||
if (id && seen.has(id)) continue;
|
||||
if (id) seen.add(id);
|
||||
// Note: not filtering fromMe here - echo detection happens in auto-reply layer
|
||||
const remoteJid = msg.key?.remoteJid;
|
||||
if (!remoteJid) continue;
|
||||
@@ -179,6 +187,10 @@ export async function monitorWebInbox(options: {
|
||||
if (remoteJid.endsWith("@status") || remoteJid.endsWith("@broadcast"))
|
||||
continue;
|
||||
const group = isJidGroup(remoteJid);
|
||||
if (id) {
|
||||
const dedupeKey = `${options.accountId}:${remoteJid}:${id}`;
|
||||
if (recentInboundMessages.check(dedupeKey)) continue;
|
||||
}
|
||||
const participantJid = msg.key?.participant ?? undefined;
|
||||
const from = group ? remoteJid : await resolveInboundJid(remoteJid);
|
||||
// Skip if we still can't resolve an id to key conversation
|
||||
|
||||
@@ -76,7 +76,7 @@ import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { resetLogger, setLoggerOverride } from "../logging.js";
|
||||
import { monitorWebInbox } from "./inbound.js";
|
||||
import { monitorWebInbox, resetWebInboundDedupe } from "./inbound.js";
|
||||
|
||||
const ACCOUNT_ID = "default";
|
||||
let authDir: string;
|
||||
@@ -89,6 +89,7 @@ describe("web monitor inbox", () => {
|
||||
code: "PAIRCODE",
|
||||
created: true,
|
||||
});
|
||||
resetWebInboundDedupe();
|
||||
authDir = fsSync.mkdtempSync(path.join(os.tmpdir(), "clawdbot-auth-"));
|
||||
});
|
||||
|
||||
@@ -151,6 +152,39 @@ describe("web monitor inbox", () => {
|
||||
await listener.close();
|
||||
});
|
||||
|
||||
it("deduplicates redelivered messages by id", async () => {
|
||||
const onMessage = vi.fn(async () => {
|
||||
return;
|
||||
});
|
||||
|
||||
const listener = await monitorWebInbox({
|
||||
verbose: false,
|
||||
onMessage,
|
||||
accountId: ACCOUNT_ID,
|
||||
authDir,
|
||||
});
|
||||
const sock = await createWaSocket();
|
||||
const upsert = {
|
||||
type: "notify",
|
||||
messages: [
|
||||
{
|
||||
key: { id: "abc", fromMe: false, remoteJid: "999@s.whatsapp.net" },
|
||||
message: { conversation: "ping" },
|
||||
messageTimestamp: 1_700_000_000,
|
||||
pushName: "Tester",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
sock.ev.emit("messages.upsert", upsert);
|
||||
sock.ev.emit("messages.upsert", upsert);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(onMessage).toHaveBeenCalledTimes(1);
|
||||
|
||||
await listener.close();
|
||||
});
|
||||
|
||||
it("resolves LID JIDs using Baileys LID mapping store", async () => {
|
||||
const onMessage = vi.fn(async () => {
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user