From b759cb6f37697ceadf3d9aae0d6480244fe20a38 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 6 Jan 2026 06:30:12 +0100 Subject: [PATCH] feat(providers): normalize location parsing --- CHANGELOG.md | 1 + docs/index.md | 1 + docs/location.md | 46 ++++++++++++++++ src/providers/location.test.ts | 60 +++++++++++++++++++++ src/providers/location.ts | 78 +++++++++++++++++++++++++++ src/telegram/bot.media.test.ts | 81 ++++++++++++++++++++++++++++ src/telegram/bot.ts | 97 ++++++++++++---------------------- src/web/auto-reply.ts | 2 + src/web/inbound.test.ts | 48 ++++++++++++++++- src/web/inbound.ts | 73 ++++++++++++++++++++++++- 10 files changed, 421 insertions(+), 66 deletions(-) create mode 100644 docs/location.md create mode 100644 src/providers/location.test.ts create mode 100644 src/providers/location.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6412fda5e..5397f0f60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ - Telegram: honor routing.groupChat.mentionPatterns for group mention gating. Thanks Kevin Kern (@regenrek) for PR #242. - Telegram: gate groups via `telegram.groups` allowlist (align with WhatsApp/iMessage). Thanks @kitze for PR #241. - Telegram: support media groups (multi-image messages). Thanks @obviyus for PR #220. +- Telegram/WhatsApp: parse shared locations (pins, places, live) and expose structured ctx fields. Thanks @nachoiacovino for PR #194. - Auto-reply: block unauthorized `/reset` and infer WhatsApp senders from E.164 inputs. - Auto-reply: track compaction count in session status; verbose mode announces auto-compactions. - Telegram: send GIF media as animations (auto-play) and improve filename sniffing. diff --git a/docs/index.md b/docs/index.md index 76893a0b7..ef0abd887 100644 --- a/docs/index.md +++ b/docs/index.md @@ -181,6 +181,7 @@ Example: ## Core Contributors - **Maxim Vovshin** (@Hyaxia, 36747317+Hyaxia@users.noreply.github.com) — Blogwatcher skill +- **Nacho Iacovino** (@nachoiacovino, nacho.iacovino@gmail.com) — Location parsing (Telegram + WhatsApp) ## License diff --git a/docs/location.md b/docs/location.md new file mode 100644 index 000000000..7d610e7ff --- /dev/null +++ b/docs/location.md @@ -0,0 +1,46 @@ +--- +summary: "Inbound provider location parsing (Telegram + WhatsApp) and context fields" +read_when: + - Adding or modifying provider location parsing + - Using location context fields in agent prompts or tools +--- + +# Provider location parsing + +Clawdbot normalizes shared locations from chat providers into: +- human-readable text appended to the inbound body, and +- structured fields in the auto-reply context payload. + +Currently supported: +- **Telegram** (location pins + venues + live locations) +- **WhatsApp** (locationMessage + liveLocationMessage) + +## Text formatting +Locations are rendered as friendly lines without brackets: + +- Pin: + - `📍 48.858844, 2.294351 ±12m` +- Named place: + - `📍 Eiffel Tower — Champ de Mars, Paris (48.858844, 2.294351 ±12m)` +- Live share: + - `🛰 Live location: 48.858844, 2.294351 ±12m` + +If the provider includes a caption/comment, it is appended on the next line: +``` +📍 48.858844, 2.294351 ±12m +Meet here +``` + +## Context fields +When a location is present, these fields are added to `ctx`: +- `LocationLat` (number) +- `LocationLon` (number) +- `LocationAccuracy` (number, meters; optional) +- `LocationName` (string; optional) +- `LocationAddress` (string; optional) +- `LocationSource` (`pin | place | live`) +- `LocationIsLive` (boolean) + +## Provider notes +- **Telegram**: venues map to `LocationName/LocationAddress`; live locations use `live_period`. +- **WhatsApp**: `locationMessage.comment` and `liveLocationMessage.caption` are appended as the caption line. diff --git a/src/providers/location.test.ts b/src/providers/location.test.ts new file mode 100644 index 000000000..1db7e2115 --- /dev/null +++ b/src/providers/location.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { formatLocationText, toLocationContext } from "./location.js"; + +describe("provider location helpers", () => { + it("formats pin locations with accuracy", () => { + const text = formatLocationText({ + latitude: 48.858844, + longitude: 2.294351, + accuracy: 12, + }); + expect(text).toBe("📍 48.858844, 2.294351 ±12m"); + }); + + it("formats named places with address and caption", () => { + const text = formatLocationText({ + latitude: 40.689247, + longitude: -74.044502, + name: "Statue of Liberty", + address: "Liberty Island, NY", + accuracy: 8, + caption: "Bring snacks", + }); + expect(text).toBe( + "📍 Statue of Liberty — Liberty Island, NY (40.689247, -74.044502 ±8m)\nBring snacks", + ); + }); + + it("formats live locations with live label", () => { + const text = formatLocationText({ + latitude: 37.819929, + longitude: -122.478255, + accuracy: 20, + caption: "On the move", + isLive: true, + source: "live", + }); + expect(text).toBe( + "🛰 Live location: 37.819929, -122.478255 ±20m\nOn the move", + ); + }); + + it("builds ctx fields with normalized source", () => { + const ctx = toLocationContext({ + latitude: 1, + longitude: 2, + name: "Cafe", + address: "Main St", + }); + expect(ctx).toEqual({ + LocationLat: 1, + LocationLon: 2, + LocationAccuracy: undefined, + LocationName: "Cafe", + LocationAddress: "Main St", + LocationSource: "place", + LocationIsLive: false, + }); + }); +}); diff --git a/src/providers/location.ts b/src/providers/location.ts new file mode 100644 index 000000000..6cc4997ef --- /dev/null +++ b/src/providers/location.ts @@ -0,0 +1,78 @@ +export type LocationSource = "pin" | "place" | "live"; + +export type NormalizedLocation = { + latitude: number; + longitude: number; + accuracy?: number; + name?: string; + address?: string; + isLive?: boolean; + source?: LocationSource; + caption?: string; +}; + +type ResolvedLocation = NormalizedLocation & { + source: LocationSource; + isLive: boolean; +}; + +function resolveLocation(location: NormalizedLocation): ResolvedLocation { + const source = + location.source ?? + (location.isLive + ? "live" + : location.name || location.address + ? "place" + : "pin"); + const isLive = Boolean(location.isLive ?? source === "live"); + return { ...location, source, isLive }; +} + +function formatAccuracy(accuracy?: number): string { + if (!Number.isFinite(accuracy)) return ""; + return ` ±${Math.round(accuracy ?? 0)}m`; +} + +function formatCoords(latitude: number, longitude: number): string { + return `${latitude.toFixed(6)}, ${longitude.toFixed(6)}`; +} + +export function formatLocationText(location: NormalizedLocation): string { + const resolved = resolveLocation(location); + const coords = formatCoords(resolved.latitude, resolved.longitude); + const accuracy = formatAccuracy(resolved.accuracy); + const caption = resolved.caption?.trim(); + let header = ""; + + if (resolved.source === "live" || resolved.isLive) { + header = `🛰 Live location: ${coords}${accuracy}`; + } else if (resolved.name || resolved.address) { + const label = [resolved.name, resolved.address].filter(Boolean).join(" — "); + header = `📍 ${label} (${coords}${accuracy})`; + } else { + header = `📍 ${coords}${accuracy}`; + } + + return caption ? `${header}\n${caption}` : header; +} + +export function toLocationContext(location: NormalizedLocation): { + LocationLat: number; + LocationLon: number; + LocationAccuracy?: number; + LocationName?: string; + LocationAddress?: string; + LocationSource: LocationSource; + LocationIsLive: boolean; +} { + const resolved = resolveLocation(location); + return { + LocationLat: resolved.latitude, + LocationLon: resolved.longitude, + LocationAccuracy: resolved.accuracy, + LocationName: resolved.name, + LocationAddress: resolved.address, + LocationSource: resolved.source, + LocationIsLive: resolved.isLive, + }; +} diff --git a/src/telegram/bot.media.test.ts b/src/telegram/bot.media.test.ts index 6f94e7cf8..09ca06c20 100644 --- a/src/telegram/bot.media.test.ts +++ b/src/telegram/bot.media.test.ts @@ -341,3 +341,84 @@ describe("telegram media groups", () => { fetchSpy.mockRestore(); }, 2000); }); + +describe("telegram location parsing", () => { + it("includes location text and ctx fields for pins", async () => { + const { createTelegramBot } = await import("./bot.js"); + const replyModule = await import("../auto-reply/reply.js"); + const replySpy = replyModule.__replySpy as unknown as ReturnType< + typeof vi.fn + >; + + onSpy.mockReset(); + replySpy.mockReset(); + + createTelegramBot({ token: "tok" }); + const handler = onSpy.mock.calls[0]?.[1] as ( + ctx: Record, + ) => Promise; + + await handler({ + message: { + chat: { id: 42, type: "private" }, + message_id: 5, + caption: "Meet here", + date: 1736380800, + location: { + latitude: 48.858844, + longitude: 2.294351, + horizontal_accuracy: 12, + }, + }, + me: { username: "clawdbot_bot" }, + getFile: async () => ({ file_path: "unused" }), + }); + + expect(replySpy).toHaveBeenCalledTimes(1); + const payload = replySpy.mock.calls[0][0]; + expect(payload.Body).toContain("Meet here"); + expect(payload.Body).toContain("48.858844"); + expect(payload.LocationLat).toBe(48.858844); + expect(payload.LocationLon).toBe(2.294351); + expect(payload.LocationSource).toBe("pin"); + expect(payload.LocationIsLive).toBe(false); + }); + + it("captures venue fields for named places", async () => { + const { createTelegramBot } = await import("./bot.js"); + const replyModule = await import("../auto-reply/reply.js"); + const replySpy = replyModule.__replySpy as unknown as ReturnType< + typeof vi.fn + >; + + onSpy.mockReset(); + replySpy.mockReset(); + + createTelegramBot({ token: "tok" }); + const handler = onSpy.mock.calls[0]?.[1] as ( + ctx: Record, + ) => Promise; + + await handler({ + message: { + chat: { id: 42, type: "private" }, + message_id: 6, + date: 1736380800, + venue: { + title: "Eiffel Tower", + address: "Champ de Mars, Paris", + location: { latitude: 48.858844, longitude: 2.294351 }, + }, + }, + me: { username: "clawdbot_bot" }, + getFile: async () => ({ file_path: "unused" }), + }); + + expect(replySpy).toHaveBeenCalledTimes(1); + const payload = replySpy.mock.calls[0][0]; + expect(payload.Body).toContain("Eiffel Tower"); + expect(payload.LocationName).toBe("Eiffel Tower"); + expect(payload.LocationAddress).toBe("Champ de Mars, Paris"); + expect(payload.LocationSource).toBe("place"); + }); +}); diff --git a/src/telegram/bot.ts b/src/telegram/bot.ts index 58a00060c..6fe351080 100644 --- a/src/telegram/bot.ts +++ b/src/telegram/bot.ts @@ -28,6 +28,11 @@ import { getChildLogger } from "../logging.js"; import { mediaKindFromMime } from "../media/constants.js"; import { detectMime, isGifMedia } from "../media/mime.js"; import { saveMediaBuffer } from "../media/store.js"; +import { + formatLocationText, + type NormalizedLocation, + toLocationContext, +} from "../providers/location.js"; import type { RuntimeEnv } from "../runtime.js"; import { loadWebMedia } from "../web/media.js"; @@ -68,12 +73,6 @@ interface TelegramVenue { google_place_type?: string; } -/** Extended message type that may include location/venue */ -type TelegramMessageWithLocation = TelegramMessage & { - location?: TelegramLocation; - venue?: TelegramVenue; -}; - type TelegramContext = { message: TelegramMessage; me?: { username?: string }; @@ -252,15 +251,13 @@ export function createTelegramBot(opts: TelegramBotOptions) { else if (msg.document) placeholder = ""; const replyTarget = describeReplyTarget(msg); - const locationText = formatLocationMessage( - msg as TelegramMessageWithLocation, - ); - const rawBody = ( - msg.text ?? - msg.caption ?? - locationText ?? - placeholder - ).trim(); + const locationData = extractTelegramLocation(msg); + const locationText = locationData + ? formatLocationText(locationData) + : undefined; + const rawText = (msg.text ?? msg.caption ?? "").trim(); + let rawBody = [rawText, locationText].filter(Boolean).join("\n").trim(); + if (!rawBody) rawBody = placeholder; if (!rawBody && allMedia.length === 0) return; let bodyText = rawBody; @@ -282,8 +279,6 @@ export function createTelegramBot(opts: TelegramBotOptions) { body: `${bodyText}${replySuffix}`, }); - const locationData = extractLocationData(msg); - const ctxPayload = { Body: body, From: isGroup ? `group:${chatId}` : `telegram:${chatId}`, @@ -309,11 +304,7 @@ export function createTelegramBot(opts: TelegramBotOptions) { allMedia.length > 0 ? (allMedia.map((m) => m.contentType).filter(Boolean) as string[]) : undefined, - LocationLat: locationData?.latitude, - LocationLon: locationData?.longitude, - LocationAccuracy: locationData?.accuracy, - VenueName: locationData?.venueName, - VenueAddress: locationData?.venueAddress, + ...(locationData ? toLocationContext(locationData) : undefined), CommandAuthorized: commandAuthorized, }; @@ -739,10 +730,10 @@ function describeReplyTarget(msg: TelegramMessage) { else if (reply.video) body = ""; else if (reply.audio || reply.voice) body = ""; else if (reply.document) body = ""; - else if ((reply as TelegramMessageWithLocation).location) - body = - formatLocationMessage(reply as TelegramMessageWithLocation) ?? - ""; + else { + const locationData = extractTelegramLocation(reply); + if (locationData) body = formatLocationText(locationData); + } } if (!body) return null; const sender = buildSenderName(reply); @@ -754,60 +745,38 @@ function describeReplyTarget(msg: TelegramMessage) { }; } -/** - * Extract structured location data from a message. - */ -function extractLocationData(msg: TelegramMessage): { - latitude: number; - longitude: number; - accuracy?: number; - venueName?: string; - venueAddress?: string; -} | null { - const msgWithLocation = msg as TelegramMessageWithLocation; +function extractTelegramLocation( + msg: TelegramMessage, +): NormalizedLocation | null { + const msgWithLocation = msg as { + location?: TelegramLocation; + venue?: TelegramVenue; + }; const { venue, location } = msgWithLocation; if (venue) { return { latitude: venue.location.latitude, longitude: venue.location.longitude, - venueName: venue.title, - venueAddress: venue.address, + accuracy: venue.location.horizontal_accuracy, + name: venue.title, + address: venue.address, + source: "place", + isLive: false, }; } if (location) { + const isLive = + typeof location.live_period === "number" && location.live_period > 0; return { latitude: location.latitude, longitude: location.longitude, accuracy: location.horizontal_accuracy, + source: isLive ? "live" : "pin", + isLive, }; } return null; } - -/** - * Format location or venue message into text. - * Handles both raw location shares and venue shares (places with names). - */ -function formatLocationMessage( - msg: TelegramMessageWithLocation, -): string | null { - const { venue, location } = msg; - - if (venue) { - const { latitude, longitude } = venue.location; - return `[Venue: ${venue.title} - ${venue.address} (${latitude.toFixed(6)}, ${longitude.toFixed(6)})]`; - } - - if (location) { - const { latitude, longitude, horizontal_accuracy } = location; - const accuracy = horizontal_accuracy - ? ` ±${Math.round(horizontal_accuracy)}m` - : ""; - return `[Location: ${latitude.toFixed(6)}, ${longitude.toFixed(6)}${accuracy}]`; - } - - return null; -} diff --git a/src/web/auto-reply.ts b/src/web/auto-reply.ts index 8ac1a0002..86d65cfce 100644 --- a/src/web/auto-reply.ts +++ b/src/web/auto-reply.ts @@ -39,6 +39,7 @@ import { emitHeartbeatEvent } from "../infra/heartbeat-events.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { registerUnhandledRejectionHandler } from "../infra/unhandled-rejections.js"; import { createSubsystemLogger, getChildLogger } from "../logging.js"; +import { toLocationContext } from "../providers/location.js"; import { defaultRuntime, type RuntimeEnv } from "../runtime.js"; import { isSelfChatMode, jidToE164, normalizeE164 } from "../utils.js"; import { setActiveWebListener } from "./active-listener.js"; @@ -1216,6 +1217,7 @@ export async function monitorWebProvider( SenderName: msg.senderName, SenderE164: msg.senderE164, WasMentioned: msg.wasMentioned, + ...(msg.location ? toLocationContext(msg.location) : {}), Surface: "whatsapp", }, cfg, diff --git a/src/web/inbound.test.ts b/src/web/inbound.test.ts index 161b0d62b..6efcfa9e0 100644 --- a/src/web/inbound.test.ts +++ b/src/web/inbound.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; -import { extractMediaPlaceholder, extractText } from "./inbound.js"; +import { + extractLocationData, + extractMediaPlaceholder, + extractText, +} from "./inbound.js"; describe("web inbound helpers", () => { it("prefers the main conversation body", () => { @@ -45,4 +49,46 @@ describe("web inbound helpers", () => { } as unknown as import("@whiskeysockets/baileys").proto.IMessage), ).toBe(""); }); + + it("extracts WhatsApp location messages", () => { + const location = extractLocationData({ + locationMessage: { + degreesLatitude: 48.858844, + degreesLongitude: 2.294351, + name: "Eiffel Tower", + address: "Champ de Mars, Paris", + accuracyInMeters: 12, + comment: "Meet here", + }, + } as unknown as import("@whiskeysockets/baileys").proto.IMessage); + expect(location).toEqual({ + latitude: 48.858844, + longitude: 2.294351, + accuracy: 12, + name: "Eiffel Tower", + address: "Champ de Mars, Paris", + caption: "Meet here", + source: "place", + isLive: false, + }); + }); + + it("extracts WhatsApp live location messages", () => { + const location = extractLocationData({ + liveLocationMessage: { + degreesLatitude: 37.819929, + degreesLongitude: -122.478255, + accuracyInMeters: 20, + caption: "On the move", + }, + } as unknown as import("@whiskeysockets/baileys").proto.IMessage); + expect(location).toEqual({ + latitude: 37.819929, + longitude: -122.478255, + accuracy: 20, + caption: "On the move", + source: "live", + isLive: true, + }); + }); }); diff --git a/src/web/inbound.ts b/src/web/inbound.ts index 545ad8d63..9c1c81659 100644 --- a/src/web/inbound.ts +++ b/src/web/inbound.ts @@ -16,6 +16,10 @@ import { loadConfig } from "../config/config.js"; import { logVerbose, shouldLogVerbose } from "../globals.js"; import { createSubsystemLogger, getChildLogger } from "../logging.js"; import { saveMediaBuffer } from "../media/store.js"; +import { + formatLocationText, + type NormalizedLocation, +} from "../providers/location.js"; import { isSelfChatMode, jidToE164, @@ -56,6 +60,7 @@ export type WebInboundMessage = { mentionedJids?: string[]; selfJid?: string | null; selfE164?: string | null; + location?: NormalizedLocation; sendComposing: () => Promise; reply: (text: string) => Promise; sendMedia: (payload: AnyMessageContent) => Promise; @@ -241,7 +246,12 @@ export async function monitorWebInbox(options: { // but we skip triggering the auto-reply logic to avoid spamming old context. if (upsert.type === "append") continue; + const location = extractLocationData(msg.message ?? undefined); + const locationText = location ? formatLocationText(location) : undefined; let body = extractText(msg.message ?? undefined); + if (locationText) { + body = [body, locationText].filter(Boolean).join("\n").trim(); + } if (!body) { body = extractMediaPlaceholder(msg.message ?? undefined); if (!body) continue; @@ -319,6 +329,7 @@ export async function monitorWebInbox(options: { mentionedJids: mentionedJids ?? undefined, selfJid, selfE164, + location: location ?? undefined, sendComposing, reply, sendMedia, @@ -598,6 +609,62 @@ export function extractMediaPlaceholder( return undefined; } +export function extractLocationData( + rawMessage: proto.IMessage | undefined, +): NormalizedLocation | null { + const message = unwrapMessage(rawMessage); + if (!message) return null; + + const live = message.liveLocationMessage ?? undefined; + if (live) { + const latitudeRaw = live.degreesLatitude; + const longitudeRaw = live.degreesLongitude; + if (latitudeRaw != null && longitudeRaw != null) { + const latitude = Number(latitudeRaw); + const longitude = Number(longitudeRaw); + if (Number.isFinite(latitude) && Number.isFinite(longitude)) { + return { + latitude, + longitude, + accuracy: live.accuracyInMeters ?? undefined, + caption: live.caption ?? undefined, + source: "live", + isLive: true, + }; + } + } + } + + const location = message.locationMessage ?? undefined; + if (location) { + const latitudeRaw = location.degreesLatitude; + const longitudeRaw = location.degreesLongitude; + if (latitudeRaw != null && longitudeRaw != null) { + const latitude = Number(latitudeRaw); + const longitude = Number(longitudeRaw); + if (Number.isFinite(latitude) && Number.isFinite(longitude)) { + const isLive = Boolean(location.isLive); + return { + latitude, + longitude, + accuracy: location.accuracyInMeters ?? undefined, + name: location.name ?? undefined, + address: location.address ?? undefined, + caption: location.comment ?? undefined, + source: isLive + ? "live" + : location.name || location.address + ? "place" + : "pin", + isLive, + }; + } + } + } + + return null; +} + function describeReplyContext(rawMessage: proto.IMessage | undefined): { id?: string; body: string; @@ -610,7 +677,11 @@ function describeReplyContext(rawMessage: proto.IMessage | undefined): { contextInfo?.quotedMessage as proto.IMessage | undefined, ) as proto.IMessage | undefined; if (!quoted) return null; - const body = extractText(quoted) ?? extractMediaPlaceholder(quoted); + const location = extractLocationData(quoted); + const locationText = location ? formatLocationText(location) : undefined; + const text = extractText(quoted); + let body = [text, locationText].filter(Boolean).join("\n").trim(); + if (!body) body = extractMediaPlaceholder(quoted); if (!body) { const quotedType = quoted ? getContentType(quoted) : undefined; logVerbose(