diff --git a/src/msteams/attachments.test.ts b/src/msteams/attachments.test.ts new file mode 100644 index 000000000..1c690e580 --- /dev/null +++ b/src/msteams/attachments.test.ts @@ -0,0 +1,179 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { + buildMSTeamsAttachmentPlaceholder, + buildMSTeamsMediaPayload, + downloadMSTeamsImageAttachments, +} from "./attachments.js"; + +const detectMimeMock = vi.fn(async () => "image/png"); +const saveMediaBufferMock = vi.fn(async () => ({ + path: "/tmp/saved.png", + contentType: "image/png", +})); + +vi.mock("../media/mime.js", () => ({ + detectMime: (...args: unknown[]) => detectMimeMock(...args), +})); + +vi.mock("../media/store.js", () => ({ + saveMediaBuffer: (...args: unknown[]) => saveMediaBufferMock(...args), +})); + +describe("msteams attachments", () => { + beforeEach(() => { + detectMimeMock.mockClear(); + saveMediaBufferMock.mockClear(); + }); + + describe("buildMSTeamsAttachmentPlaceholder", () => { + it("returns empty string when no attachments", () => { + expect(buildMSTeamsAttachmentPlaceholder(undefined)).toBe(""); + expect(buildMSTeamsAttachmentPlaceholder([])).toBe(""); + }); + + it("returns image placeholder for image attachments", () => { + expect( + buildMSTeamsAttachmentPlaceholder([ + { contentType: "image/png", contentUrl: "https://x/img.png" }, + ]), + ).toBe(""); + expect( + buildMSTeamsAttachmentPlaceholder([ + { contentType: "image/png", contentUrl: "https://x/1.png" }, + { contentType: "image/jpeg", contentUrl: "https://x/2.jpg" }, + ]), + ).toBe(" (2 images)"); + }); + + it("treats Teams file.download.info image attachments as images", () => { + expect( + buildMSTeamsAttachmentPlaceholder([ + { + contentType: "application/vnd.microsoft.teams.file.download.info", + content: { downloadUrl: "https://x/dl", fileType: "png" }, + }, + ]), + ).toBe(""); + }); + + it("returns document placeholder for non-image attachments", () => { + expect( + buildMSTeamsAttachmentPlaceholder([ + { contentType: "application/pdf", contentUrl: "https://x/x.pdf" }, + ]), + ).toBe(""); + expect( + buildMSTeamsAttachmentPlaceholder([ + { contentType: "application/pdf", contentUrl: "https://x/1.pdf" }, + { contentType: "application/pdf", contentUrl: "https://x/2.pdf" }, + ]), + ).toBe(" (2 files)"); + }); + }); + + describe("downloadMSTeamsImageAttachments", () => { + it("downloads and stores image contentUrl attachments", async () => { + const fetchMock = vi.fn(async () => { + return new Response(Buffer.from("png"), { + status: 200, + headers: { "content-type": "image/png" }, + }); + }); + + const media = await downloadMSTeamsImageAttachments({ + attachments: [ + { contentType: "image/png", contentUrl: "https://x/img" }, + ], + maxBytes: 1024 * 1024, + fetchFn: fetchMock as unknown as typeof fetch, + }); + + expect(media).toHaveLength(1); + expect(media[0]?.path).toBe("/tmp/saved.png"); + expect(fetchMock).toHaveBeenCalledWith("https://x/img"); + expect(saveMediaBufferMock).toHaveBeenCalled(); + }); + + it("supports Teams file.download.info downloadUrl attachments", async () => { + const fetchMock = vi.fn(async () => { + return new Response(Buffer.from("png"), { + status: 200, + headers: { "content-type": "image/png" }, + }); + }); + + const media = await downloadMSTeamsImageAttachments({ + attachments: [ + { + contentType: "application/vnd.microsoft.teams.file.download.info", + content: { downloadUrl: "https://x/dl", fileType: "png" }, + }, + ], + maxBytes: 1024 * 1024, + fetchFn: fetchMock as unknown as typeof fetch, + }); + + expect(media).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledWith("https://x/dl"); + }); + + it("retries with auth when the first request is unauthorized", async () => { + const fetchMock = vi.fn(async (_url: string, opts?: RequestInit) => { + const hasAuth = Boolean( + opts && + typeof opts === "object" && + "headers" in opts && + (opts.headers as Record)?.Authorization, + ); + if (!hasAuth) { + return new Response("unauthorized", { status: 401 }); + } + return new Response(Buffer.from("png"), { + status: 200, + headers: { "content-type": "image/png" }, + }); + }); + + const media = await downloadMSTeamsImageAttachments({ + attachments: [ + { contentType: "image/png", contentUrl: "https://x/img" }, + ], + maxBytes: 1024 * 1024, + tokenProvider: { getAccessToken: vi.fn(async () => "token") }, + fetchFn: fetchMock as unknown as typeof fetch, + }); + + expect(media).toHaveLength(1); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("ignores non-image attachments", async () => { + const fetchMock = vi.fn(); + const media = await downloadMSTeamsImageAttachments({ + attachments: [ + { contentType: "application/pdf", contentUrl: "https://x/x.pdf" }, + ], + maxBytes: 1024 * 1024, + fetchFn: fetchMock as unknown as typeof fetch, + }); + + expect(media).toHaveLength(0); + expect(fetchMock).not.toHaveBeenCalled(); + }); + }); + + describe("buildMSTeamsMediaPayload", () => { + it("returns single and multi-file fields", () => { + const payload = buildMSTeamsMediaPayload([ + { path: "/tmp/a.png", contentType: "image/png" }, + { path: "/tmp/b.png", contentType: "image/png" }, + ]); + expect(payload.MediaPath).toBe("/tmp/a.png"); + expect(payload.MediaUrl).toBe("/tmp/a.png"); + expect(payload.MediaPaths).toEqual(["/tmp/a.png", "/tmp/b.png"]); + expect(payload.MediaUrls).toEqual(["/tmp/a.png", "/tmp/b.png"]); + expect(payload.MediaTypes).toEqual(["image/png", "image/png"]); + }); + }); +}); diff --git a/src/msteams/attachments.ts b/src/msteams/attachments.ts new file mode 100644 index 000000000..6c136b7d3 --- /dev/null +++ b/src/msteams/attachments.ts @@ -0,0 +1,272 @@ +import { detectMime } from "../media/mime.js"; +import { saveMediaBuffer } from "../media/store.js"; + +export type MSTeamsAttachmentLike = { + contentType?: string | null; + contentUrl?: string | null; + name?: string | null; + thumbnailUrl?: string | null; + content?: unknown; +}; + +export type MSTeamsAccessTokenProvider = { + getAccessToken: (scope: string) => Promise; +}; + +type DownloadCandidate = { + url: string; + fileHint?: string; + contentTypeHint?: string; + placeholder: string; +}; + +export type MSTeamsInboundMedia = { + path: string; + contentType?: string; + placeholder: string; +}; + +const IMAGE_EXT_RE = /\.(avif|bmp|gif|heic|heif|jpe?g|png|tiff?|webp)$/i; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function normalizeContentType(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed ? trimmed : undefined; +} + +function inferPlaceholder(params: { + contentType?: string; + fileName?: string; + fileType?: string; +}): string { + const mime = params.contentType?.toLowerCase() ?? ""; + const name = params.fileName?.toLowerCase() ?? ""; + const fileType = params.fileType?.toLowerCase() ?? ""; + + const looksLikeImage = + mime.startsWith("image/") || + IMAGE_EXT_RE.test(name) || + IMAGE_EXT_RE.test(`x.${fileType}`); + + return looksLikeImage ? "" : ""; +} + +function isLikelyImageAttachment(att: MSTeamsAttachmentLike): boolean { + const contentType = normalizeContentType(att.contentType) ?? ""; + const name = typeof att.name === "string" ? att.name : ""; + if (contentType.startsWith("image/")) return true; + if (IMAGE_EXT_RE.test(name)) return true; + + if ( + contentType === "application/vnd.microsoft.teams.file.download.info" && + isRecord(att.content) + ) { + const fileType = + typeof att.content.fileType === "string" ? att.content.fileType : ""; + if (fileType && IMAGE_EXT_RE.test(`x.${fileType}`)) return true; + const fileName = + typeof att.content.fileName === "string" ? att.content.fileName : ""; + if (fileName && IMAGE_EXT_RE.test(fileName)) return true; + } + + return false; +} + +export function buildMSTeamsAttachmentPlaceholder( + attachments: MSTeamsAttachmentLike[] | undefined, +): string { + const list = Array.isArray(attachments) ? attachments : []; + if (list.length === 0) return ""; + const imageCount = list.filter(isLikelyImageAttachment).length; + if (imageCount > 0) { + return `${imageCount > 1 ? ` (${imageCount} images)` : ""}`; + } + const count = list.length; + return `${count > 1 ? ` (${count} files)` : ""}`; +} + +function resolveDownloadCandidate( + att: MSTeamsAttachmentLike, +): DownloadCandidate | null { + const contentType = normalizeContentType(att.contentType); + const name = typeof att.name === "string" ? att.name.trim() : ""; + + if (contentType === "application/vnd.microsoft.teams.file.download.info") { + if (!isRecord(att.content)) return null; + const downloadUrl = + typeof att.content.downloadUrl === "string" + ? att.content.downloadUrl.trim() + : ""; + if (!downloadUrl) return null; + + const fileType = + typeof att.content.fileType === "string" + ? att.content.fileType.trim() + : ""; + const uniqueId = + typeof att.content.uniqueId === "string" + ? att.content.uniqueId.trim() + : ""; + const fileName = + typeof att.content.fileName === "string" + ? att.content.fileName.trim() + : ""; + + const fileHint = + name || + fileName || + (uniqueId && fileType ? `${uniqueId}.${fileType}` : ""); + return { + url: downloadUrl, + fileHint: fileHint || undefined, + contentTypeHint: undefined, + placeholder: inferPlaceholder({ + contentType, + fileName: fileHint, + fileType, + }), + }; + } + + const contentUrl = + typeof att.contentUrl === "string" ? att.contentUrl.trim() : ""; + if (!contentUrl) return null; + + return { + url: contentUrl, + fileHint: name || undefined, + contentTypeHint: contentType, + placeholder: inferPlaceholder({ contentType, fileName: name }), + }; +} + +function scopeCandidatesForUrl(url: string): string[] { + try { + const host = new URL(url).hostname.toLowerCase(); + const looksLikeGraph = + host.endsWith("graph.microsoft.com") || + host.endsWith("sharepoint.com") || + host.endsWith("1drv.ms") || + host.includes("sharepoint"); + return looksLikeGraph + ? [ + "https://graph.microsoft.com/.default", + "https://api.botframework.com/.default", + ] + : [ + "https://api.botframework.com/.default", + "https://graph.microsoft.com/.default", + ]; + } catch { + return [ + "https://api.botframework.com/.default", + "https://graph.microsoft.com/.default", + ]; + } +} + +async function fetchWithAuthFallback(params: { + url: string; + tokenProvider?: MSTeamsAccessTokenProvider; + fetchFn?: typeof fetch; +}): Promise { + const fetchFn = params.fetchFn ?? fetch; + const firstAttempt = await fetchFn(params.url); + if (firstAttempt.ok) return firstAttempt; + if (!params.tokenProvider) return firstAttempt; + if (firstAttempt.status !== 401 && firstAttempt.status !== 403) + return firstAttempt; + + const scopes = scopeCandidatesForUrl(params.url); + for (const scope of scopes) { + try { + const token = await params.tokenProvider.getAccessToken(scope); + const res = await fetchFn(params.url, { + headers: { Authorization: `Bearer ${token}` }, + }); + if (res.ok) return res; + } catch { + // Try the next scope. + } + } + + return firstAttempt; +} + +export async function downloadMSTeamsImageAttachments(params: { + attachments: MSTeamsAttachmentLike[] | undefined; + maxBytes: number; + tokenProvider?: MSTeamsAccessTokenProvider; + fetchFn?: typeof fetch; +}): Promise { + const list = Array.isArray(params.attachments) ? params.attachments : []; + if (list.length === 0) return []; + + const candidates = list + .filter(isLikelyImageAttachment) + .map(resolveDownloadCandidate) + .filter(Boolean) as DownloadCandidate[]; + + if (candidates.length === 0) return []; + + const out: MSTeamsInboundMedia[] = []; + for (const candidate of candidates) { + try { + const res = await fetchWithAuthFallback({ + url: candidate.url, + tokenProvider: params.tokenProvider, + fetchFn: params.fetchFn, + }); + if (!res.ok) continue; + const buffer = Buffer.from(await res.arrayBuffer()); + if (buffer.byteLength > params.maxBytes) continue; + const mime = await detectMime({ + buffer, + headerMime: + candidate.contentTypeHint ?? res.headers.get("content-type"), + filePath: candidate.fileHint ?? candidate.url, + }); + const saved = await saveMediaBuffer( + buffer, + mime, + "inbound", + params.maxBytes, + ); + out.push({ + path: saved.path, + contentType: saved.contentType, + placeholder: candidate.placeholder, + }); + } catch { + // Ignore download failures and continue. + } + } + return out; +} + +export function buildMSTeamsMediaPayload( + mediaList: Array<{ path: string; contentType?: string }>, +): { + MediaPath?: string; + MediaType?: string; + MediaUrl?: string; + MediaPaths?: string[]; + MediaUrls?: string[]; + MediaTypes?: string[]; +} { + const first = mediaList[0]; + const mediaPaths = mediaList.map((media) => media.path); + const mediaTypes = mediaList.map((media) => media.contentType ?? ""); + return { + MediaPath: first?.path, + MediaType: first?.contentType, + MediaUrl: first?.path, + MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined, + MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined, + MediaTypes: mediaPaths.length > 0 ? mediaTypes : undefined, + }; +} diff --git a/src/msteams/monitor.ts b/src/msteams/monitor.ts index 11be94d43..e032f6d2e 100644 --- a/src/msteams/monitor.ts +++ b/src/msteams/monitor.ts @@ -13,6 +13,12 @@ import { } from "../pairing/pairing-store.js"; import { resolveAgentRoute } from "../routing/resolve-route.js"; import type { RuntimeEnv } from "../runtime.js"; +import { + buildMSTeamsAttachmentPlaceholder, + buildMSTeamsMediaPayload, + downloadMSTeamsImageAttachments, + type MSTeamsAttachmentLike, +} from "./attachments.js"; import type { MSTeamsConversationStore, StoredConversationReference, @@ -82,6 +88,11 @@ export async function monitorMSTeamsProvider( const port = msteamsCfg.webhook?.port ?? 3978; const textLimit = resolveTextChunkLimit(cfg, "msteams"); + const MB = 1024 * 1024; + const mediaMaxBytes = + typeof cfg.agent?.mediaMaxMb === "number" && cfg.agent.mediaMaxMb > 0 + ? Math.floor(cfg.agent.mediaMaxMb * MB) + : 8 * MB; const conversationStore = opts.conversationStore ?? createMSTeamsConversationStoreFs(); @@ -94,6 +105,7 @@ export async function monitorMSTeamsProvider( const { ActivityHandler, CloudAdapter, + MsalTokenProvider, authorizeJWT, getAuthConfigWithDefaults, } = agentsHosting; @@ -104,6 +116,7 @@ export async function monitorMSTeamsProvider( clientSecret: creds.appPassword, tenantId: creds.tenantId, }); + const tokenProvider = new MsalTokenProvider(authConfig); const adapter = new CloudAdapter(authConfig); // Handler for incoming messages @@ -111,17 +124,32 @@ export async function monitorMSTeamsProvider( const activity = context.activity; const rawText = activity.text?.trim() ?? ""; const text = stripMSTeamsMentionTags(rawText); + const attachments = Array.isArray(activity.attachments) + ? (activity.attachments as unknown as MSTeamsAttachmentLike[]) + : []; + const attachmentPlaceholder = + buildMSTeamsAttachmentPlaceholder(attachments); + const rawBody = text || attachmentPlaceholder; const from = activity.from; const conversation = activity.conversation; + const attachmentTypes = attachments + .map((att) => + typeof att.contentType === "string" ? att.contentType : undefined, + ) + .filter(Boolean) + .slice(0, 3); + log.info("received message", { rawText: rawText.slice(0, 50), text: text.slice(0, 50), + attachments: attachments.length, + attachmentTypes, from: from?.id, conversation: conversation?.id, }); - if (!text) { + if (!rawBody) { log.debug("skipping empty message after stripping mentions"); return; } @@ -189,7 +217,7 @@ export async function monitorMSTeamsProvider( }, }); - const preview = text.replace(/\s+/g, " ").slice(0, 160); + const preview = rawBody.replace(/\s+/g, " ").slice(0, 160); const inboundLabel = isDirectMessage ? `Teams DM from ${senderName}` : `Teams message in ${conversationType} from ${senderName}`; @@ -274,11 +302,22 @@ export async function monitorMSTeamsProvider( // Format the message body with envelope const timestamp = parseMSTeamsActivityTimestamp(activity.timestamp); + const mediaList = await downloadMSTeamsImageAttachments({ + attachments, + maxBytes: mediaMaxBytes, + tokenProvider: { + getAccessToken: (scope) => tokenProvider.getAccessToken(scope), + }, + }); + if (mediaList.length > 0) { + log.debug("downloaded image attachments", { count: mediaList.length }); + } + const mediaPayload = buildMSTeamsMediaPayload(mediaList); const body = formatAgentEnvelope({ provider: "Teams", from: senderName, timestamp, - body: text, + body: rawBody, }); // Build context payload for agent @@ -300,6 +339,7 @@ export async function monitorMSTeamsProvider( CommandAuthorized: true, OriginatingChannel: "msteams" as const, OriginatingTo: teamsTo, + ...mediaPayload, }; if (shouldLogVerbose()) {