MS Teams: ingest inbound image attachments

src/msteams/attachments.test.ts

@@ -0,0 +1,179 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+  buildMSTeamsAttachmentPlaceholder,
+  buildMSTeamsMediaPayload,
+  downloadMSTeamsImageAttachments,
+} from "./attachments.js";
+
+const detectMimeMock = vi.fn(async () => "image/png");
+const saveMediaBufferMock = vi.fn(async () => ({
+  path: "/tmp/saved.png",
+  contentType: "image/png",
+}));
+
+vi.mock("../media/mime.js", () => ({
+  detectMime: (...args: unknown[]) => detectMimeMock(...args),
+}));
+
+vi.mock("../media/store.js", () => ({
+  saveMediaBuffer: (...args: unknown[]) => saveMediaBufferMock(...args),
+}));
+
+describe("msteams attachments", () => {
+  beforeEach(() => {
+    detectMimeMock.mockClear();
+    saveMediaBufferMock.mockClear();
+  });
+
+  describe("buildMSTeamsAttachmentPlaceholder", () => {
+    it("returns empty string when no attachments", () => {
+      expect(buildMSTeamsAttachmentPlaceholder(undefined)).toBe("");
+      expect(buildMSTeamsAttachmentPlaceholder([])).toBe("");
+    });
+
+    it("returns image placeholder for image attachments", () => {
+      expect(
+        buildMSTeamsAttachmentPlaceholder([
+          { contentType: "image/png", contentUrl: "https://x/img.png" },
+        ]),
+      ).toBe("<media:image>");
+      expect(
+        buildMSTeamsAttachmentPlaceholder([
+          { contentType: "image/png", contentUrl: "https://x/1.png" },
+          { contentType: "image/jpeg", contentUrl: "https://x/2.jpg" },
+        ]),
+      ).toBe("<media:image> (2 images)");
+    });
+
+    it("treats Teams file.download.info image attachments as images", () => {
+      expect(
+        buildMSTeamsAttachmentPlaceholder([
+          {
+            contentType: "application/vnd.microsoft.teams.file.download.info",
+            content: { downloadUrl: "https://x/dl", fileType: "png" },
+          },
+        ]),
+      ).toBe("<media:image>");
+    });
+
+    it("returns document placeholder for non-image attachments", () => {
+      expect(
+        buildMSTeamsAttachmentPlaceholder([
+          { contentType: "application/pdf", contentUrl: "https://x/x.pdf" },
+        ]),
+      ).toBe("<media:document>");
+      expect(
+        buildMSTeamsAttachmentPlaceholder([
+          { contentType: "application/pdf", contentUrl: "https://x/1.pdf" },
+          { contentType: "application/pdf", contentUrl: "https://x/2.pdf" },
+        ]),
+      ).toBe("<media:document> (2 files)");
+    });
+  });
+
+  describe("downloadMSTeamsImageAttachments", () => {
+    it("downloads and stores image contentUrl attachments", async () => {
+      const fetchMock = vi.fn(async () => {
+        return new Response(Buffer.from("png"), {
+          status: 200,
+          headers: { "content-type": "image/png" },
+        });
+      });
+
+      const media = await downloadMSTeamsImageAttachments({
+        attachments: [
+          { contentType: "image/png", contentUrl: "https://x/img" },
+        ],
+        maxBytes: 1024 * 1024,
+        fetchFn: fetchMock as unknown as typeof fetch,
+      });
+
+      expect(media).toHaveLength(1);
+      expect(media[0]?.path).toBe("/tmp/saved.png");
+      expect(fetchMock).toHaveBeenCalledWith("https://x/img");
+      expect(saveMediaBufferMock).toHaveBeenCalled();
+    });
+
+    it("supports Teams file.download.info downloadUrl attachments", async () => {
+      const fetchMock = vi.fn(async () => {
+        return new Response(Buffer.from("png"), {
+          status: 200,
+          headers: { "content-type": "image/png" },
+        });
+      });
+
+      const media = await downloadMSTeamsImageAttachments({
+        attachments: [
+          {
+            contentType: "application/vnd.microsoft.teams.file.download.info",
+            content: { downloadUrl: "https://x/dl", fileType: "png" },
+          },
+        ],
+        maxBytes: 1024 * 1024,
+        fetchFn: fetchMock as unknown as typeof fetch,
+      });
+
+      expect(media).toHaveLength(1);
+      expect(fetchMock).toHaveBeenCalledWith("https://x/dl");
+    });
+
+    it("retries with auth when the first request is unauthorized", async () => {
+      const fetchMock = vi.fn(async (_url: string, opts?: RequestInit) => {
+        const hasAuth = Boolean(
+          opts &&
+            typeof opts === "object" &&
+            "headers" in opts &&
+            (opts.headers as Record<string, string>)?.Authorization,
+        );
+        if (!hasAuth) {
+          return new Response("unauthorized", { status: 401 });
+        }
+        return new Response(Buffer.from("png"), {
+          status: 200,
+          headers: { "content-type": "image/png" },
+        });
+      });
+
+      const media = await downloadMSTeamsImageAttachments({
+        attachments: [
+          { contentType: "image/png", contentUrl: "https://x/img" },
+        ],
+        maxBytes: 1024 * 1024,
+        tokenProvider: { getAccessToken: vi.fn(async () => "token") },
+        fetchFn: fetchMock as unknown as typeof fetch,
+      });
+
+      expect(media).toHaveLength(1);
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+
+    it("ignores non-image attachments", async () => {
+      const fetchMock = vi.fn();
+      const media = await downloadMSTeamsImageAttachments({
+        attachments: [
+          { contentType: "application/pdf", contentUrl: "https://x/x.pdf" },
+        ],
+        maxBytes: 1024 * 1024,
+        fetchFn: fetchMock as unknown as typeof fetch,
+      });
+
+      expect(media).toHaveLength(0);
+      expect(fetchMock).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("buildMSTeamsMediaPayload", () => {
+    it("returns single and multi-file fields", () => {
+      const payload = buildMSTeamsMediaPayload([
+        { path: "/tmp/a.png", contentType: "image/png" },
+        { path: "/tmp/b.png", contentType: "image/png" },
+      ]);
+      expect(payload.MediaPath).toBe("/tmp/a.png");
+      expect(payload.MediaUrl).toBe("/tmp/a.png");
+      expect(payload.MediaPaths).toEqual(["/tmp/a.png", "/tmp/b.png"]);
+      expect(payload.MediaUrls).toEqual(["/tmp/a.png", "/tmp/b.png"]);
+      expect(payload.MediaTypes).toEqual(["image/png", "image/png"]);
+    });
+  });
+});

src/msteams/attachments.ts

@@ -0,0 +1,272 @@
+import { detectMime } from "../media/mime.js";
+import { saveMediaBuffer } from "../media/store.js";
+
+export type MSTeamsAttachmentLike = {
+  contentType?: string | null;
+  contentUrl?: string | null;
+  name?: string | null;
+  thumbnailUrl?: string | null;
+  content?: unknown;
+};
+
+export type MSTeamsAccessTokenProvider = {
+  getAccessToken: (scope: string) => Promise<string>;
+};
+
+type DownloadCandidate = {
+  url: string;
+  fileHint?: string;
+  contentTypeHint?: string;
+  placeholder: string;
+};
+
+export type MSTeamsInboundMedia = {
+  path: string;
+  contentType?: string;
+  placeholder: string;
+};
+
+const IMAGE_EXT_RE = /\.(avif|bmp|gif|heic|heif|jpe?g|png|tiff?|webp)$/i;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+
+function normalizeContentType(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  const trimmed = value.trim();
+  return trimmed ? trimmed : undefined;
+}
+
+function inferPlaceholder(params: {
+  contentType?: string;
+  fileName?: string;
+  fileType?: string;
+}): string {
+  const mime = params.contentType?.toLowerCase() ?? "";
+  const name = params.fileName?.toLowerCase() ?? "";
+  const fileType = params.fileType?.toLowerCase() ?? "";
+
+  const looksLikeImage =
+    mime.startsWith("image/") ||
+    IMAGE_EXT_RE.test(name) ||
+    IMAGE_EXT_RE.test(`x.${fileType}`);
+
+  return looksLikeImage ? "<media:image>" : "<media:document>";
+}
+
+function isLikelyImageAttachment(att: MSTeamsAttachmentLike): boolean {
+  const contentType = normalizeContentType(att.contentType) ?? "";
+  const name = typeof att.name === "string" ? att.name : "";
+  if (contentType.startsWith("image/")) return true;
+  if (IMAGE_EXT_RE.test(name)) return true;
+
+  if (
+    contentType === "application/vnd.microsoft.teams.file.download.info" &&
+    isRecord(att.content)
+  ) {
+    const fileType =
+      typeof att.content.fileType === "string" ? att.content.fileType : "";
+    if (fileType && IMAGE_EXT_RE.test(`x.${fileType}`)) return true;
+    const fileName =
+      typeof att.content.fileName === "string" ? att.content.fileName : "";
+    if (fileName && IMAGE_EXT_RE.test(fileName)) return true;
+  }
+
+  return false;
+}
+
+export function buildMSTeamsAttachmentPlaceholder(
+  attachments: MSTeamsAttachmentLike[] | undefined,
+): string {
+  const list = Array.isArray(attachments) ? attachments : [];
+  if (list.length === 0) return "";
+  const imageCount = list.filter(isLikelyImageAttachment).length;
+  if (imageCount > 0) {
+    return `<media:image>${imageCount > 1 ? ` (${imageCount} images)` : ""}`;
+  }
+  const count = list.length;
+  return `<media:document>${count > 1 ? ` (${count} files)` : ""}`;
+}
+
+function resolveDownloadCandidate(
+  att: MSTeamsAttachmentLike,
+): DownloadCandidate | null {
+  const contentType = normalizeContentType(att.contentType);
+  const name = typeof att.name === "string" ? att.name.trim() : "";
+
+  if (contentType === "application/vnd.microsoft.teams.file.download.info") {
+    if (!isRecord(att.content)) return null;
+    const downloadUrl =
+      typeof att.content.downloadUrl === "string"
+        ? att.content.downloadUrl.trim()
+        : "";
+    if (!downloadUrl) return null;
+
+    const fileType =
+      typeof att.content.fileType === "string"
+        ? att.content.fileType.trim()
+        : "";
+    const uniqueId =
+      typeof att.content.uniqueId === "string"
+        ? att.content.uniqueId.trim()
+        : "";
+    const fileName =
+      typeof att.content.fileName === "string"
+        ? att.content.fileName.trim()
+        : "";
+
+    const fileHint =
+      name ||
+      fileName ||
+      (uniqueId && fileType ? `${uniqueId}.${fileType}` : "");
+    return {
+      url: downloadUrl,
+      fileHint: fileHint || undefined,
+      contentTypeHint: undefined,
+      placeholder: inferPlaceholder({
+        contentType,
+        fileName: fileHint,
+        fileType,
+      }),
+    };
+  }
+
+  const contentUrl =
+    typeof att.contentUrl === "string" ? att.contentUrl.trim() : "";
+  if (!contentUrl) return null;
+
+  return {
+    url: contentUrl,
+    fileHint: name || undefined,
+    contentTypeHint: contentType,
+    placeholder: inferPlaceholder({ contentType, fileName: name }),
+  };
+}
+
+function scopeCandidatesForUrl(url: string): string[] {
+  try {
+    const host = new URL(url).hostname.toLowerCase();
+    const looksLikeGraph =
+      host.endsWith("graph.microsoft.com") ||
+      host.endsWith("sharepoint.com") ||
+      host.endsWith("1drv.ms") ||
+      host.includes("sharepoint");
+    return looksLikeGraph
+      ? [
+          "https://graph.microsoft.com/.default",
+          "https://api.botframework.com/.default",
+        ]
+      : [
+          "https://api.botframework.com/.default",
+          "https://graph.microsoft.com/.default",
+        ];
+  } catch {
+    return [
+      "https://api.botframework.com/.default",
+      "https://graph.microsoft.com/.default",
+    ];
+  }
+}
+
+async function fetchWithAuthFallback(params: {
+  url: string;
+  tokenProvider?: MSTeamsAccessTokenProvider;
+  fetchFn?: typeof fetch;
+}): Promise<Response> {
+  const fetchFn = params.fetchFn ?? fetch;
+  const firstAttempt = await fetchFn(params.url);
+  if (firstAttempt.ok) return firstAttempt;
+  if (!params.tokenProvider) return firstAttempt;
+  if (firstAttempt.status !== 401 && firstAttempt.status !== 403)
+    return firstAttempt;
+
+  const scopes = scopeCandidatesForUrl(params.url);
+  for (const scope of scopes) {
+    try {
+      const token = await params.tokenProvider.getAccessToken(scope);
+      const res = await fetchFn(params.url, {
+        headers: { Authorization: `Bearer ${token}` },
+      });
+      if (res.ok) return res;
+    } catch {
+      // Try the next scope.
+    }
+  }
+
+  return firstAttempt;
+}
+
+export async function downloadMSTeamsImageAttachments(params: {
+  attachments: MSTeamsAttachmentLike[] | undefined;
+  maxBytes: number;
+  tokenProvider?: MSTeamsAccessTokenProvider;
+  fetchFn?: typeof fetch;
+}): Promise<MSTeamsInboundMedia[]> {
+  const list = Array.isArray(params.attachments) ? params.attachments : [];
+  if (list.length === 0) return [];
+
+  const candidates = list
+    .filter(isLikelyImageAttachment)
+    .map(resolveDownloadCandidate)
+    .filter(Boolean) as DownloadCandidate[];
+
+  if (candidates.length === 0) return [];
+
+  const out: MSTeamsInboundMedia[] = [];
+  for (const candidate of candidates) {
+    try {
+      const res = await fetchWithAuthFallback({
+        url: candidate.url,
+        tokenProvider: params.tokenProvider,
+        fetchFn: params.fetchFn,
+      });
+      if (!res.ok) continue;
+      const buffer = Buffer.from(await res.arrayBuffer());
+      if (buffer.byteLength > params.maxBytes) continue;
+      const mime = await detectMime({
+        buffer,
+        headerMime:
+          candidate.contentTypeHint ?? res.headers.get("content-type"),
+        filePath: candidate.fileHint ?? candidate.url,
+      });
+      const saved = await saveMediaBuffer(
+        buffer,
+        mime,
+        "inbound",
+        params.maxBytes,
+      );
+      out.push({
+        path: saved.path,
+        contentType: saved.contentType,
+        placeholder: candidate.placeholder,
+      });
+    } catch {
+      // Ignore download failures and continue.
+    }
+  }
+  return out;
+}
+
+export function buildMSTeamsMediaPayload(
+  mediaList: Array<{ path: string; contentType?: string }>,
+): {
+  MediaPath?: string;
+  MediaType?: string;
+  MediaUrl?: string;
+  MediaPaths?: string[];
+  MediaUrls?: string[];
+  MediaTypes?: string[];
+} {
+  const first = mediaList[0];
+  const mediaPaths = mediaList.map((media) => media.path);
+  const mediaTypes = mediaList.map((media) => media.contentType ?? "");
+  return {
+    MediaPath: first?.path,
+    MediaType: first?.contentType,
+    MediaUrl: first?.path,
+    MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined,
+    MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined,
+    MediaTypes: mediaPaths.length > 0 ? mediaTypes : undefined,
+  };
+}

src/msteams/monitor.ts

@@ -13,6 +13,12 @@ import {
 } from "../pairing/pairing-store.js";
 import { resolveAgentRoute } from "../routing/resolve-route.js";
 import type { RuntimeEnv } from "../runtime.js";
+import {
+  buildMSTeamsAttachmentPlaceholder,
+  buildMSTeamsMediaPayload,
+  downloadMSTeamsImageAttachments,
+  type MSTeamsAttachmentLike,
+} from "./attachments.js";
 import type {
   MSTeamsConversationStore,
   StoredConversationReference,
@@ -82,6 +88,11 @@ export async function monitorMSTeamsProvider(
 
   const port = msteamsCfg.webhook?.port ?? 3978;
   const textLimit = resolveTextChunkLimit(cfg, "msteams");
+  const MB = 1024 * 1024;
+  const mediaMaxBytes =
+    typeof cfg.agent?.mediaMaxMb === "number" && cfg.agent.mediaMaxMb > 0
+      ? Math.floor(cfg.agent.mediaMaxMb * MB)
+      : 8 * MB;
   const conversationStore =
     opts.conversationStore ?? createMSTeamsConversationStoreFs();
 
@@ -94,6 +105,7 @@ export async function monitorMSTeamsProvider(
   const {
     ActivityHandler,
     CloudAdapter,
+    MsalTokenProvider,
     authorizeJWT,
     getAuthConfigWithDefaults,
   } = agentsHosting;
@@ -104,6 +116,7 @@ export async function monitorMSTeamsProvider(
     clientSecret: creds.appPassword,
     tenantId: creds.tenantId,
   });
+  const tokenProvider = new MsalTokenProvider(authConfig);
   const adapter = new CloudAdapter(authConfig);
 
   // Handler for incoming messages
@@ -111,17 +124,32 @@ export async function monitorMSTeamsProvider(
     const activity = context.activity;
     const rawText = activity.text?.trim() ?? "";
     const text = stripMSTeamsMentionTags(rawText);
+    const attachments = Array.isArray(activity.attachments)
+      ? (activity.attachments as unknown as MSTeamsAttachmentLike[])
+      : [];
+    const attachmentPlaceholder =
+      buildMSTeamsAttachmentPlaceholder(attachments);
+    const rawBody = text || attachmentPlaceholder;
     const from = activity.from;
     const conversation = activity.conversation;
 
+    const attachmentTypes = attachments
+      .map((att) =>
+        typeof att.contentType === "string" ? att.contentType : undefined,
+      )
+      .filter(Boolean)
+      .slice(0, 3);
+
     log.info("received message", {
       rawText: rawText.slice(0, 50),
       text: text.slice(0, 50),
+      attachments: attachments.length,
+      attachmentTypes,
       from: from?.id,
       conversation: conversation?.id,
     });
 
-    if (!text) {
+    if (!rawBody) {
       log.debug("skipping empty message after stripping mentions");
       return;
     }
@@ -189,7 +217,7 @@ export async function monitorMSTeamsProvider(
       },
     });
 
-    const preview = text.replace(/\s+/g, " ").slice(0, 160);
+    const preview = rawBody.replace(/\s+/g, " ").slice(0, 160);
     const inboundLabel = isDirectMessage
       ? `Teams DM from ${senderName}`
       : `Teams message in ${conversationType} from ${senderName}`;
@@ -274,11 +302,22 @@ export async function monitorMSTeamsProvider(
 
     // Format the message body with envelope
     const timestamp = parseMSTeamsActivityTimestamp(activity.timestamp);
+    const mediaList = await downloadMSTeamsImageAttachments({
+      attachments,
+      maxBytes: mediaMaxBytes,
+      tokenProvider: {
+        getAccessToken: (scope) => tokenProvider.getAccessToken(scope),
+      },
+    });
+    if (mediaList.length > 0) {
+      log.debug("downloaded image attachments", { count: mediaList.length });
+    }
+    const mediaPayload = buildMSTeamsMediaPayload(mediaList);
     const body = formatAgentEnvelope({
       provider: "Teams",
       from: senderName,
       timestamp,
-      body: text,
+      body: rawBody,
     });
 
     // Build context payload for agent
@@ -300,6 +339,7 @@ export async function monitorMSTeamsProvider(
       CommandAuthorized: true,
       OriginatingChannel: "msteams" as const,
       OriginatingTo: teamsTo,
+      ...mediaPayload,
     };
 
     if (shouldLogVerbose()) {