MS Teams: ingest inbound image attachments
This commit is contained in:
Onur
Peter Steinberger
179
src/msteams/attachments.test.ts
Normal file
179
src/msteams/attachments.test.ts
Normal file
@@ -0,0 +1,179 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
buildMSTeamsAttachmentPlaceholder,
|
||||
buildMSTeamsMediaPayload,
|
||||
downloadMSTeamsImageAttachments,
|
||||
} from "./attachments.js";
|
||||
|
||||
const detectMimeMock = vi.fn(async () => "image/png");
|
||||
const saveMediaBufferMock = vi.fn(async () => ({
|
||||
path: "/tmp/saved.png",
|
||||
contentType: "image/png",
|
||||
}));
|
||||
|
||||
vi.mock("../media/mime.js", () => ({
|
||||
detectMime: (...args: unknown[]) => detectMimeMock(...args),
|
||||
}));
|
||||
|
||||
vi.mock("../media/store.js", () => ({
|
||||
saveMediaBuffer: (...args: unknown[]) => saveMediaBufferMock(...args),
|
||||
}));
|
||||
|
||||
describe("msteams attachments", () => {
|
||||
beforeEach(() => {
|
||||
detectMimeMock.mockClear();
|
||||
saveMediaBufferMock.mockClear();
|
||||
});
|
||||
|
||||
describe("buildMSTeamsAttachmentPlaceholder", () => {
|
||||
it("returns empty string when no attachments", () => {
|
||||
expect(buildMSTeamsAttachmentPlaceholder(undefined)).toBe("");
|
||||
expect(buildMSTeamsAttachmentPlaceholder([])).toBe("");
|
||||
});
|
||||
|
||||
it("returns image placeholder for image attachments", () => {
|
||||
expect(
|
||||
buildMSTeamsAttachmentPlaceholder([
|
||||
{ contentType: "image/png", contentUrl: "https://x/img.png" },
|
||||
]),
|
||||
).toBe("<media:image>");
|
||||
expect(
|
||||
buildMSTeamsAttachmentPlaceholder([
|
||||
{ contentType: "image/png", contentUrl: "https://x/1.png" },
|
||||
{ contentType: "image/jpeg", contentUrl: "https://x/2.jpg" },
|
||||
]),
|
||||
).toBe("<media:image> (2 images)");
|
||||
});
|
||||
|
||||
it("treats Teams file.download.info image attachments as images", () => {
|
||||
expect(
|
||||
buildMSTeamsAttachmentPlaceholder([
|
||||
{
|
||||
contentType: "application/vnd.microsoft.teams.file.download.info",
|
||||
content: { downloadUrl: "https://x/dl", fileType: "png" },
|
||||
},
|
||||
]),
|
||||
).toBe("<media:image>");
|
||||
});
|
||||
|
||||
it("returns document placeholder for non-image attachments", () => {
|
||||
expect(
|
||||
buildMSTeamsAttachmentPlaceholder([
|
||||
{ contentType: "application/pdf", contentUrl: "https://x/x.pdf" },
|
||||
]),
|
||||
).toBe("<media:document>");
|
||||
expect(
|
||||
buildMSTeamsAttachmentPlaceholder([
|
||||
{ contentType: "application/pdf", contentUrl: "https://x/1.pdf" },
|
||||
{ contentType: "application/pdf", contentUrl: "https://x/2.pdf" },
|
||||
]),
|
||||
).toBe("<media:document> (2 files)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("downloadMSTeamsImageAttachments", () => {
|
||||
it("downloads and stores image contentUrl attachments", async () => {
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(Buffer.from("png"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
});
|
||||
});
|
||||
|
||||
const media = await downloadMSTeamsImageAttachments({
|
||||
attachments: [
|
||||
{ contentType: "image/png", contentUrl: "https://x/img" },
|
||||
],
|
||||
maxBytes: 1024 * 1024,
|
||||
fetchFn: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(media[0]?.path).toBe("/tmp/saved.png");
|
||||
expect(fetchMock).toHaveBeenCalledWith("https://x/img");
|
||||
expect(saveMediaBufferMock).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("supports Teams file.download.info downloadUrl attachments", async () => {
|
||||
const fetchMock = vi.fn(async () => {
|
||||
return new Response(Buffer.from("png"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
});
|
||||
});
|
||||
|
||||
const media = await downloadMSTeamsImageAttachments({
|
||||
attachments: [
|
||||
{
|
||||
contentType: "application/vnd.microsoft.teams.file.download.info",
|
||||
content: { downloadUrl: "https://x/dl", fileType: "png" },
|
||||
},
|
||||
],
|
||||
maxBytes: 1024 * 1024,
|
||||
fetchFn: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(fetchMock).toHaveBeenCalledWith("https://x/dl");
|
||||
});
|
||||
|
||||
it("retries with auth when the first request is unauthorized", async () => {
|
||||
const fetchMock = vi.fn(async (_url: string, opts?: RequestInit) => {
|
||||
const hasAuth = Boolean(
|
||||
opts &&
|
||||
typeof opts === "object" &&
|
||||
"headers" in opts &&
|
||||
(opts.headers as Record<string, string>)?.Authorization,
|
||||
);
|
||||
if (!hasAuth) {
|
||||
return new Response("unauthorized", { status: 401 });
|
||||
}
|
||||
return new Response(Buffer.from("png"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
});
|
||||
});
|
||||
|
||||
const media = await downloadMSTeamsImageAttachments({
|
||||
attachments: [
|
||||
{ contentType: "image/png", contentUrl: "https://x/img" },
|
||||
],
|
||||
maxBytes: 1024 * 1024,
|
||||
tokenProvider: { getAccessToken: vi.fn(async () => "token") },
|
||||
fetchFn: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(media).toHaveLength(1);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("ignores non-image attachments", async () => {
|
||||
const fetchMock = vi.fn();
|
||||
const media = await downloadMSTeamsImageAttachments({
|
||||
attachments: [
|
||||
{ contentType: "application/pdf", contentUrl: "https://x/x.pdf" },
|
||||
],
|
||||
maxBytes: 1024 * 1024,
|
||||
fetchFn: fetchMock as unknown as typeof fetch,
|
||||
});
|
||||
|
||||
expect(media).toHaveLength(0);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildMSTeamsMediaPayload", () => {
|
||||
it("returns single and multi-file fields", () => {
|
||||
const payload = buildMSTeamsMediaPayload([
|
||||
{ path: "/tmp/a.png", contentType: "image/png" },
|
||||
{ path: "/tmp/b.png", contentType: "image/png" },
|
||||
]);
|
||||
expect(payload.MediaPath).toBe("/tmp/a.png");
|
||||
expect(payload.MediaUrl).toBe("/tmp/a.png");
|
||||
expect(payload.MediaPaths).toEqual(["/tmp/a.png", "/tmp/b.png"]);
|
||||
expect(payload.MediaUrls).toEqual(["/tmp/a.png", "/tmp/b.png"]);
|
||||
expect(payload.MediaTypes).toEqual(["image/png", "image/png"]);
|
||||
});
|
||||
});
|
||||
});
|
||||
272
src/msteams/attachments.ts
Normal file
272
src/msteams/attachments.ts
Normal file
@@ -0,0 +1,272 @@
|
||||
import { detectMime } from "../media/mime.js";
|
||||
import { saveMediaBuffer } from "../media/store.js";
|
||||
|
||||
export type MSTeamsAttachmentLike = {
|
||||
contentType?: string | null;
|
||||
contentUrl?: string | null;
|
||||
name?: string | null;
|
||||
thumbnailUrl?: string | null;
|
||||
content?: unknown;
|
||||
};
|
||||
|
||||
export type MSTeamsAccessTokenProvider = {
|
||||
getAccessToken: (scope: string) => Promise<string>;
|
||||
};
|
||||
|
||||
type DownloadCandidate = {
|
||||
url: string;
|
||||
fileHint?: string;
|
||||
contentTypeHint?: string;
|
||||
placeholder: string;
|
||||
};
|
||||
|
||||
export type MSTeamsInboundMedia = {
|
||||
path: string;
|
||||
contentType?: string;
|
||||
placeholder: string;
|
||||
};
|
||||
|
||||
const IMAGE_EXT_RE = /\.(avif|bmp|gif|heic|heif|jpe?g|png|tiff?|webp)$/i;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function normalizeContentType(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const trimmed = value.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function inferPlaceholder(params: {
|
||||
contentType?: string;
|
||||
fileName?: string;
|
||||
fileType?: string;
|
||||
}): string {
|
||||
const mime = params.contentType?.toLowerCase() ?? "";
|
||||
const name = params.fileName?.toLowerCase() ?? "";
|
||||
const fileType = params.fileType?.toLowerCase() ?? "";
|
||||
|
||||
const looksLikeImage =
|
||||
mime.startsWith("image/") ||
|
||||
IMAGE_EXT_RE.test(name) ||
|
||||
IMAGE_EXT_RE.test(`x.${fileType}`);
|
||||
|
||||
return looksLikeImage ? "<media:image>" : "<media:document>";
|
||||
}
|
||||
|
||||
function isLikelyImageAttachment(att: MSTeamsAttachmentLike): boolean {
|
||||
const contentType = normalizeContentType(att.contentType) ?? "";
|
||||
const name = typeof att.name === "string" ? att.name : "";
|
||||
if (contentType.startsWith("image/")) return true;
|
||||
if (IMAGE_EXT_RE.test(name)) return true;
|
||||
|
||||
if (
|
||||
contentType === "application/vnd.microsoft.teams.file.download.info" &&
|
||||
isRecord(att.content)
|
||||
) {
|
||||
const fileType =
|
||||
typeof att.content.fileType === "string" ? att.content.fileType : "";
|
||||
if (fileType && IMAGE_EXT_RE.test(`x.${fileType}`)) return true;
|
||||
const fileName =
|
||||
typeof att.content.fileName === "string" ? att.content.fileName : "";
|
||||
if (fileName && IMAGE_EXT_RE.test(fileName)) return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function buildMSTeamsAttachmentPlaceholder(
|
||||
attachments: MSTeamsAttachmentLike[] | undefined,
|
||||
): string {
|
||||
const list = Array.isArray(attachments) ? attachments : [];
|
||||
if (list.length === 0) return "";
|
||||
const imageCount = list.filter(isLikelyImageAttachment).length;
|
||||
if (imageCount > 0) {
|
||||
return `<media:image>${imageCount > 1 ? ` (${imageCount} images)` : ""}`;
|
||||
}
|
||||
const count = list.length;
|
||||
return `<media:document>${count > 1 ? ` (${count} files)` : ""}`;
|
||||
}
|
||||
|
||||
function resolveDownloadCandidate(
|
||||
att: MSTeamsAttachmentLike,
|
||||
): DownloadCandidate | null {
|
||||
const contentType = normalizeContentType(att.contentType);
|
||||
const name = typeof att.name === "string" ? att.name.trim() : "";
|
||||
|
||||
if (contentType === "application/vnd.microsoft.teams.file.download.info") {
|
||||
if (!isRecord(att.content)) return null;
|
||||
const downloadUrl =
|
||||
typeof att.content.downloadUrl === "string"
|
||||
? att.content.downloadUrl.trim()
|
||||
: "";
|
||||
if (!downloadUrl) return null;
|
||||
|
||||
const fileType =
|
||||
typeof att.content.fileType === "string"
|
||||
? att.content.fileType.trim()
|
||||
: "";
|
||||
const uniqueId =
|
||||
typeof att.content.uniqueId === "string"
|
||||
? att.content.uniqueId.trim()
|
||||
: "";
|
||||
const fileName =
|
||||
typeof att.content.fileName === "string"
|
||||
? att.content.fileName.trim()
|
||||
: "";
|
||||
|
||||
const fileHint =
|
||||
name ||
|
||||
fileName ||
|
||||
(uniqueId && fileType ? `${uniqueId}.${fileType}` : "");
|
||||
return {
|
||||
url: downloadUrl,
|
||||
fileHint: fileHint || undefined,
|
||||
contentTypeHint: undefined,
|
||||
placeholder: inferPlaceholder({
|
||||
contentType,
|
||||
fileName: fileHint,
|
||||
fileType,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const contentUrl =
|
||||
typeof att.contentUrl === "string" ? att.contentUrl.trim() : "";
|
||||
if (!contentUrl) return null;
|
||||
|
||||
return {
|
||||
url: contentUrl,
|
||||
fileHint: name || undefined,
|
||||
contentTypeHint: contentType,
|
||||
placeholder: inferPlaceholder({ contentType, fileName: name }),
|
||||
};
|
||||
}
|
||||
|
||||
function scopeCandidatesForUrl(url: string): string[] {
|
||||
try {
|
||||
const host = new URL(url).hostname.toLowerCase();
|
||||
const looksLikeGraph =
|
||||
host.endsWith("graph.microsoft.com") ||
|
||||
host.endsWith("sharepoint.com") ||
|
||||
host.endsWith("1drv.ms") ||
|
||||
host.includes("sharepoint");
|
||||
return looksLikeGraph
|
||||
? [
|
||||
"https://graph.microsoft.com/.default",
|
||||
"https://api.botframework.com/.default",
|
||||
]
|
||||
: [
|
||||
"https://api.botframework.com/.default",
|
||||
"https://graph.microsoft.com/.default",
|
||||
];
|
||||
} catch {
|
||||
return [
|
||||
"https://api.botframework.com/.default",
|
||||
"https://graph.microsoft.com/.default",
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchWithAuthFallback(params: {
|
||||
url: string;
|
||||
tokenProvider?: MSTeamsAccessTokenProvider;
|
||||
fetchFn?: typeof fetch;
|
||||
}): Promise<Response> {
|
||||
const fetchFn = params.fetchFn ?? fetch;
|
||||
const firstAttempt = await fetchFn(params.url);
|
||||
if (firstAttempt.ok) return firstAttempt;
|
||||
if (!params.tokenProvider) return firstAttempt;
|
||||
if (firstAttempt.status !== 401 && firstAttempt.status !== 403)
|
||||
return firstAttempt;
|
||||
|
||||
const scopes = scopeCandidatesForUrl(params.url);
|
||||
for (const scope of scopes) {
|
||||
try {
|
||||
const token = await params.tokenProvider.getAccessToken(scope);
|
||||
const res = await fetchFn(params.url, {
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
});
|
||||
if (res.ok) return res;
|
||||
} catch {
|
||||
// Try the next scope.
|
||||
}
|
||||
}
|
||||
|
||||
return firstAttempt;
|
||||
}
|
||||
|
||||
export async function downloadMSTeamsImageAttachments(params: {
|
||||
attachments: MSTeamsAttachmentLike[] | undefined;
|
||||
maxBytes: number;
|
||||
tokenProvider?: MSTeamsAccessTokenProvider;
|
||||
fetchFn?: typeof fetch;
|
||||
}): Promise<MSTeamsInboundMedia[]> {
|
||||
const list = Array.isArray(params.attachments) ? params.attachments : [];
|
||||
if (list.length === 0) return [];
|
||||
|
||||
const candidates = list
|
||||
.filter(isLikelyImageAttachment)
|
||||
.map(resolveDownloadCandidate)
|
||||
.filter(Boolean) as DownloadCandidate[];
|
||||
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const out: MSTeamsInboundMedia[] = [];
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const res = await fetchWithAuthFallback({
|
||||
url: candidate.url,
|
||||
tokenProvider: params.tokenProvider,
|
||||
fetchFn: params.fetchFn,
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const buffer = Buffer.from(await res.arrayBuffer());
|
||||
if (buffer.byteLength > params.maxBytes) continue;
|
||||
const mime = await detectMime({
|
||||
buffer,
|
||||
headerMime:
|
||||
candidate.contentTypeHint ?? res.headers.get("content-type"),
|
||||
filePath: candidate.fileHint ?? candidate.url,
|
||||
});
|
||||
const saved = await saveMediaBuffer(
|
||||
buffer,
|
||||
mime,
|
||||
"inbound",
|
||||
params.maxBytes,
|
||||
);
|
||||
out.push({
|
||||
path: saved.path,
|
||||
contentType: saved.contentType,
|
||||
placeholder: candidate.placeholder,
|
||||
});
|
||||
} catch {
|
||||
// Ignore download failures and continue.
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function buildMSTeamsMediaPayload(
|
||||
mediaList: Array<{ path: string; contentType?: string }>,
|
||||
): {
|
||||
MediaPath?: string;
|
||||
MediaType?: string;
|
||||
MediaUrl?: string;
|
||||
MediaPaths?: string[];
|
||||
MediaUrls?: string[];
|
||||
MediaTypes?: string[];
|
||||
} {
|
||||
const first = mediaList[0];
|
||||
const mediaPaths = mediaList.map((media) => media.path);
|
||||
const mediaTypes = mediaList.map((media) => media.contentType ?? "");
|
||||
return {
|
||||
MediaPath: first?.path,
|
||||
MediaType: first?.contentType,
|
||||
MediaUrl: first?.path,
|
||||
MediaPaths: mediaPaths.length > 0 ? mediaPaths : undefined,
|
||||
MediaUrls: mediaPaths.length > 0 ? mediaPaths : undefined,
|
||||
MediaTypes: mediaPaths.length > 0 ? mediaTypes : undefined,
|
||||
};
|
||||
}
|
||||
@@ -13,6 +13,12 @@ import {
|
||||
} from "../pairing/pairing-store.js";
|
||||
import { resolveAgentRoute } from "../routing/resolve-route.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import {
|
||||
buildMSTeamsAttachmentPlaceholder,
|
||||
buildMSTeamsMediaPayload,
|
||||
downloadMSTeamsImageAttachments,
|
||||
type MSTeamsAttachmentLike,
|
||||
} from "./attachments.js";
|
||||
import type {
|
||||
MSTeamsConversationStore,
|
||||
StoredConversationReference,
|
||||
@@ -82,6 +88,11 @@ export async function monitorMSTeamsProvider(
|
||||
|
||||
const port = msteamsCfg.webhook?.port ?? 3978;
|
||||
const textLimit = resolveTextChunkLimit(cfg, "msteams");
|
||||
const MB = 1024 * 1024;
|
||||
const mediaMaxBytes =
|
||||
typeof cfg.agent?.mediaMaxMb === "number" && cfg.agent.mediaMaxMb > 0
|
||||
? Math.floor(cfg.agent.mediaMaxMb * MB)
|
||||
: 8 * MB;
|
||||
const conversationStore =
|
||||
opts.conversationStore ?? createMSTeamsConversationStoreFs();
|
||||
|
||||
@@ -94,6 +105,7 @@ export async function monitorMSTeamsProvider(
|
||||
const {
|
||||
ActivityHandler,
|
||||
CloudAdapter,
|
||||
MsalTokenProvider,
|
||||
authorizeJWT,
|
||||
getAuthConfigWithDefaults,
|
||||
} = agentsHosting;
|
||||
@@ -104,6 +116,7 @@ export async function monitorMSTeamsProvider(
|
||||
clientSecret: creds.appPassword,
|
||||
tenantId: creds.tenantId,
|
||||
});
|
||||
const tokenProvider = new MsalTokenProvider(authConfig);
|
||||
const adapter = new CloudAdapter(authConfig);
|
||||
|
||||
// Handler for incoming messages
|
||||
@@ -111,17 +124,32 @@ export async function monitorMSTeamsProvider(
|
||||
const activity = context.activity;
|
||||
const rawText = activity.text?.trim() ?? "";
|
||||
const text = stripMSTeamsMentionTags(rawText);
|
||||
const attachments = Array.isArray(activity.attachments)
|
||||
? (activity.attachments as unknown as MSTeamsAttachmentLike[])
|
||||
: [];
|
||||
const attachmentPlaceholder =
|
||||
buildMSTeamsAttachmentPlaceholder(attachments);
|
||||
const rawBody = text || attachmentPlaceholder;
|
||||
const from = activity.from;
|
||||
const conversation = activity.conversation;
|
||||
|
||||
const attachmentTypes = attachments
|
||||
.map((att) =>
|
||||
typeof att.contentType === "string" ? att.contentType : undefined,
|
||||
)
|
||||
.filter(Boolean)
|
||||
.slice(0, 3);
|
||||
|
||||
log.info("received message", {
|
||||
rawText: rawText.slice(0, 50),
|
||||
text: text.slice(0, 50),
|
||||
attachments: attachments.length,
|
||||
attachmentTypes,
|
||||
from: from?.id,
|
||||
conversation: conversation?.id,
|
||||
});
|
||||
|
||||
if (!text) {
|
||||
if (!rawBody) {
|
||||
log.debug("skipping empty message after stripping mentions");
|
||||
return;
|
||||
}
|
||||
@@ -189,7 +217,7 @@ export async function monitorMSTeamsProvider(
|
||||
},
|
||||
});
|
||||
|
||||
const preview = text.replace(/\s+/g, " ").slice(0, 160);
|
||||
const preview = rawBody.replace(/\s+/g, " ").slice(0, 160);
|
||||
const inboundLabel = isDirectMessage
|
||||
? `Teams DM from ${senderName}`
|
||||
: `Teams message in ${conversationType} from ${senderName}`;
|
||||
@@ -274,11 +302,22 @@ export async function monitorMSTeamsProvider(
|
||||
|
||||
// Format the message body with envelope
|
||||
const timestamp = parseMSTeamsActivityTimestamp(activity.timestamp);
|
||||
const mediaList = await downloadMSTeamsImageAttachments({
|
||||
attachments,
|
||||
maxBytes: mediaMaxBytes,
|
||||
tokenProvider: {
|
||||
getAccessToken: (scope) => tokenProvider.getAccessToken(scope),
|
||||
},
|
||||
});
|
||||
if (mediaList.length > 0) {
|
||||
log.debug("downloaded image attachments", { count: mediaList.length });
|
||||
}
|
||||
const mediaPayload = buildMSTeamsMediaPayload(mediaList);
|
||||
const body = formatAgentEnvelope({
|
||||
provider: "Teams",
|
||||
from: senderName,
|
||||
timestamp,
|
||||
body: text,
|
||||
body: rawBody,
|
||||
});
|
||||
|
||||
// Build context payload for agent
|
||||
@@ -300,6 +339,7 @@ export async function monitorMSTeamsProvider(
|
||||
CommandAuthorized: true,
|
||||
OriginatingChannel: "msteams" as const,
|
||||
OriginatingTo: teamsTo,
|
||||
...mediaPayload,
|
||||
};
|
||||
|
||||
if (shouldLogVerbose()) {
|
||||
|
||||
Reference in New Issue
Block a user