feat: enhance BlueBubbles message actions with support for message editing, reply metadata, and improved effect handling

This commit is contained in:
Tyler Yust
2026-01-19 23:40:22 -08:00
committed by Peter Steinberger
parent 2e6c58bf75
commit 574b848863
22 changed files with 1366 additions and 83 deletions

View File

@@ -61,6 +61,9 @@ export type RunEmbeddedPiAgentParams = {
text?: string;
mediaUrls?: string[];
audioAsVoice?: boolean;
replyToId?: string;
replyToTag?: boolean;
replyToCurrent?: boolean;
}) => void | Promise<void>;
onBlockReplyFlush?: () => void | Promise<void>;
blockReplyBreak?: "text_end" | "message_end";

View File

@@ -56,6 +56,9 @@ export type EmbeddedRunAttemptParams = {
text?: string;
mediaUrls?: string[];
audioAsVoice?: boolean;
replyToId?: string;
replyToTag?: boolean;
replyToCurrent?: boolean;
}) => void | Promise<void>;
onBlockReplyFlush?: () => void | Promise<void>;
blockReplyBreak?: "text_end" | "message_end";

View File

@@ -226,13 +226,23 @@ export function handleMessageEnd(
);
} else {
ctx.state.lastBlockReplyText = text;
const { text: cleanedText, mediaUrls, audioAsVoice } = parseReplyDirectives(text);
const {
text: cleanedText,
mediaUrls,
audioAsVoice,
replyToId,
replyToTag,
replyToCurrent,
} = parseReplyDirectives(text);
// Emit if there's content OR audioAsVoice flag (to propagate the flag).
if (cleanedText || (mediaUrls && mediaUrls.length > 0) || audioAsVoice) {
void onBlockReply({
text: cleanedText,
mediaUrls: mediaUrls?.length ? mediaUrls : undefined,
audioAsVoice,
replyToId,
replyToTag,
replyToCurrent,
});
}
}

View File

@@ -342,13 +342,23 @@ export function subscribeEmbeddedPiSession(params: SubscribeEmbeddedPiSessionPar
assistantTexts.push(chunk);
if (!params.onBlockReply) return;
const splitResult = parseReplyDirectives(chunk);
const { text: cleanedText, mediaUrls, audioAsVoice } = splitResult;
const {
text: cleanedText,
mediaUrls,
audioAsVoice,
replyToId,
replyToTag,
replyToCurrent,
} = splitResult;
// Skip empty payloads, but always emit if audioAsVoice is set (to propagate the flag)
if (!cleanedText && (!mediaUrls || mediaUrls.length === 0) && !audioAsVoice) return;
void params.onBlockReply({
text: cleanedText,
mediaUrls: mediaUrls?.length ? mediaUrls : undefined,
audioAsVoice,
replyToId,
replyToTag,
replyToCurrent,
});
};

View File

@@ -19,6 +19,9 @@ export type SubscribeEmbeddedPiSessionParams = {
text?: string;
mediaUrls?: string[];
audioAsVoice?: boolean;
replyToId?: string;
replyToTag?: boolean;
replyToCurrent?: boolean;
}) => void | Promise<void>;
/** Flush pending block replies (e.g., before tool execution to preserve message boundaries). */
onBlockReplyFlush?: () => void | Promise<void>;

View File

@@ -1,6 +1,9 @@
import { describe, expect, it, vi } from "vitest";
import type { MessageActionRunResult } from "../../infra/outbound/message-action-runner.js";
import { setActivePluginRegistry } from "../../plugins/runtime.js";
import type { ChannelPlugin } from "../../channels/plugins/types.js";
import { createTestRegistry } from "../../test-utils/channel-plugins.js";
import { createMessageTool } from "./message-tool.js";
const mocks = vi.hoisted(() => ({
@@ -82,3 +85,59 @@ describe("message tool mirroring", () => {
expect(mocks.appendAssistantMessageToSessionTranscript).not.toHaveBeenCalled();
});
});
describe("message tool description", () => {
const bluebubblesPlugin: ChannelPlugin = {
id: "bluebubbles",
meta: {
id: "bluebubbles",
label: "BlueBubbles",
selectionLabel: "BlueBubbles",
docsPath: "/channels/bluebubbles",
blurb: "BlueBubbles test plugin.",
},
capabilities: { chatTypes: ["direct", "group"], media: true },
config: {
listAccountIds: () => ["default"],
resolveAccount: () => ({}),
},
messaging: {
normalizeTarget: (raw) => {
const trimmed = raw.trim().replace(/^bluebubbles:/i, "");
const lower = trimmed.toLowerCase();
if (lower.startsWith("chat_guid:")) {
const guid = trimmed.slice("chat_guid:".length);
const parts = guid.split(";");
if (parts.length === 3 && parts[1] === "-") {
return parts[2]?.trim() || trimmed;
}
return `chat_guid:${guid}`;
}
return trimmed;
},
},
actions: {
listActions: () =>
["react", "renameGroup", "addParticipant", "removeParticipant", "leaveGroup"] as const,
},
};
it("hides BlueBubbles group actions for DM targets", () => {
setActivePluginRegistry(
createTestRegistry([{ pluginId: "bluebubbles", source: "test", plugin: bluebubblesPlugin }]),
);
const tool = createMessageTool({
config: {} as never,
currentChannelProvider: "bluebubbles",
currentChannelId: "bluebubbles:chat_guid:iMessage;-;+15551234567",
});
expect(tool.description).not.toContain("renameGroup");
expect(tool.description).not.toContain("addParticipant");
expect(tool.description).not.toContain("removeParticipant");
expect(tool.description).not.toContain("leaveGroup");
setActivePluginRegistry(createTestRegistry([]));
});
});

View File

@@ -14,15 +14,23 @@ import {
resolveMirroredTranscriptText,
} from "../../config/sessions.js";
import { GATEWAY_CLIENT_IDS, GATEWAY_CLIENT_MODES } from "../../gateway/protocol/client-info.js";
import { normalizeTargetForProvider } from "../../infra/outbound/target-normalization.js";
import { getToolResult, runMessageAction } from "../../infra/outbound/message-action-runner.js";
import { resolveSessionAgentId } from "../agent-scope.js";
import { normalizeAccountId } from "../../routing/session-key.js";
import { channelTargetSchema, channelTargetsSchema, stringEnum } from "../schema/typebox.js";
import { listChannelSupportedActions } from "../channel-tools.js";
import { normalizeMessageChannel } from "../../utils/message-channel.js";
import type { AnyAgentTool } from "./common.js";
import { jsonResult, readNumberParam, readStringParam } from "./common.js";
const AllMessageActions = CHANNEL_MESSAGE_ACTION_NAMES;
const BLUEBUBBLES_GROUP_ACTIONS = new Set<ChannelMessageActionName>([
"renameGroup",
"addParticipant",
"removeParticipant",
"leaveGroup",
]);
function buildRoutingSchema() {
return {
@@ -37,7 +45,26 @@ function buildRoutingSchema() {
function buildSendSchema(options: { includeButtons: boolean }) {
const props: Record<string, unknown> = {
message: Type.Optional(Type.String()),
effectId: Type.Optional(
Type.String({
description: "Message effect name/id for sendWithEffect (e.g., invisible ink).",
}),
),
effect: Type.Optional(
Type.String({ description: "Alias for effectId (e.g., invisible-ink, balloons)." }),
),
media: Type.Optional(Type.String()),
filename: Type.Optional(Type.String()),
buffer: Type.Optional(
Type.String({
description: "Base64 payload for attachments (optionally a data: URL).",
}),
),
contentType: Type.Optional(Type.String()),
mimeType: Type.Optional(Type.String()),
caption: Type.Optional(Type.String()),
path: Type.Optional(Type.String()),
filePath: Type.Optional(Type.String()),
replyTo: Type.Optional(Type.String()),
threadId: Type.Optional(Type.String()),
asVoice: Type.Optional(Type.Boolean()),
@@ -228,17 +255,43 @@ function resolveAgentAccountId(value?: string): string | undefined {
return normalizeAccountId(trimmed);
}
function filterActionsForContext(params: {
actions: ChannelMessageActionName[];
channel?: string;
currentChannelId?: string;
}): ChannelMessageActionName[] {
const channel = normalizeMessageChannel(params.channel);
if (!channel || channel !== "bluebubbles") return params.actions;
const currentChannelId = params.currentChannelId?.trim();
if (!currentChannelId) return params.actions;
const normalizedTarget =
normalizeTargetForProvider(channel, currentChannelId) ?? currentChannelId;
const lowered = normalizedTarget.trim().toLowerCase();
const isGroupTarget =
lowered.startsWith("chat_guid:") ||
lowered.startsWith("chat_id:") ||
lowered.startsWith("chat_identifier:") ||
lowered.startsWith("group:");
if (isGroupTarget) return params.actions;
return params.actions.filter((action) => !BLUEBUBBLES_GROUP_ACTIONS.has(action));
}
function buildMessageToolDescription(options?: {
config?: ClawdbotConfig;
currentChannel?: string;
currentChannelId?: string;
}): string {
const baseDescription = "Send, delete, and manage messages via channel plugins.";
// If we have a current channel, show only its supported actions
if (options?.currentChannel) {
const channelActions = listChannelSupportedActions({
cfg: options.config,
const channelActions = filterActionsForContext({
actions: listChannelSupportedActions({
cfg: options.config,
channel: options.currentChannel,
}),
channel: options.currentChannel,
currentChannelId: options.currentChannelId,
});
if (channelActions.length > 0) {
// Always include "send" as a base action
@@ -265,6 +318,7 @@ export function createMessageTool(options?: MessageToolOptions): AnyAgentTool {
const description = buildMessageToolDescription({
config: options?.config,
currentChannel: options?.currentChannelProvider,
currentChannelId: options?.currentChannelId,
});
return {

View File

@@ -302,6 +302,9 @@ export async function runAgentTurnWithFallback(params: {
text,
mediaUrls: payload.mediaUrls,
mediaUrl: payload.mediaUrls?.[0],
replyToId: payload.replyToId,
replyToTag: payload.replyToTag,
replyToCurrent: payload.replyToCurrent,
},
params.sessionCtx.MessageSid,
);

View File

@@ -1,4 +1,8 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { ClawdbotConfig } from "../../config/config.js";
import { setActivePluginRegistry } from "../../plugins/runtime.js";
@@ -6,7 +10,14 @@ import { createIMessageTestPlugin, createTestRegistry } from "../../test-utils/c
import { slackPlugin } from "../../../extensions/slack/src/channel.js";
import { telegramPlugin } from "../../../extensions/telegram/src/channel.js";
import { whatsappPlugin } from "../../../extensions/whatsapp/src/channel.js";
import { loadWebMedia } from "../../web/media.js";
import { runMessageAction } from "./message-action-runner.js";
import { jsonResult } from "../../agents/tools/common.js";
import type { ChannelPlugin } from "../../channels/plugins/types.js";
vi.mock("../../web/media.js", () => ({
loadWebMedia: vi.fn(),
}));
const slackConfig = {
channels: {
@@ -64,6 +75,67 @@ describe("runMessageAction context isolation", () => {
afterEach(() => {
setActivePluginRegistry(createTestRegistry([]));
});
it("maps sendAttachment media to buffer + filename", async () => {
const filePath = path.join(os.tmpdir(), `clawdbot-attachment-${Date.now()}.txt`);
await fs.writeFile(filePath, "hello");
const handleAction = vi.fn(async (ctx) => {
return jsonResult({ ok: true, params: ctx.params });
});
const testPlugin: ChannelPlugin = {
id: "bluebubbles",
meta: {
id: "bluebubbles",
label: "BlueBubbles",
selectionLabel: "BlueBubbles",
docsPath: "/channels/bluebubbles",
blurb: "BlueBubbles test plugin.",
},
capabilities: { chatTypes: ["direct", "group"], media: true },
config: {
listAccountIds: () => [],
resolveAccount: () => ({}),
},
messaging: {
targetResolver: {
looksLikeId: () => true,
hint: "<target>",
},
normalizeTarget: (raw) => raw.trim(),
},
actions: {
listActions: () => ["sendAttachment"],
handleAction: handleAction as NonNullable<ChannelPlugin["actions"]>["handleAction"],
},
};
setActivePluginRegistry(
createTestRegistry([{ pluginId: "bluebubbles", source: "test", plugin: testPlugin }]),
);
try {
const result = await runMessageAction({
cfg: { channels: { bluebubbles: {} } } as ClawdbotConfig,
action: "sendAttachment",
params: {
channel: "bluebubbles",
target: "chat_guid:TEST",
media: filePath,
},
dryRun: false,
});
expect(result.kind).toBe("action");
expect(handleAction).toHaveBeenCalledTimes(1);
const params = handleAction.mock.calls[0]?.[0]?.params as Record<string, unknown>;
expect(params.filename).toBe(path.basename(filePath));
expect(params.buffer).toBe(Buffer.from("hello").toString("base64"));
} finally {
await fs.unlink(filePath).catch(() => {});
}
});
it("allows send when target matches current channel", async () => {
const result = await runMessageAction({
cfg: slackConfig,
@@ -80,6 +152,21 @@ describe("runMessageAction context isolation", () => {
expect(result.kind).toBe("send");
});
it("defaults to current channel when target is omitted", async () => {
const result = await runMessageAction({
cfg: slackConfig,
action: "send",
params: {
channel: "slack",
message: "hi",
},
toolContext: { currentChannelId: "C12345678" },
dryRun: true,
});
expect(result.kind).toBe("send");
});
it("allows media-only send when target matches current channel", async () => {
const result = await runMessageAction({
cfg: slackConfig,
@@ -210,6 +297,33 @@ describe("runMessageAction context isolation", () => {
expect(result.kind).toBe("send");
});
it("infers channel + target from tool context when missing", async () => {
const multiConfig = {
channels: {
slack: {
botToken: "xoxb-test",
appToken: "xapp-test",
},
telegram: {
token: "tg-test",
},
},
} as ClawdbotConfig;
const result = await runMessageAction({
cfg: multiConfig,
action: "send",
params: {
message: "hi",
},
toolContext: { currentChannelId: "C12345678", currentChannelProvider: "slack" },
dryRun: true,
});
expect(result.kind).toBe("send");
expect(result.channel).toBe("slack");
});
it("blocks cross-provider sends by default", async () => {
await expect(
runMessageAction({
@@ -253,3 +367,91 @@ describe("runMessageAction context isolation", () => {
).rejects.toThrow(/Cross-context messaging denied/);
});
});
describe("runMessageAction sendAttachment hydration", () => {
const attachmentPlugin: ChannelPlugin = {
id: "bluebubbles",
meta: {
id: "bluebubbles",
label: "BlueBubbles",
selectionLabel: "BlueBubbles",
docsPath: "/channels/bluebubbles",
blurb: "BlueBubbles test plugin.",
},
capabilities: { chatTypes: ["direct"], media: true },
config: {
listAccountIds: () => ["default"],
resolveAccount: () => ({ enabled: true }),
isConfigured: () => true,
},
actions: {
listActions: () => ["sendAttachment"],
supportsAction: ({ action }) => action === "sendAttachment",
handleAction: async ({ params }) =>
jsonResult({
ok: true,
buffer: params.buffer,
filename: params.filename,
caption: params.caption,
contentType: params.contentType,
}),
},
};
beforeEach(() => {
setActivePluginRegistry(
createTestRegistry([
{
pluginId: "bluebubbles",
source: "test",
plugin: attachmentPlugin,
},
]),
);
vi.mocked(loadWebMedia).mockResolvedValue({
buffer: Buffer.from("hello"),
contentType: "image/png",
kind: "image",
fileName: "pic.png",
});
});
afterEach(() => {
setActivePluginRegistry(createTestRegistry([]));
vi.clearAllMocks();
});
it("hydrates buffer and filename from media for sendAttachment", async () => {
const cfg = {
channels: {
bluebubbles: {
enabled: true,
serverUrl: "http://localhost:1234",
password: "test-password",
},
},
} as ClawdbotConfig;
const result = await runMessageAction({
cfg,
action: "sendAttachment",
params: {
channel: "bluebubbles",
target: "+15551234567",
media: "https://example.com/pic.png",
message: "caption",
},
});
expect(result.kind).toBe("action");
expect(result.payload).toMatchObject({
ok: true,
filename: "pic.png",
caption: "caption",
contentType: "image/png",
});
expect((result.payload as { buffer?: string }).buffer).toBe(
Buffer.from("hello").toString("base64"),
);
});
});

View File

@@ -1,3 +1,6 @@
import path from "node:path";
import { fileURLToPath } from "node:url";
import type { AgentToolResult } from "@mariozechner/pi-agent-core";
import {
readNumberParam,
@@ -12,7 +15,12 @@ import type {
ChannelThreadingToolContext,
} from "../../channels/plugins/types.js";
import type { ClawdbotConfig } from "../../config/config.js";
import type { GatewayClientMode, GatewayClientName } from "../../utils/message-channel.js";
import {
isDeliverableMessageChannel,
normalizeMessageChannel,
type GatewayClientMode,
type GatewayClientName,
} from "../../utils/message-channel.js";
import {
listConfiguredMessageChannels,
resolveMessageChannelSelection,
@@ -30,6 +38,8 @@ import {
import { executePollAction, executeSendAction } from "./outbound-send-service.js";
import { actionHasTarget, actionRequiresTarget } from "./message-action-spec.js";
import { resolveChannelTarget } from "./target-resolver.js";
import { loadWebMedia } from "../../web/media.js";
import { extensionForMime } from "../../media/mime.js";
export type MessageActionRunnerGateway = {
url?: string;
@@ -194,6 +204,124 @@ function readBooleanParam(params: Record<string, unknown>, key: string): boolean
return undefined;
}
function resolveAttachmentMaxBytes(params: {
cfg: ClawdbotConfig;
channel: ChannelId;
accountId?: string | null;
}): number | undefined {
const fallback = params.cfg.agents?.defaults?.mediaMaxMb;
if (params.channel !== "bluebubbles") {
return typeof fallback === "number" ? fallback * 1024 * 1024 : undefined;
}
const accountId = typeof params.accountId === "string" ? params.accountId.trim() : "";
const channelCfg = params.cfg.channels?.bluebubbles;
const accountCfg = accountId ? channelCfg?.accounts?.[accountId] : undefined;
const limitMb =
accountCfg?.mediaMaxMb ?? channelCfg?.mediaMaxMb ?? params.cfg.agents?.defaults?.mediaMaxMb;
return typeof limitMb === "number" ? limitMb * 1024 * 1024 : undefined;
}
function inferAttachmentFilename(params: {
mediaHint?: string;
contentType?: string;
}): string | undefined {
const mediaHint = params.mediaHint?.trim();
if (mediaHint) {
try {
if (mediaHint.startsWith("file://")) {
const filePath = fileURLToPath(mediaHint);
const base = path.basename(filePath);
if (base) return base;
} else if (/^https?:\/\//i.test(mediaHint)) {
const url = new URL(mediaHint);
const base = path.basename(url.pathname);
if (base) return base;
} else {
const base = path.basename(mediaHint);
if (base) return base;
}
} catch {
// fall through to content-type based default
}
}
const ext = params.contentType ? extensionForMime(params.contentType) : undefined;
return ext ? `attachment${ext}` : "attachment";
}
function normalizeBase64Payload(params: { base64?: string; contentType?: string }): {
base64?: string;
contentType?: string;
} {
if (!params.base64) return { base64: params.base64, contentType: params.contentType };
const match = /^data:([^;]+);base64,(.*)$/i.exec(params.base64.trim());
if (!match) return { base64: params.base64, contentType: params.contentType };
const [, mime, payload] = match;
return {
base64: payload,
contentType: params.contentType ?? mime,
};
}
async function hydrateSendAttachmentParams(params: {
cfg: ClawdbotConfig;
channel: ChannelId;
accountId?: string | null;
args: Record<string, unknown>;
action: ChannelMessageActionName;
dryRun?: boolean;
}): Promise<void> {
if (params.action !== "sendAttachment") return;
const mediaHint = readStringParam(params.args, "media", { trim: false });
const fileHint =
readStringParam(params.args, "path", { trim: false }) ??
readStringParam(params.args, "filePath", { trim: false });
const contentTypeParam =
readStringParam(params.args, "contentType") ?? readStringParam(params.args, "mimeType");
const caption = readStringParam(params.args, "caption", { allowEmpty: true })?.trim();
const message = readStringParam(params.args, "message", { allowEmpty: true })?.trim();
if (!caption && message) params.args.caption = message;
const rawBuffer = readStringParam(params.args, "buffer", { trim: false });
const normalized = normalizeBase64Payload({
base64: rawBuffer,
contentType: contentTypeParam ?? undefined,
});
if (normalized.base64 !== rawBuffer && normalized.base64) {
params.args.buffer = normalized.base64;
if (normalized.contentType && !contentTypeParam) {
params.args.contentType = normalized.contentType;
}
}
const filename = readStringParam(params.args, "filename");
const mediaSource = mediaHint ?? fileHint;
if (!params.dryRun && !readStringParam(params.args, "buffer", { trim: false }) && mediaSource) {
const maxBytes = resolveAttachmentMaxBytes({
cfg: params.cfg,
channel: params.channel,
accountId: params.accountId,
});
const media = await loadWebMedia(mediaSource, maxBytes);
params.args.buffer = media.buffer.toString("base64");
if (!contentTypeParam && media.contentType) {
params.args.contentType = media.contentType;
}
if (!filename) {
params.args.filename = inferAttachmentFilename({
mediaHint: media.fileName ?? mediaSource,
contentType: media.contentType ?? contentTypeParam ?? undefined,
});
}
} else if (!filename) {
params.args.filename = inferAttachmentFilename({
mediaHint: mediaSource,
contentType: contentTypeParam ?? undefined,
});
}
}
function parseButtonsParam(params: Record<string, unknown>): void {
const raw = params.buttons;
if (typeof raw !== "string") return;
@@ -534,6 +662,29 @@ export async function runMessageAction(
return handleBroadcastAction(input, params);
}
const explicitTarget = typeof params.target === "string" ? params.target.trim() : "";
const hasLegacyTarget =
(typeof params.to === "string" && params.to.trim().length > 0) ||
(typeof params.channelId === "string" && params.channelId.trim().length > 0);
if (
!explicitTarget &&
!hasLegacyTarget &&
actionRequiresTarget(action) &&
!actionHasTarget(action, params)
) {
const inferredTarget = input.toolContext?.currentChannelId?.trim();
if (inferredTarget) {
params.target = inferredTarget;
}
}
const explicitChannel = typeof params.channel === "string" ? params.channel.trim() : "";
if (!explicitChannel) {
const inferredChannel = normalizeMessageChannel(input.toolContext?.currentChannelProvider);
if (inferredChannel && isDeliverableMessageChannel(inferredChannel)) {
params.channel = inferredChannel;
}
}
applyTargetToParams({ action, args: params });
if (actionRequiresTarget(action)) {
if (!actionHasTarget(action, params)) {
@@ -545,6 +696,15 @@ export async function runMessageAction(
const accountId = readStringParam(params, "accountId") ?? input.defaultAccountId;
const dryRun = Boolean(input.dryRun ?? readBooleanParam(params, "dryRun"));
await hydrateSendAttachmentParams({
cfg,
channel,
accountId,
args: params,
action,
dryRun,
});
await resolveActionTarget({
cfg,
channel,
@@ -561,6 +721,14 @@ export async function runMessageAction(
cfg,
});
await hydrateSendAttachmentParams({
cfg,
channel,
accountId,
args: params,
dryRun,
});
const gateway = resolveGateway(input);
if (action === "send") {