feat: add chunking mode option for BlueBubbles (#1645)
* feat: add chunking mode for outbound messages - Introduced `chunkMode` option in various account configurations to allow splitting messages by "length" or "newline". - Updated message processing to handle chunking based on the selected mode. - Added tests for new chunking functionality, ensuring correct behavior for both modes. * feat: enhance chunking mode documentation and configuration - Added `chunkMode` option to the BlueBubbles account configuration, allowing users to choose between "length" and "newline" for message chunking. - Updated documentation to clarify the behavior of the `chunkMode` setting. - Adjusted account merging logic to incorporate the new `chunkMode` configuration. * refactor: simplify chunk mode handling for BlueBubbles - Removed `chunkMode` configuration from various account schemas and types, centralizing chunk mode logic to BlueBubbles only. - Updated `processMessage` to default to "newline" for BlueBubbles chunking. - Adjusted tests to reflect changes in chunk mode handling for BlueBubbles, ensuring proper functionality. * fix: update default chunk mode to 'length' for BlueBubbles - Changed the default value of `chunkMode` from 'newline' to 'length' in the BlueBubbles configuration and related processing functions. - Updated documentation to reflect the new default behavior for chunking messages. - Adjusted tests to ensure the correct default value is returned for BlueBubbles chunk mode.
This commit is contained in:
@@ -1,6 +1,13 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { chunkMarkdownText, chunkText, resolveTextChunkLimit } from "./chunk.js";
|
||||
import {
|
||||
chunkByNewline,
|
||||
chunkMarkdownText,
|
||||
chunkText,
|
||||
chunkTextWithMode,
|
||||
resolveChunkMode,
|
||||
resolveTextChunkLimit,
|
||||
} from "./chunk.js";
|
||||
|
||||
function expectFencesBalanced(chunks: string[]) {
|
||||
for (const chunk of chunks) {
|
||||
@@ -231,3 +238,95 @@ describe("chunkMarkdownText", () => {
|
||||
expect(chunks.join("")).toBe(text);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkByNewline", () => {
|
||||
it("splits text on newlines", () => {
|
||||
const text = "Line one\nLine two\nLine three";
|
||||
const chunks = chunkByNewline(text, 1000);
|
||||
expect(chunks).toEqual(["Line one", "Line two", "Line three"]);
|
||||
});
|
||||
|
||||
it("filters empty lines", () => {
|
||||
const text = "Line one\n\n\nLine two\n\nLine three";
|
||||
const chunks = chunkByNewline(text, 1000);
|
||||
expect(chunks).toEqual(["Line one", "Line two", "Line three"]);
|
||||
});
|
||||
|
||||
it("trims whitespace from lines", () => {
|
||||
const text = " Line one \n Line two ";
|
||||
const chunks = chunkByNewline(text, 1000);
|
||||
expect(chunks).toEqual(["Line one", "Line two"]);
|
||||
});
|
||||
|
||||
it("falls back to length-based for long lines", () => {
|
||||
const text = "Short line\n" + "a".repeat(50) + "\nAnother short";
|
||||
const chunks = chunkByNewline(text, 20);
|
||||
expect(chunks[0]).toBe("Short line");
|
||||
// Long line gets split into multiple chunks
|
||||
expect(chunks[1].length).toBe(20);
|
||||
expect(chunks[2].length).toBe(20);
|
||||
expect(chunks[3].length).toBe(10);
|
||||
expect(chunks[4]).toBe("Another short");
|
||||
});
|
||||
|
||||
it("returns empty array for empty input", () => {
|
||||
expect(chunkByNewline("", 100)).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for whitespace-only input", () => {
|
||||
expect(chunkByNewline(" \n\n ", 100)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkTextWithMode", () => {
|
||||
it("uses length-based chunking for length mode", () => {
|
||||
const text = "Line one\nLine two";
|
||||
const chunks = chunkTextWithMode(text, 1000, "length");
|
||||
expect(chunks).toEqual(["Line one\nLine two"]);
|
||||
});
|
||||
|
||||
it("uses newline-based chunking for newline mode", () => {
|
||||
const text = "Line one\nLine two";
|
||||
const chunks = chunkTextWithMode(text, 1000, "newline");
|
||||
expect(chunks).toEqual(["Line one", "Line two"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveChunkMode", () => {
|
||||
it("returns length as default", () => {
|
||||
expect(resolveChunkMode(undefined, "telegram")).toBe("length");
|
||||
expect(resolveChunkMode({}, "discord")).toBe("length");
|
||||
expect(resolveChunkMode(undefined, "bluebubbles")).toBe("length");
|
||||
});
|
||||
|
||||
it("returns length for internal channel", () => {
|
||||
const cfg = { channels: { bluebubbles: { chunkMode: "newline" as const } } };
|
||||
expect(resolveChunkMode(cfg, "__internal__")).toBe("length");
|
||||
});
|
||||
|
||||
it("supports provider-level overrides for bluebubbles", () => {
|
||||
const cfg = { channels: { bluebubbles: { chunkMode: "newline" as const } } };
|
||||
expect(resolveChunkMode(cfg, "bluebubbles")).toBe("newline");
|
||||
expect(resolveChunkMode(cfg, "discord")).toBe("length");
|
||||
});
|
||||
|
||||
it("supports account-level overrides for bluebubbles", () => {
|
||||
const cfg = {
|
||||
channels: {
|
||||
bluebubbles: {
|
||||
chunkMode: "length" as const,
|
||||
accounts: {
|
||||
primary: { chunkMode: "newline" as const },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(resolveChunkMode(cfg, "bluebubbles", "primary")).toBe("newline");
|
||||
expect(resolveChunkMode(cfg, "bluebubbles", "other")).toBe("length");
|
||||
});
|
||||
|
||||
it("ignores chunkMode for non-bluebubbles providers", () => {
|
||||
const cfg = { channels: { ["telegram" as string]: { chunkMode: "newline" as const } } };
|
||||
expect(resolveChunkMode(cfg, "telegram")).toBe("length");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user