fix: make forced block chunking fence-safe

This commit is contained in:
Peter Steinberger
2026-01-09 21:52:47 +00:00
parent dc3a0bfd1e
commit 3adec35632
3 changed files with 11 additions and 4 deletions

View File

@@ -68,7 +68,7 @@ export class EmbeddedBlockChunker {
const breakResult =
force && this.#buffer.length <= maxChars
? this.#pickSoftBreakIndex(this.#buffer, 1)
: this.#pickBreakIndex(this.#buffer);
: this.#pickBreakIndex(this.#buffer, force ? 1 : undefined);
if (breakResult.index <= 0) {
if (force) {
emit(this.#buffer);
@@ -169,8 +169,11 @@ export class EmbeddedBlockChunker {
return { index: -1 };
}
#pickBreakIndex(buffer: string): BreakResult {
const minChars = Math.max(1, Math.floor(this.#chunking.minChars));
#pickBreakIndex(buffer: string, minCharsOverride?: number): BreakResult {
const minChars = Math.max(
1,
Math.floor(minCharsOverride ?? this.#chunking.minChars),
);
const maxChars = Math.max(minChars, Math.floor(this.#chunking.maxChars));
if (buffer.length < minChars) return { index: -1 };
const window = buffer.slice(0, Math.min(maxChars, buffer.length));

View File

@@ -74,6 +74,7 @@ describe("block streaming", () => {
{
onReplyStart,
onBlockReply,
disableBlockStreaming: false,
},
{
agents: {
@@ -131,6 +132,7 @@ describe("block streaming", () => {
{
onReplyStart,
onBlockReply,
disableBlockStreaming: false,
},
{
agents: {
@@ -178,6 +180,7 @@ describe("block streaming", () => {
},
{
onBlockReply,
disableBlockStreaming: false,
},
{
agents: {
@@ -234,6 +237,7 @@ describe("block streaming", () => {
{
onBlockReply,
blockReplyTimeoutMs: 10,
disableBlockStreaming: false,
},
{
agents: {

View File

@@ -44,7 +44,7 @@ export function parseFenceSpans(buffer: string): FenceSpan[] {
open.markerChar === markerChar &&
markerLen >= open.markerLen
) {
const end = nextNewline === -1 ? buffer.length : nextNewline + 1;
const end = lineEnd;
spans.push({
start: open.start,
end,