fix: make forced block chunking fence-safe
This commit is contained in:
@@ -68,7 +68,7 @@ export class EmbeddedBlockChunker {
|
|||||||
const breakResult =
|
const breakResult =
|
||||||
force && this.#buffer.length <= maxChars
|
force && this.#buffer.length <= maxChars
|
||||||
? this.#pickSoftBreakIndex(this.#buffer, 1)
|
? this.#pickSoftBreakIndex(this.#buffer, 1)
|
||||||
: this.#pickBreakIndex(this.#buffer);
|
: this.#pickBreakIndex(this.#buffer, force ? 1 : undefined);
|
||||||
if (breakResult.index <= 0) {
|
if (breakResult.index <= 0) {
|
||||||
if (force) {
|
if (force) {
|
||||||
emit(this.#buffer);
|
emit(this.#buffer);
|
||||||
@@ -169,8 +169,11 @@ export class EmbeddedBlockChunker {
|
|||||||
return { index: -1 };
|
return { index: -1 };
|
||||||
}
|
}
|
||||||
|
|
||||||
#pickBreakIndex(buffer: string): BreakResult {
|
#pickBreakIndex(buffer: string, minCharsOverride?: number): BreakResult {
|
||||||
const minChars = Math.max(1, Math.floor(this.#chunking.minChars));
|
const minChars = Math.max(
|
||||||
|
1,
|
||||||
|
Math.floor(minCharsOverride ?? this.#chunking.minChars),
|
||||||
|
);
|
||||||
const maxChars = Math.max(minChars, Math.floor(this.#chunking.maxChars));
|
const maxChars = Math.max(minChars, Math.floor(this.#chunking.maxChars));
|
||||||
if (buffer.length < minChars) return { index: -1 };
|
if (buffer.length < minChars) return { index: -1 };
|
||||||
const window = buffer.slice(0, Math.min(maxChars, buffer.length));
|
const window = buffer.slice(0, Math.min(maxChars, buffer.length));
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ describe("block streaming", () => {
|
|||||||
{
|
{
|
||||||
onReplyStart,
|
onReplyStart,
|
||||||
onBlockReply,
|
onBlockReply,
|
||||||
|
disableBlockStreaming: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
agents: {
|
agents: {
|
||||||
@@ -131,6 +132,7 @@ describe("block streaming", () => {
|
|||||||
{
|
{
|
||||||
onReplyStart,
|
onReplyStart,
|
||||||
onBlockReply,
|
onBlockReply,
|
||||||
|
disableBlockStreaming: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
agents: {
|
agents: {
|
||||||
@@ -178,6 +180,7 @@ describe("block streaming", () => {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
onBlockReply,
|
onBlockReply,
|
||||||
|
disableBlockStreaming: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
agents: {
|
agents: {
|
||||||
@@ -234,6 +237,7 @@ describe("block streaming", () => {
|
|||||||
{
|
{
|
||||||
onBlockReply,
|
onBlockReply,
|
||||||
blockReplyTimeoutMs: 10,
|
blockReplyTimeoutMs: 10,
|
||||||
|
disableBlockStreaming: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
agents: {
|
agents: {
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ export function parseFenceSpans(buffer: string): FenceSpan[] {
|
|||||||
open.markerChar === markerChar &&
|
open.markerChar === markerChar &&
|
||||||
markerLen >= open.markerLen
|
markerLen >= open.markerLen
|
||||||
) {
|
) {
|
||||||
const end = nextNewline === -1 ? buffer.length : nextNewline + 1;
|
const end = lineEnd;
|
||||||
spans.push({
|
spans.push({
|
||||||
start: open.start,
|
start: open.start,
|
||||||
end,
|
end,
|
||||||
|
|||||||
Reference in New Issue
Block a user