fix: preserve markdown fences when chunking

src/markdown/fences.ts

@@ -0,0 +1,85 @@
+export type FenceSpan = {
+  start: number;
+  end: number;
+  openLine: string;
+  marker: string;
+  indent: string;
+};
+
+export function parseFenceSpans(buffer: string): FenceSpan[] {
+  const spans: FenceSpan[] = [];
+  let open:
+    | {
+        start: number;
+        markerChar: string;
+        markerLen: number;
+        openLine: string;
+        marker: string;
+        indent: string;
+      }
+    | undefined;
+
+  let offset = 0;
+  while (offset <= buffer.length) {
+    const nextNewline = buffer.indexOf("\n", offset);
+    const lineEnd = nextNewline === -1 ? buffer.length : nextNewline;
+    const line = buffer.slice(offset, lineEnd);
+
+    const match = line.match(/^( {0,3})(`{3,}|~{3,})(.*)$/);
+    if (match) {
+      const indent = match[1];
+      const marker = match[2];
+      const markerChar = marker[0];
+      const markerLen = marker.length;
+      if (!open) {
+        open = {
+          start: offset,
+          markerChar,
+          markerLen,
+          openLine: line,
+          marker,
+          indent,
+        };
+      } else if (
+        open.markerChar === markerChar &&
+        markerLen >= open.markerLen
+      ) {
+        const end = nextNewline === -1 ? buffer.length : nextNewline + 1;
+        spans.push({
+          start: open.start,
+          end,
+          openLine: open.openLine,
+          marker: open.marker,
+          indent: open.indent,
+        });
+        open = undefined;
+      }
+    }
+
+    if (nextNewline === -1) break;
+    offset = nextNewline + 1;
+  }
+
+  if (open) {
+    spans.push({
+      start: open.start,
+      end: buffer.length,
+      openLine: open.openLine,
+      marker: open.marker,
+      indent: open.indent,
+    });
+  }
+
+  return spans;
+}
+
+export function findFenceSpanAt(
+  spans: FenceSpan[],
+  index: number,
+): FenceSpan | undefined {
+  return spans.find((span) => index > span.start && index < span.end);
+}
+
+export function isSafeFenceBreak(spans: FenceSpan[], index: number): boolean {
+  return !findFenceSpanAt(spans, index);
+}