https://jayreddinjr-jr-odysseus-hf.hf.space/static/js/streamingSegmenter.js

1// streamingSegmenter.js
2//
3// Pure logic for incremental ("block-at-a-time") streaming markdown rendering.
4//
5// While an assistant message streams in, re-rendering the whole accumulated
6// markdown on every token is wasteful (O(N^2)) and recreates DOM nodes, which
7// makes code-block hover buttons flicker. The fix is to FREEZE the leading part
8// of the message that can no longer change, and only re-render the growing tail.
9//
10// This module answers the one hard question that makes freezing safe:
11//
12//     Given the full markdown received so far, how many leading characters can
13//     be finalized without changing the rendered output?
14//
15// The contract callers rely on (`render` is the canonical markdown renderer):
16//
17//     const n = splitFinalized(text, render);
18//     render(text.slice(0, n)) + render(text.slice(n))  ===  render(text)
19//
20// The module is intentionally DOM-free and renderer-agnostic so it can be unit
21// tested in isolation and reused for any markdown renderer with no long-range
22// cross-block dependencies (no reference-style links / footnotes).
23//
24// Known limitations (both bounded by the same mitigation):
25//   - cutIsRenderSafe proves only PRESENT-tense equivalence. If the renderer pairs
26//     an inline delimiter across a blank line (e.g. markdown.js will turn
27//     `*a\n\nb*` into emphasis spanning two paragraphs), a block frozen before the
28//     closing delimiter arrives can disagree with the final full render.
29//   - afterClosedFence boundaries are trusted without the equivalence check, so a
30//     fence the real renderer parses differently (e.g. a stray 4-backtick line) can
31//     be mis-detected as a close.
32//   Both only occur for input the renderer itself handles oddly, and both are
33//   transient: chat.js re-renders the finished message from source, so the settled
34//   output is always canonical.
35
36// A fenced-code delimiter line: up to 3 leading spaces, then >=3 backticks or
37// tildes, then an optional info string.
38const FENCE_RE = /^ {0,3}(`{3,}|~{3,})(.*)$/;
39
40/**
41 * Scan `text` starting at `fromOffset` — which MUST be at top level (callers only
42 * ever advance to a finalized boundary, never into a fence) — and collect the
43 * candidate cut points.
44 *
45 * @returns {{ boundaries: Array<{offset:number, afterClosedFence:boolean}>, inFence:boolean }}
46 *   - A blank-line run at top level yields a boundary at the start of the next
47 *     non-blank line (`afterClosedFence: false`).
48 *   - A fence close yields a boundary just past the closing fence line
49 *     (`afterClosedFence: true`) — such a cut is unconditionally safe, since
50 *     nothing can ever merge into a completed code block.
51 */
52function findBoundaries(text, fromOffset) {
53  const boundaries = [];
54  const n = text.length;
55  let inFence = false;
56  let fenceMarker = '';
57  let i = fromOffset;
58
59  while (i < n) {
60    const nl = text.indexOf('\n', i);
61    const lineEnd = nl === -1 ? n : nl;
62    const afterNl = nl === -1 ? n : nl + 1;
63    const line = text.slice(i, lineEnd);
64    const fence = line.match(FENCE_RE);
65
66    if (fence) {
67      const marker = fence[1];
68      if (!inFence) {
69        inFence = true;
70        fenceMarker = marker;
71      } else if (
72        marker[0] === fenceMarker[0] &&
73        marker.length >= fenceMarker.length &&
74        fence[2].trim() === '' // a closing fence carries no info string
75      ) {
76        inFence = false;
77        fenceMarker = '';
78        boundaries.push({ offset: afterNl, afterClosedFence: true });
79      }
80      i = afterNl;
81    } else if (!inFence && line.trim() === '') {
82      // Consume the entire run of blank lines; the boundary is the start of the
83      // next non-blank line so the finalized side owns the separator and the tail
84      // starts clean.
85      let j = afterNl;
86      while (j < n) {
87        const nl2 = text.indexOf('\n', j);
88        const lineEnd2 = nl2 === -1 ? n : nl2;
89        if (text.slice(j, lineEnd2).trim() !== '') break;
90        if (nl2 === -1) {
91          j = n;
92          break;
93        }
94        j = nl2 + 1;
95      }
96      boundaries.push({ offset: j, afterClosedFence: false });
97      i = j;
98    } else {
99      i = afterNl;
100    }
101  }
102
103  return { boundaries, inFence };
104}
105
106/**
107 * Does cutting between `before` and `after` leave the rendered output unchanged?
108 * This is the self-verifying safety check: it directly compares rendering the two
109 * sides separately against rendering them joined, so constructs that span the cut
110 * (loose lists, setext headings, lazy blockquote continuations, tables) are caught

111 * with no hand-coded grammar rules.
112 *
113 * Renderer non-determinism (e.g. mermaid ids seeded with Date.now()) can only make
114 * this return a false negative, never a false positive — so the bias is always
115 * toward under-finalizing, which is the safe direction.
116 */
117function cutIsRenderSafe(before, after, render) {
118  return render(before) + render(after) === render(before + after);
119}
120
121/**
122 * Return how many leading characters of `text` can be safely finalized, scanning
123 * forward from `committedLen` (the amount already finalized).
124 *
125 * Guarantees `render(text.slice(0, n)) + render(text.slice(n)) === render(text)`,
126 * and `committedLen <= n <= text.length`.
127 *
128 * @param {string} text       Full markdown accumulated so far.
129 * @param {(src:string)=>string} render  Canonical markdown renderer.
130 * @param {number} [committedLen=0]  Characters already finalized (always a prior boundary).
131 * @returns {number}
132 */
133export function splitFinalized(text, render, committedLen = 0) {
134  const { boundaries } = findBoundaries(text, committedLen);
135
136  let best = committedLen;
137  let segStart = committedLen;
138
139  for (let k = 0; k < boundaries.length; k++) {
140    const { offset, afterClosedFence } = boundaries[k];
141
142    if (afterClosedFence) {
143      // A completed code block — always safe to freeze through here.
144      best = offset;
145    } else {
146      // A prose/list/table boundary. We need a following block to compare
147      // against (the last block must stay live, it can still grow), and the cut
148      // must be render-equivalent locally.
149      const nextOffset = k + 1 < boundaries.length ? boundaries[k + 1].offset : text.length;
150      const before = text.slice(segStart, offset);
151      const after = text.slice(offset, nextOffset);
152      if (after.trim() !== '' && cutIsRenderSafe(before, after, render)) {
153        best = offset;
154      }
155    }
156    segStart = offset;
157  }
158
159  return best;
160}
161
162/**
163 * If `text` begins with a fenced-code opener whose fence never closes, describe it
164 * so the renderer can stream the code in append-mode instead of re-rendering it.
165 * Returns `{ lang, contentStart }` (contentStart = offset of the first code char),
166 * or null when `text` does not start with a still-open fence.
167 *
168 * The opener line must be complete (terminated by a newline) so the info string /
169 * language is known before append-mode begins.
170 */
171export function describeOpenFence(text) {
172  const open = text.match(/^( {0,3})(`{3,}|~{3,})([^\n]*)\n/);
173  if (!open) return null;
174  const marker = open[2];
175  const contentStart = open[0].length;
176
177  for (let i = contentStart; i < text.length; ) {
178    const nl = text.indexOf('\n', i);
179    const line = text.slice(i, nl === -1 ? text.length : nl);
180    const close = line.match(/^ {0,3}(`{3,}|~{3,})\s*$/);
181    if (close && close[1][0] === marker[0] && close[1].length >= marker.length) {
182      return null; // the fence closes — let the normal finalize path handle it
183    }
184    if (nl === -1) break;
185    i = nl + 1;
186  }
187
188  const lang = (open[3] || '').trim().split(/\s+/)[0] || '';
189  return { lang, contentStart };
190}