@devries-cameron20 / t3code Mirror · read-only / commits / 42bfd1f377
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 13:30 UTC

fix(shared): stop rescanning unclosed TeX math (#34398)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Lakshmi Tanmay committed Nov 15, 2026 at 03:04 UTC 42bfd1f37750da082bc8967297653745699abc5e
3 files changed +148 -15
packages/shared/src/markdownMath.test.ts
+61
@@ -1,4 +1,8 @@
1 import type { Nodes } from "mdast";
2 +import remarkGfm from "remark-gfm";
3 +import remarkMath from "remark-math";
4 +import remarkParse from "remark-parse";
5 +import { unified } from "unified";
6 import { describe, expect, it } from "vite-plus/test";
7
8 import { mathKind } from "./markdownMath.ts";
@@ -266,6 +270,63 @@ describe("chat math parsing", () => {
270 });
271 });
272
273 + it.each(["\\( a ", "\\[ a ", "\\[\n"])(
274 + "bounds tokenizer work for repeated unclosed %j",
275 + (opener) => {
276 + let consumed = 0;
277 + const parser = unified()
278 + .use(remarkParse)
279 + .use(remarkGfm)
280 + .use(remarkMath)
281 + .use(function () {
282 + for (const extension of this.data().micromarkExtensions ?? []) {
283 + for (const constructs of [extension.text, extension.flow]) {
284 + const entries = constructs?.[92];
285 + if (!entries || !constructs) continue;
286 + constructs[92] = (Array.isArray(entries) ? entries : [entries]).map((construct) => ({
287 + ...construct,
288 + tokenize(effects, ok, nok) {
289 + return construct.tokenize.call(
290 + this,
291 + {
292 + ...effects,
293 + consume(code) {
294 + consumed++;
295 + effects.consume(code);
296 + },
297 + },
298 + ok,
299 + nok,
300 + );
301 + },
302 + }));
303 + }
304 + }
305 + });
306 + const source = opener.repeat(200);
307 + expect(
308 + collect(parser.parse(source)).some((node) => /^(inlineMath|math)$/.test(node.type)),
309 + ).toBe(false);
310 + expect(consumed).toBeGreaterThan(source.length);
311 + expect(consumed).toBeLessThan(source.length * 24);
312 + },
313 + );
314 +
315 + it.each([
316 + ["\\( a \\[b\\]", [["display", "b"]]],
317 + ["\\[ a \\(b\\)", [["inline", "b"]]],
318 + ["\\( a \\( b`code` \\(c\\)", [["inline", "c"]]],
319 + ["\\[1\\] \\[x\\]", [["display", "x"]]],
320 + ["\\( a\n\n\\(b\\)", [["inline", "b"]]],
321 + ["> \\[\n> \\[\n\n\\[\nx\n\\]", [["display", "x"]]],
322 + ["\\[\n\\[\nx\n\\]", [["display", "\\[\nx"]]],
323 + ["\\(a \\\\\\(b\\)", [["inline", "a \\\\\\(b"]]],
324 + ["\\(a \\\\) \\(b\\)", [["inline", "a \\\\) \\(b"]]],
325 + ["| a | b |\n| --- | --- |\n| \\(a \\(b | c |\n\n\\(x\\)", [["inline", "x"]]],
326 + ])("keeps valid formulas after failed scans in %j", (source, expected) => {
327 + expect(formulas(source)).toEqual(expected);
328 + });
329 +
330 it("shows unfinished formulas as text while a response streams", () => {
331 const message = "Energy \\(E = mc^2\\) and $$\\sum_i x_i$$ done.";
332 for (let end = 0; end <= message.length; end++) {
patches/micromark-extension-llm-math@3.1.0.patch
+84 -12
@@ -163,7 +163,18 @@ diff --git a/dev/lib/math-flow.js b/dev/lib/math-flow.js
163 diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
164 --- a/dev/lib/math-tex-flow.js
165 +++ b/dev/lib/math-tex-flow.js
166 -@@ -26,6 +26,7 @@
166 +@@ -7,6 +7,10 @@
167 + import {markdownLineEnding} from 'micromark-util-character'
168 + import {codes, constants, types} from 'micromark-util-symbol'
169 +
170 ++// Only EOF proves that every later opener in this flow lacks a closing fence.
171 ++/** @type {WeakMap<TokenizeContext, number>} */
172 ++const failedUntil = new WeakMap()
173 ++
174 + /** @type {Construct} */
175 + export const mathTexFlow = {
176 + tokenize: tokenizeMathFenced,
177 +@@ -26,6 +30,7 @@
178 */
179 function tokenizeMathFenced(effects, ok, nok) {
180 const self = this
@@ -171,7 +182,15 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
182 const tail = self.events[self.events.length - 1]
183 const initialSize =
184 tail && tail[1].type === types.linePrefix
174 -@@ -90,6 +91,7 @@
185 +@@ -47,6 +52,7 @@
186 + * @type {State}
187 + */
188 + function start(code) {
189 ++ if ((failedUntil.get(self) || 0) > self.now().offset) return nok(code)
190 + assert(code === codes.backslash, 'expected `\\`')
191 + effects.enter('mathFlow')
192 + effects.enter('mathFlowFence')
193 +@@ -90,6 +96,7 @@
194 * @type {State}
195 */
196 function metaAfter(code) {
@@ -179,15 +198,18 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
198 effects.exit('mathFlowFence')
199
200 if (self.interrupt) {
182 -@@ -245,6 +247,7 @@
201 +@@ -245,6 +252,10 @@
202 * @type {State}
203 */
204 function after(code) {
186 -+ if (!closed) return nok(code)
205 ++ if (!closed) {
206 ++ if (code === codes.eof) failedUntil.set(self, self.now().offset)
207 ++ return nok(code)
208 ++ }
209 effects.exit('mathFlow')
210 return ok(code)
211 }
190 -@@ -327,6 +330,7 @@
212 +@@ -327,6 +338,7 @@
213 function afterSequenceClose(code) {
214 if (code === codes.eof || markdownLineEnding(code)) {
215 effects.exit('mathFlowFence')
@@ -198,7 +220,32 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
220 diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
221 --- a/dev/lib/math-tex-text.js
222 +++ b/dev/lib/math-tex-text.js
201 -@@ -25,6 +25,7 @@
223 +@@ -1,5 +1,5 @@
224 + /**
225 +- * @import {Construct, Previous, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'
226 ++ * @import {Construct, Event, Previous, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'
227 + */
228 +
229 + import {ok as assert} from 'devlop'
230 +@@ -7,6 +7,12 @@
231 + import {codes, types} from 'micromark-util-symbol'
232 + import {mathText} from './math-text.js'
233 +
234 ++// Failed scans are local to a text tokenizer and delimiter, never shared across documents.
235 ++/** @type {WeakMap<Array<Event>, Record<number, number>>} */
236 ++const failedUntil = new WeakMap()
237 ++/** @type {WeakMap<Array<Event>, Record<number, number>>} */
238 ++const singleLineFailedUntil = new WeakMap()
239 ++
240 + /** @type {Construct} */
241 + export const mathTexText = {
242 + tokenize: tokenizeMathText,
243 +@@ -21,10 +27,12 @@
244 + */
245 + function tokenizeMathText(effects, ok, nok) {
246 + const self = this
247 ++ const failures = self.mathSingleLine ? singleLineFailedUntil : failedUntil
248 + /** @type {Token} */
249 let token
250 /** @type {number} */
251 let openSequenceCode
@@ -206,25 +253,50 @@ diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
253
254 return start
255
209 -@@ -83,7 +84,7 @@
256 +@@ -66,6 +74,8 @@
257 + return nok(code)
258 + }
259 +
260 ++ if ((failures.get(self.events)?.[code] || 0) > self.now().offset) return nok(code)
261 ++
262 + openSequenceCode = code
263 + effects.consume(code)
264 + effects.exit('mathTextSequence')
265 +@@ -83,8 +93,8 @@
266 * @type {State}
267 */
268 function between(code) {
269 - if (code === codes.eof) {
270 +- return nok(code)
271 + if (code === codes.eof || code === codes.graveAccent || (self.mathSingleLine && markdownLineEnding(code))) {
215 - return nok(code)
272 ++ return unmatched(code)
273 }
274
218 -@@ -124,6 +125,8 @@
275 + if (code === codes.backslash) {
276 +@@ -124,6 +134,8 @@
277 * @type {State}
278 */
279 function data(code) {
222 -+ if (code === codes.graveAccent) return nok(code)
280 ++ if (code === codes.graveAccent) return unmatched(code)
281 + if (code !== codes.backslash && code !== codes.eof && code !== codes.space && code !== codes.comma && !markdownLineEnding(code) && !(code >= codes.digit0 && code <= codes.digit9)) substantive = true
282 if (
283 code === codes.eof ||
284 code === codes.space ||
227 -@@ -155,6 +158,7 @@
285 +@@ -138,6 +150,13 @@
286 + return data
287 + }
288 +
289 ++ function unmatched(code) {
290 ++ const ends = failures.get(self.events) || {}
291 ++ ends[openSequenceCode] = self.now().offset
292 ++ failures.set(self.events, ends)
293 ++ return nok(code)
294 ++ }
295 ++
296 + /**
297 + * In closing sequence.
298 + *
299 +@@ -155,6 +174,7 @@
300 ? codes.rightParenthesis
301 : codes.rightSquareBracket
302 if (code === closeSequenceCode) {
@@ -232,7 +304,7 @@ diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
304 effects.consume(code)
305 effects.exit('mathTextSequence')
306 effects.exit('mathText')
235 -@@ -180,5 +184,8 @@
307 +@@ -180,5 +200,8 @@
308 */
309 function previous(code) {
310 // If there is a previous code, there will always be a tail.
pnpm-lock.yaml
+3 -3
@@ -386,7 +386,7 @@ patchedDependencies:
386 mdast-util-math@3.0.0: afe6b2c89e1eaddfd3c90cf2723f12c9a17d513e65beff51653781486013251c
387 mdast-util-to-hast@13.2.1: 57ea46c73aea3c876ab8d351fe0482ddd081dcb4a0610576e399dbe5eb59d2b1
388 micromark-extension-gfm-table@2.1.1: 9ad23a530e45d9f83cba1da8654fe16d5af8b59d5c73d5c7a7c91fd25b9be1d8
389 - micromark-extension-llm-math@3.1.0: 4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b
389 + micromark-extension-llm-math@3.1.0: 6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add
390 micromark@4.0.2: cb79edde7eca83bbcaaafb6ceb7b044b81b987314cde4910823b15257ecada67
391 node-pty@1.2.0-beta.15: f1e680afec85f650699ea756f91f469844ec4c976f4f168492a45c6b3f67d651
392 react-native-gesture-handler@3.5.0: 171379dd378f8f11e9187876dec2116e5cbf5f21cef6576cdde6956518324f98
@@ -21273,7 +21273,7 @@ snapshots:
21273 micromark-util-combine-extensions: 2.0.1
21274 micromark-util-types: 2.0.2
21275
21276 - micromark-extension-llm-math@3.1.0(patch_hash=4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b):
21276 + micromark-extension-llm-math@3.1.0(patch_hash=6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add):
21277 dependencies:
21278 '@types/katex': 0.16.8
21279 devlop: 1.1.0
@@ -22742,7 +22742,7 @@ snapshots:
22742 dependencies:
22743 '@types/mdast': 4.0.4
22744 mdast-util-math: 3.0.0(patch_hash=afe6b2c89e1eaddfd3c90cf2723f12c9a17d513e65beff51653781486013251c)(supports-color@8.1.1)
22745 - micromark-extension-math: micromark-extension-llm-math@3.1.0(patch_hash=4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b)
22745 + micromark-extension-math: micromark-extension-llm-math@3.1.0(patch_hash=6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add)
22746 unified: 11.0.5
22747 transitivePeerDependencies:
22748 - supports-color