The best way to control Claude Code, Codex and more from any computer or phone
fix(shared): stop rescanning unclosed TeX math (#34398)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Lakshmi Tanmay committed
Nov 15, 2026 at 03:04 UTC
42bfd1f37750da082bc8967297653745699abc5e
3 files changed
+148
-15
packages/shared/src/markdownMath.test.ts
+61
@@ -1,4 +1,8 @@
1
import type { Nodes } from "mdast";
2
+import remarkGfm from "remark-gfm";
3
+import remarkMath from "remark-math";
4
+import remarkParse from "remark-parse";
5
+import { unified } from "unified";
6
import { describe, expect, it } from "vite-plus/test";
7
8
import { mathKind } from "./markdownMath.ts";
@@ -266,6 +270,63 @@ describe("chat math parsing", () => {
270
});
271
});
272
273
+ it.each(["\\( a ", "\\[ a ", "\\[\n"])(
274
+ "bounds tokenizer work for repeated unclosed %j",
275
+ (opener) => {
276
+ let consumed = 0;
277
+ const parser = unified()
278
+ .use(remarkParse)
279
+ .use(remarkGfm)
280
+ .use(remarkMath)
281
+ .use(function () {
282
+ for (const extension of this.data().micromarkExtensions ?? []) {
283
+ for (const constructs of [extension.text, extension.flow]) {
284
+ const entries = constructs?.[92];
285
+ if (!entries || !constructs) continue;
286
+ constructs[92] = (Array.isArray(entries) ? entries : [entries]).map((construct) => ({
287
+ ...construct,
288
+ tokenize(effects, ok, nok) {
289
+ return construct.tokenize.call(
290
+ this,
291
+ {
292
+ ...effects,
293
+ consume(code) {
294
+ consumed++;
295
+ effects.consume(code);
296
+ },
297
+ },
298
+ ok,
299
+ nok,
300
+ );
301
+ },
302
+ }));
303
+ }
304
+ }
305
+ });
306
+ const source = opener.repeat(200);
307
+ expect(
308
+ collect(parser.parse(source)).some((node) => /^(inlineMath|math)$/.test(node.type)),
309
+ ).toBe(false);
310
+ expect(consumed).toBeGreaterThan(source.length);
311
+ expect(consumed).toBeLessThan(source.length * 24);
312
+ },
313
+ );
314
+
315
+ it.each([
316
+ ["\\( a \\[b\\]", [["display", "b"]]],
317
+ ["\\[ a \\(b\\)", [["inline", "b"]]],
318
+ ["\\( a \\( b`code` \\(c\\)", [["inline", "c"]]],
319
+ ["\\[1\\] \\[x\\]", [["display", "x"]]],
320
+ ["\\( a\n\n\\(b\\)", [["inline", "b"]]],
321
+ ["> \\[\n> \\[\n\n\\[\nx\n\\]", [["display", "x"]]],
322
+ ["\\[\n\\[\nx\n\\]", [["display", "\\[\nx"]]],
323
+ ["\\(a \\\\\\(b\\)", [["inline", "a \\\\\\(b"]]],
324
+ ["\\(a \\\\) \\(b\\)", [["inline", "a \\\\) \\(b"]]],
325
+ ["| a | b |\n| --- | --- |\n| \\(a \\(b | c |\n\n\\(x\\)", [["inline", "x"]]],
326
+ ])("keeps valid formulas after failed scans in %j", (source, expected) => {
327
+ expect(formulas(source)).toEqual(expected);
328
+ });
329
+
330
it("shows unfinished formulas as text while a response streams", () => {
331
const message = "Energy \\(E = mc^2\\) and $$\\sum_i x_i$$ done.";
332
for (let end = 0; end <= message.length; end++) {
patches/micromark-extension-llm-math@3.1.0.patch
+84
-12
@@ -163,7 +163,18 @@ diff --git a/dev/lib/math-flow.js b/dev/lib/math-flow.js
163
diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
164
--- a/dev/lib/math-tex-flow.js
165
+++ b/dev/lib/math-tex-flow.js
166
-@@ -26,6 +26,7 @@
166
+@@ -7,6 +7,10 @@
167
+ import {markdownLineEnding} from 'micromark-util-character'
168
+ import {codes, constants, types} from 'micromark-util-symbol'
169
+
170
++// Only EOF proves that every later opener in this flow lacks a closing fence.
171
++/** @type {WeakMap<TokenizeContext, number>} */
172
++const failedUntil = new WeakMap()
173
++
174
+ /** @type {Construct} */
175
+ export const mathTexFlow = {
176
+ tokenize: tokenizeMathFenced,
177
+@@ -26,6 +30,7 @@
178
*/
179
function tokenizeMathFenced(effects, ok, nok) {
180
const self = this
@@ -171,7 +182,15 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
182
const tail = self.events[self.events.length - 1]
183
const initialSize =
184
tail && tail[1].type === types.linePrefix
174
-@@ -90,6 +91,7 @@
185
+@@ -47,6 +52,7 @@
186
+ * @type {State}
187
+ */
188
+ function start(code) {
189
++ if ((failedUntil.get(self) || 0) > self.now().offset) return nok(code)
190
+ assert(code === codes.backslash, 'expected `\\`')
191
+ effects.enter('mathFlow')
192
+ effects.enter('mathFlowFence')
193
+@@ -90,6 +96,7 @@
194
* @type {State}
195
*/
196
function metaAfter(code) {
@@ -179,15 +198,18 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
198
effects.exit('mathFlowFence')
199
200
if (self.interrupt) {
182
-@@ -245,6 +247,7 @@
201
+@@ -245,6 +252,10 @@
202
* @type {State}
203
*/
204
function after(code) {
186
-+ if (!closed) return nok(code)
205
++ if (!closed) {
206
++ if (code === codes.eof) failedUntil.set(self, self.now().offset)
207
++ return nok(code)
208
++ }
209
effects.exit('mathFlow')
210
return ok(code)
211
}
190
-@@ -327,6 +330,7 @@
212
+@@ -327,6 +338,7 @@
213
function afterSequenceClose(code) {
214
if (code === codes.eof || markdownLineEnding(code)) {
215
effects.exit('mathFlowFence')
@@ -198,7 +220,32 @@ diff --git a/dev/lib/math-tex-flow.js b/dev/lib/math-tex-flow.js
220
diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
221
--- a/dev/lib/math-tex-text.js
222
+++ b/dev/lib/math-tex-text.js
201
-@@ -25,6 +25,7 @@
223
+@@ -1,5 +1,5 @@
224
+ /**
225
+- * @import {Construct, Previous, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'
226
++ * @import {Construct, Event, Previous, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'
227
+ */
228
+
229
+ import {ok as assert} from 'devlop'
230
+@@ -7,6 +7,12 @@
231
+ import {codes, types} from 'micromark-util-symbol'
232
+ import {mathText} from './math-text.js'
233
+
234
++// Failed scans are local to a text tokenizer and delimiter, never shared across documents.
235
++/** @type {WeakMap<Array<Event>, Record<number, number>>} */
236
++const failedUntil = new WeakMap()
237
++/** @type {WeakMap<Array<Event>, Record<number, number>>} */
238
++const singleLineFailedUntil = new WeakMap()
239
++
240
+ /** @type {Construct} */
241
+ export const mathTexText = {
242
+ tokenize: tokenizeMathText,
243
+@@ -21,10 +27,12 @@
244
+ */
245
+ function tokenizeMathText(effects, ok, nok) {
246
+ const self = this
247
++ const failures = self.mathSingleLine ? singleLineFailedUntil : failedUntil
248
+ /** @type {Token} */
249
let token
250
/** @type {number} */
251
let openSequenceCode
@@ -206,25 +253,50 @@ diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
253
254
return start
255
209
-@@ -83,7 +84,7 @@
256
+@@ -66,6 +74,8 @@
257
+ return nok(code)
258
+ }
259
+
260
++ if ((failures.get(self.events)?.[code] || 0) > self.now().offset) return nok(code)
261
++
262
+ openSequenceCode = code
263
+ effects.consume(code)
264
+ effects.exit('mathTextSequence')
265
+@@ -83,8 +93,8 @@
266
* @type {State}
267
*/
268
function between(code) {
269
- if (code === codes.eof) {
270
+- return nok(code)
271
+ if (code === codes.eof || code === codes.graveAccent || (self.mathSingleLine && markdownLineEnding(code))) {
215
- return nok(code)
272
++ return unmatched(code)
273
}
274
218
-@@ -124,6 +125,8 @@
275
+ if (code === codes.backslash) {
276
+@@ -124,6 +134,8 @@
277
* @type {State}
278
*/
279
function data(code) {
222
-+ if (code === codes.graveAccent) return nok(code)
280
++ if (code === codes.graveAccent) return unmatched(code)
281
+ if (code !== codes.backslash && code !== codes.eof && code !== codes.space && code !== codes.comma && !markdownLineEnding(code) && !(code >= codes.digit0 && code <= codes.digit9)) substantive = true
282
if (
283
code === codes.eof ||
284
code === codes.space ||
227
-@@ -155,6 +158,7 @@
285
+@@ -138,6 +150,13 @@
286
+ return data
287
+ }
288
+
289
++ function unmatched(code) {
290
++ const ends = failures.get(self.events) || {}
291
++ ends[openSequenceCode] = self.now().offset
292
++ failures.set(self.events, ends)
293
++ return nok(code)
294
++ }
295
++
296
+ /**
297
+ * In closing sequence.
298
+ *
299
+@@ -155,6 +174,7 @@
300
? codes.rightParenthesis
301
: codes.rightSquareBracket
302
if (code === closeSequenceCode) {
@@ -232,7 +304,7 @@ diff --git a/dev/lib/math-tex-text.js b/dev/lib/math-tex-text.js
304
effects.consume(code)
305
effects.exit('mathTextSequence')
306
effects.exit('mathText')
235
-@@ -180,5 +184,8 @@
307
+@@ -180,5 +200,8 @@
308
*/
309
function previous(code) {
310
// If there is a previous code, there will always be a tail.
pnpm-lock.yaml
+3
-3
@@ -386,7 +386,7 @@ patchedDependencies:
386
mdast-util-math@3.0.0: afe6b2c89e1eaddfd3c90cf2723f12c9a17d513e65beff51653781486013251c
387
mdast-util-to-hast@13.2.1: 57ea46c73aea3c876ab8d351fe0482ddd081dcb4a0610576e399dbe5eb59d2b1
388
micromark-extension-gfm-table@2.1.1: 9ad23a530e45d9f83cba1da8654fe16d5af8b59d5c73d5c7a7c91fd25b9be1d8
389
- micromark-extension-llm-math@3.1.0: 4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b
389
+ micromark-extension-llm-math@3.1.0: 6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add
390
micromark@4.0.2: cb79edde7eca83bbcaaafb6ceb7b044b81b987314cde4910823b15257ecada67
391
node-pty@1.2.0-beta.15: f1e680afec85f650699ea756f91f469844ec4c976f4f168492a45c6b3f67d651
392
react-native-gesture-handler@3.5.0: 171379dd378f8f11e9187876dec2116e5cbf5f21cef6576cdde6956518324f98
@@ -21273,7 +21273,7 @@ snapshots:
21273
micromark-util-combine-extensions: 2.0.1
21274
micromark-util-types: 2.0.2
21275
21276
- micromark-extension-llm-math@3.1.0(patch_hash=4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b):
21276
+ micromark-extension-llm-math@3.1.0(patch_hash=6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add):
21277
dependencies:
21278
'@types/katex': 0.16.8
21279
devlop: 1.1.0
@@ -22742,7 +22742,7 @@ snapshots:
22742
dependencies:
22743
'@types/mdast': 4.0.4
22744
mdast-util-math: 3.0.0(patch_hash=afe6b2c89e1eaddfd3c90cf2723f12c9a17d513e65beff51653781486013251c)(supports-color@8.1.1)
22745
- micromark-extension-math: micromark-extension-llm-math@3.1.0(patch_hash=4eeb9d9e27ebc75e98795e21c0719005483dad7d233055698257f7d10e91603b)
22745
+ micromark-extension-math: micromark-extension-llm-math@3.1.0(patch_hash=6c191d17ea797a4ac6cc88b901041c5e6c5d4c4e508102a033e3f14e95bd2add)
22746
unified: 11.0.5
22747
transitivePeerDependencies:
22748
- supports-color