The best way to control Claude Code, Codex and more from any computer or phone

main
ts 457 lines 17.6 KB
Raw
1 import type { Nodes } from "mdast";
2 import remarkGfm from "remark-gfm";
3 import remarkMath from "remark-math";
4 import remarkParse from "remark-parse";
5 import { unified } from "unified";
6 import { describe, expect, it } from "vite-plus/test";
7
8 import { mathKind } from "./markdownMath.ts";
9 import { markdownMathRanges, parseMarkdownWithMath } from "./markdownMath.ts";
10
11 const parse = parseMarkdownWithMath;
12
13 function collect(node: Nodes, out: Nodes[] = []): Nodes[] {
14 out.push(node);
15 if ("children" in node) for (const child of node.children) collect(child, out);
16 return out;
17 }
18
19 /** Formulas as `[kind, tex]`, in document order. */
20 function formulas(markdown: string) {
21 return collect(parse(markdown)).flatMap((node) => {
22 if (node.type === "math") return [["display", node.value]];
23 if (node.type !== "inlineMath") return [];
24 const className = node.data?.hProperties?.className;
25 return [[mathKind(Array.isArray(className) ? className.join(" ") : undefined), node.value]];
26 });
27 }
28
29 function text(markdown: string) {
30 return collect(parse(markdown))
31 .flatMap((node) => (node.type === "text" ? [node.value] : []))
32 .join("");
33 }
34
35 describe("chat math parsing", () => {
36 it("keeps GFM autolink destinations intact while restoring literal dollars", () => {
37 const source = "Query www.example.com/?$select=id and $x$.";
38 const link = collect(parse(source)).find((node) => node.type === "link");
39 expect(link).toMatchObject({ url: "http://www.example.com/?$select=id" });
40 expect(formulas(source)).toEqual([["inline", "x"]]);
41 });
42
43 it.each(["", "", ""])(
44 "preserves decoded private-use reference %s",
45 (reference) => {
46 const source = `${reference} [${reference}](https://example.com/?q=${reference}) $x$`;
47 const nodes = collect(parse(source));
48 expect(
49 nodes
50 .filter((node) => node.type === "text")
51 .map((node) => node.value)
52 .join(""),
53 ).toBe("\uE000 \uE000 ");
54 expect(nodes.find((node) => node.type === "link")).toMatchObject({
55 url: "https://example.com/?q=\uE000",
56 });
57 expect(formulas(source)).toEqual([["inline", "x"]]);
58 },
59 );
60
61 it.each(["\\(x\\)", "$x$", "$$x$$"])("preserves indented code containing %s", (formula) => {
62 const source = ` ${formula}`;
63 expect(collect(parse(source)).find((node) => node.type === "code")).toMatchObject({
64 value: formula,
65 });
66 expect(markdownMathRanges(source)).toEqual([]);
67 });
68
69 it("parses formulas without reserving private-use characters", () => {
70 const characters = Array.from({ length: 6400 }, (_, index) =>
71 String.fromCharCode(0xe000 + index),
72 ).join("");
73 const source = "$x$ " + characters;
74 expect(formulas(source)).toEqual([["inline", "x"]]);
75 expect(text(source)).toBe(" " + characters);
76 });
77
78 it("only exposes closed explicit math fences to editors and native rendering", () => {
79 expect(markdownMathRanges("```math\nx^2")).toEqual([]);
80 expect(markdownMathRanges("```math\nx^2\n```")[0]?.tex).toBe("x^2");
81 expect(markdownMathRanges("> ```math\n> x^2\n> ```")[0]?.tex).toBe("x^2");
82 });
83 it("reads the four delimiters models emit", () => {
84 expect(formulas("Inline $x^2$ and \\(y_1\\), display $$a+b$$ and \\[c\\].")).toEqual([
85 ["inline", "x^2"],
86 ["inline", "y_1"],
87 ["display", "a+b"],
88 ["display", "c"],
89 ]);
90 });
91
92 it("reads display blocks on their own lines", () => {
93 expect(formulas("$$\n\\frac{a}{b}\n$$")).toEqual([["display", "\\frac{a}{b}"]]);
94 expect(
95 formulas("Where\n\\[\n\\begin{aligned} a &= b \\\\ c &= d \\end{aligned}\n\\]\nholds."),
96 ).toEqual([["display", "\\begin{aligned} a &= b \\\\ c &= d \\end{aligned}"]]);
97 });
98
99 it.each([" is the first formula.", ", followed by prose.", " \tthen more text."])(
100 "keeps text after a same-line formula separate from later display math: %s",
101 (trailing) => {
102 const source = `$$a$$${trailing}\nSome prose.\n$$\nb\n$$`;
103 const nodes = collect(parse(source));
104 expect(
105 nodes.filter((node) => node.type === "inlineMath" || node.type === "math"),
106 ).toMatchObject([
107 { type: "inlineMath", value: "a" },
108 { type: "math", value: "b" },
109 ]);
110 expect(text(source)).toContain(`${trailing}\nSome prose.`);
111 expect(markdownMathRanges(source).map((formula) => formula.source)).toEqual([
112 "$$a$$",
113 "$$\nb\n$$",
114 ]);
115 },
116 );
117
118 it("keeps block syntax inside display blocks as TeX", () => {
119 expect(formulas("\\[\nx\n- y\n# z\n\\]\nafter")).toEqual([["display", "x\n- y\n# z"]]);
120 expect(formulas("> $$\n> x\n> $$")).toEqual([["display", "x"]]);
121 // Same-line content is not a fence; it stays one formula.
122 expect(formulas("$$x^2\n+y^2$$")).toEqual([["display", "x^2\n+y^2"]]);
123 expect(text("$$E=mc^2")).toBe("$$E=mc^2");
124 });
125
126 it("keeps prices, shell variables, and skill tokens as prose", () => {
127 for (const prose of [
128 "It costs $5 and $10.",
129 "Budget $5-$10 per seat, or $20,000-$30,000 a year.",
130 "Plans are $5/$10 monthly.",
131 "Pay $5,$10 now.",
132 "Run $browser then $deploy on it.",
133 "Set $HOME before running.",
134 "Set $HOME,$PATH before running.",
135 "Set $HOME, $PATH before running.",
136 "Set $HOME,$_PATH before running.",
137 "Set $home,$path before running.",
138 "Set $HOME,${PATH} before running.",
139 "Set $HOME,$? before running.",
140 "Set $HOME,$@ before running.",
141 "Set $HOME,$(pwd) before running.",
142 "Pay $1,000,$2,000 now.",
143 ]) {
144 expect(formulas(prose), prose).toEqual([]);
145 expect(text(prose), prose).toBe(prose);
146 }
147 });
148
149 it("keeps prices as prose when math follows on the same line", () => {
150 for (const price of ["$5-$10", "$5 and $10", "$5/$10", "$20,000-$30,000"]) {
151 const source = `Price ${price} stays literal. The formula is $x_i$ and $5$.`;
152 expect(formulas(source)).toEqual([
153 ["inline", "x_i"],
154 ["inline", "5"],
155 ]);
156 expect(text(source)).toBe(`Price ${price} stays literal. The formula is and .`);
157 }
158 });
159
160 it("reads formulas that end in a comma", () => {
161 for (const boundary of ["", " ", "\n", ".", ",", ";", ":", ")", "]", "}"]) {
162 expect(formulas(`$x,$${boundary}`)).toEqual([["inline", "x,"]]);
163 }
164 expect(formulas("Where $x = 1,$ with $$a,$$ and $$ b,$$.")).toEqual([
165 ["inline", "x = 1,"],
166 ["display", "a,"],
167 ["display", " b,"],
168 ]);
169 });
170
171 it("leaves code, escapes, links, and citations alone", () => {
172 for (const [markdown, rendered] of [
173 ["Use `$x$` in code.", "Use in code."],
174 ["It costs $5 and $10, and `$HOME` stays code.", "It costs $5 and $10, and stays code."],
175 ["Export $PATH, then `$HOME`.", "Export $PATH, then ."],
176 ["\\(a `\\)` b", "(a b"],
177 [
178 "Set $TOKEN, then open [users](https://graph.microsoft.com/v1.0/users?$select=id).",
179 "Set $TOKEN, then open users.",
180 ],
181 [
182 "Set $TOKEN, then open <https://graph.microsoft.com/v1.0/users?$select=id>.",
183 "Set $TOKEN, then open https://graph.microsoft.com/v1.0/users?$select=id.",
184 ],
185 ["```\n$x$ \\(y\\)\n```", ""],
186 ["Escaped \\$x\\$ stays.", "Escaped $x$ stays."],
187 ["Escaped \\\\(x\\\\) stays.", "Escaped \\(x\\) stays."],
188 ["See [docs](https://example.com/\\(a\\)) here.", "See docs here."],
189 ["As shown in \\[1\\] and \\[2, 3\\].", "As shown in [1] and [2, 3]."],
190 ] as const) {
191 expect(formulas(markdown), markdown).toEqual([]);
192 expect(text(markdown), markdown).toBe(rendered);
193 }
194 });
195
196 it("starts a formula right after an escaped dollar", () => {
197 expect(formulas("Pay \\$$x$ now.")).toEqual([["inline", "x"]]);
198 });
199
200 it("starts a formula right after an escaped backslash", () => {
201 const markdown = "Path \\\\\\(x^2\\) and \\\\\\[y\\].";
202 expect(formulas(markdown)).toEqual([
203 ["inline", "x^2"],
204 ["display", "y"],
205 ]);
206 expect(text(markdown)).toBe("Path \\ and \\.");
207 });
208
209 it.each([
210 ["$a`$HOME`", "$HOME"],
211 ["\\(a`\\)`", "\\)"],
212 ["\\(a\\alpha`\\)`", "\\)"],
213 ["text \\[a`\\]`", "\\]"],
214 ])("preserves an adjacent code span in %s", (source, code) => {
215 expect(formulas(source)).toEqual([]);
216 expect(collect(parse(source)).find((node) => node.type === "inlineCode")).toMatchObject({
217 value: code,
218 });
219 });
220
221 it.each([1, 2])("respects %i backslashes before a display fence", (count) => {
222 const prefix = "a+" + "\\".repeat(count);
223 for (const opening of ["$$", "$$\n"]) {
224 const source = `${opening}${prefix}$$\nb\n$$`;
225 expect(formulas(source)).toEqual([["display", count % 2 === 1 ? `${prefix}$$\nb` : prefix]]);
226 expect(markdownMathRanges(source)[0]?.source).toBe(
227 count % 2 === 1 ? source : `${opening}${prefix}$$`,
228 );
229 }
230 });
231
232 it("keeps TeX escapes inside formulas", () => {
233 expect(formulas("Cost $a\\$b$ and \\(\\left(x\\right)\\).")).toEqual([
234 ["inline", "a\\$b"],
235 ["inline", "\\left(x\\right)"],
236 ]);
237 });
238
239 it("keeps source offsets of the original message", () => {
240 const markdown = "- [ ] check \\(x\\)\n- [x] then $y$";
241 const offsets = collect(parse(markdown)).flatMap((node) =>
242 node.type === "inlineMath" || node.type === "listItem"
243 ? [[node.type, node.position?.start.offset]]
244 : [],
245 );
246 expect(offsets).toEqual([
247 ["listItem", 0],
248 ["inlineMath", markdown.indexOf("\\(")],
249 ["listItem", markdown.indexOf("- [x]")],
250 ["inlineMath", markdown.indexOf("$y$")],
251 ]);
252 });
253
254 it.each(["\n", "\r\n", "\r"])("keeps authored lines and UTF-16 offsets with %j", (eol) => {
255 const source = `😀 \\(x${eol}+y\\)${eol}${eol}- [ ] check $z$`;
256 const nodes = collect(parse(source));
257 const formula = nodes.find(
258 (node) => node.type === "inlineMath" && node.value === "x" + eol + "+y",
259 );
260 expect(formula?.position?.start).toEqual({ line: 1, column: 4, offset: 3 });
261 expect(formula?.position?.end).toEqual({
262 line: 2,
263 column: 5,
264 offset: source.indexOf(eol + eol),
265 });
266 expect(nodes.find((node) => node.type === "listItem")?.position?.start).toEqual({
267 line: 4,
268 column: 1,
269 offset: source.indexOf("- [ ]"),
270 });
271 });
272
273 it.each(["\\( a ", "\\[ a ", "\\[\n"])(
274 "bounds tokenizer work for repeated unclosed %j",
275 (opener) => {
276 let consumed = 0;
277 const parser = unified()
278 .use(remarkParse)
279 .use(remarkGfm)
280 .use(remarkMath)
281 .use(function () {
282 for (const extension of this.data().micromarkExtensions ?? []) {
283 for (const constructs of [extension.text, extension.flow]) {
284 const entries = constructs?.[92];
285 if (!entries || !constructs) continue;
286 constructs[92] = (Array.isArray(entries) ? entries : [entries]).map((construct) => ({
287 ...construct,
288 tokenize(effects, ok, nok) {
289 return construct.tokenize.call(
290 this,
291 {
292 ...effects,
293 consume(code) {
294 consumed++;
295 effects.consume(code);
296 },
297 },
298 ok,
299 nok,
300 );
301 },
302 }));
303 }
304 }
305 });
306 const source = opener.repeat(200);
307 expect(
308 collect(parser.parse(source)).some((node) => /^(inlineMath|math)$/.test(node.type)),
309 ).toBe(false);
310 expect(consumed).toBeGreaterThan(source.length);
311 expect(consumed).toBeLessThan(source.length * 24);
312 },
313 );
314
315 it.each([
316 ["\\( a \\[b\\]", [["display", "b"]]],
317 ["\\[ a \\(b\\)", [["inline", "b"]]],
318 ["\\( a \\( b`code` \\(c\\)", [["inline", "c"]]],
319 ["\\[1\\] \\[x\\]", [["display", "x"]]],
320 ["\\( a\n\n\\(b\\)", [["inline", "b"]]],
321 ["> \\[\n> \\[\n\n\\[\nx\n\\]", [["display", "x"]]],
322 ["\\[\n\\[\nx\n\\]", [["display", "\\[\nx"]]],
323 ["\\(a \\\\\\(b\\)", [["inline", "a \\\\\\(b"]]],
324 ["\\(a \\\\) \\(b\\)", [["inline", "a \\\\) \\(b"]]],
325 ["| a | b |\n| --- | --- |\n| \\(a \\(b | c |\n\n\\(x\\)", [["inline", "x"]]],
326 ])("keeps valid formulas after failed scans in %j", (source, expected) => {
327 expect(formulas(source)).toEqual(expected);
328 });
329
330 it("shows unfinished formulas as text while a response streams", () => {
331 const message = "Energy \\(E = mc^2\\) and $$\\sum_i x_i$$ done.";
332 for (let end = 0; end <= message.length; end++) {
333 expect(() => parse(message.slice(0, end))).not.toThrow();
334 }
335 expect(text("Energy \\(E = mc")).toBe("Energy (E = mc");
336 expect(formulas("$$\nx^2")).toEqual([]);
337 expect(formulas("\\[\nx^2\n- y")).toEqual([]);
338 expect(formulas(message)).toEqual([
339 ["inline", "E = mc^2"],
340 ["display", "\\sum_i x_i"],
341 ]);
342 });
343 });
344
345 describe("math in GFM tables", () => {
346 it("renders math in link labels while retaining dollars in their destinations", () => {
347 const row = "| [norm $|x|$](https://example.com/$path) | y$ |";
348 for (const source of [`${row}\n| --- | --- |`, `| First | Second |\n| --- | --- |\n${row}`]) {
349 const tree = parse(source);
350 const table = tree.children[0];
351 expect(table?.type).toBe("table");
352 if (table?.type !== "table") throw new Error("Missing table");
353 expect(table.children.every((entry) => entry.children.length === 2)).toBe(true);
354 expect(markdownMathRanges(source, tree).map((range) => range.tex)).toEqual(["|x|"]);
355 expect(collect(tree).find((node) => node.type === "link")).toMatchObject({
356 url: "https://example.com/$path",
357 });
358 }
359 });
360
361 it.each(["[a](https://example.com/$x)", "`$x`", '<span title="$x">a</span>'])(
362 "preserves table separators after literal dollars in %s",
363 (literal) => {
364 const row = `| ${literal} | y$ |`;
365 for (const source of [`${row}\n| --- | --- |`, `| First | Second |\n| --- | --- |\n${row}`]) {
366 const tree = parse(source);
367 const table = tree.children[0];
368 expect(table?.type).toBe("table");
369 if (table?.type !== "table") throw new Error("Missing table");
370 expect(table.children.every((entry) => entry.children.length === 2)).toBe(true);
371 expect(markdownMathRanges(source, tree)).toEqual([]);
372 if (literal.startsWith("[a](https://example.com/$x)")) {
373 expect(collect(tree).find((node) => node.type === "link")).toMatchObject({
374 url: "https://example.com/$x",
375 });
376 }
377 }
378 },
379 );
380
381 it("retains multiline math after a prose pipe outside tables", () => {
382 const source = "a | b: \\(x\n+y\\)";
383 expect(formulas(source)).toEqual([["inline", "x\n+y"]]);
384 expect(markdownMathRanges(source)).toEqual([
385 expect.objectContaining({ source: "\\(x\n+y\\)", inTable: false }),
386 ]);
387 });
388 it.each(["\\(", "\\["])("keeps table rows intact when %s closes in a later row", (opener) => {
389 const closer = opener === "\\(" ? "\\)" : "\\]";
390 const source = `| Name | Value |\n| --- | --- |\n| first | ${opener}x |\n| second | y${closer} |\n| valid | $|z|$ |`;
391 const tree = parseMarkdownWithMath(source);
392 const table = tree.children[0];
393 expect(table?.type).toBe("table");
394 if (table?.type !== "table") throw new Error("Missing table");
395 expect(table.children.map((row) => row.children.length)).toEqual([2, 2, 2, 2]);
396 expect(
397 collect(tree)
398 .filter((node) => node.type === "text")
399 .map((node) => node.value),
400 ).toEqual([
401 "Name",
402 "Value",
403 "first",
404 opener.slice(1) + "x",
405 "second",
406 "y" + closer.slice(1),
407 "valid",
408 ]);
409 expect(markdownMathRanges(source)).toEqual([
410 expect.objectContaining({ source: "$|z|$", tex: "|z|" }),
411 ]);
412 expect(JSON.stringify(tree)).not.toMatch(/[\uE000-\uF8FF]/);
413 });
414 it.each(["\\(\\frac{1}{|x|}\\)", "$$|x|$$", "$|x|$"])(
415 "keeps pipes inside %s in one cell and preserves source offsets",
416 (formula) => {
417 const source = `| Name | Value |\n| --- | --- |\n| x | ${formula} |\n\n- [ ] next`;
418 const tree = parseMarkdownWithMath(source);
419 const table = tree.children[0];
420 expect(table?.type).toBe("table");
421 if (table?.type !== "table") throw new Error("Missing table");
422 expect(table.children.map((row) => row.children.length)).toEqual([2, 2]);
423 expect(markdownMathRanges(source)).toEqual([
424 expect.objectContaining({ source: formula, start: source.indexOf(formula) }),
425 ]);
426 const checkbox = collect(tree).find((node) => node.type === "listItem");
427 expect(checkbox?.position?.start.offset).toBe(source.indexOf("- [ ]"));
428 },
429 );
430
431 it("preserves TeX pipe commands inside and outside tables", () => {
432 const source = "| Formula |\n| --- |\n| \\(\\|x\\|\\) |\n\n\\(\\|x\\|\\)";
433 expect(markdownMathRanges(source).map((formula) => formula.tex)).toEqual([
434 "\\|x\\|",
435 "\\|x\\|",
436 ]);
437 });
438
439 it("leaves code, URLs, incomplete math, and private-use text unchanged", () => {
440 const source =
441 "| Value | Other |\n| --- | --- |\n| `\\(a\\|b\\)` | \\(x|y\\) |\n\n\uE000\n\n[link](https://example.com/$x|y$)\n\nUnclosed \\(x|y";
442 const tree = parseMarkdownWithMath(source);
443 expect(markdownMathRanges(source).map((formula) => formula.tex)).toEqual(["x|y"]);
444 expect(JSON.stringify(tree)).toContain("https://example.com/$x|y$");
445 expect(collect(tree).find((node) => node.type === "inlineCode")).toMatchObject({
446 value: "\\(a|b\\)",
447 });
448 expect(JSON.stringify(tree)).toContain("\uE000");
449 });
450
451 it("keeps a display equation's equality line out of Markdown headings", () => {
452 const source = "\\[\na\n=\nb\n\\]";
453 expect(parseMarkdownWithMath(source).children).toEqual([
454 expect.objectContaining({ type: "math", value: "a\n=\nb" }),
455 ]);
456 });
457 });