@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 954 lines 39.5 KB
Raw
1 import { expect, it } from "vite-plus/test";
2 import { type PluggableList, unified } from "unified";
3 import rehypeRaw from "rehype-raw";
4 import remarkGfm from "remark-gfm";
5 import remarkParse from "remark-parse";
6 import remarkRehype from "remark-rehype";
7 import type { Link, Root, RootContent } from "mdast";
8 import {
9 CHAT_MARKDOWN_REHYPE_PLUGINS,
10 CHAT_MARKDOWN_REMARK_PLUGINS,
11 CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
12 CHAT_USER_MARKDOWN_REMARK_PLUGINS,
13 remarkAssistantInsights,
14 remarkEnableOrderedListNumbers,
15 } from "./markdownPipeline.ts";
16
17 it.each([
18 ["default", CHAT_MARKDOWN_REMARK_PLUGINS],
19 ["with breaks", CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS],
20 ] as const)("preserves bare URL boundaries (%s)", (_, plugins) => {
21 const processor = unified().use(remarkParse).use(plugins);
22 const shape = (node: Root | RootContent): string => {
23 if (node.type === "text") return node.value;
24 if (node.type === "inlineCode") return `\`${node.value}\``;
25 if (node.type === "code") return `\`\`\`${node.value}\`\`\``;
26 const children = "children" in node ? node.children.map(shape).join("") : "";
27 if (node.type === "link") return `[${children}](${node.url})`;
28 if (node.type === "linkReference") return `[${children}][${node.identifier}]`;
29 if (node.type === "emphasis") return `*${children}*`;
30 if (node.type === "strong") return `**${children}**`;
31 if (node.type === "delete") return `~~${children}~~`;
32 return children;
33 };
34 const link = (url: string) => `[${url}](${/^www\./i.test(url) ? `http://${url}` : url})`;
35 for (const url of [
36 "https://x.com/dimasike_",
37 "http://example.com/name__",
38 "HTTPS://example.com/name_",
39 "https://example.com/a_(b)_",
40 "https://example.com/?name=value_",
41 "https://example.com/#section_",
42 "https://example.com/a_?name=value_#section_",
43 "https://example.com/name~",
44 "https://example.com/name~~",
45 "https://example.com/name_~",
46 "https://example.com/name~_",
47 "https://example.com/name__~~_",
48 "https://example.com/?name=value~",
49 "https://example.com/#section~",
50 "www.example.com/name_",
51 "www.example.com/name~_",
52 "WWW.example.com/name~~",
53 "https://example.com/name-",
54 "https://example.com/name/",
55 "https://example.com/name%5F",
56 "https://example.com/name%7E",
57 ]) {
58 for (const punctuation of ["", ".", ",", "!", "?", ":", ";", ")", "]", '"']) {
59 const markdown = `See ${url}${punctuation} next`;
60 expect(shape(processor.runSync(processor.parse(markdown), markdown) as Root), markdown).toBe(
61 `See ${link(url)}${punctuation} next`,
62 );
63 }
64 }
65 for (const [markdown, expected] of [
66 ["*https://x.com/dimasike_*", `*${link("https://x.com/dimasike_")}*`],
67 ["**https://x.com/dimasike_**", `**${link("https://x.com/dimasike_")}**`],
68 ["_https://x.com/dimasike_", `*${link("https://x.com/dimasike")}*`],
69 ["__https://x.com/dimasike__", `**${link("https://x.com/dimasike")}**`],
70 ["~~https://example.com/name_~~", `~~${link("https://example.com/name_")}~~`],
71 ["~~https://example.com/name~~", `~~${link("https://example.com/name")}~~`],
72 ["~https://example.com/name~", `~~${link("https://example.com/name")}~~`],
73 ["*https://example.com/name~*", `*${link("https://example.com/name~")}*`],
74 ["_https://example.com/name~_", `*${link("https://example.com/name~")}*`],
75 ["[test](https://example.com/name~)", "[test](https://example.com/name~)"],
76 ["[https://x.com/a](https://x.com/a)~_", `${link("https://x.com/a")}~_`],
77 ["<https://x.com/a>~_", `${link("https://x.com/a")}~_`],
78 ["[test][ref]~_\n\n[ref]: https://x.com/a~_", "[test][ref]~_"],
79 ["`https://example.com/name~_`", "`https://example.com/name~_`"],
80 ["```\nhttps://example.com/name~_\n```", "```https://example.com/name~_```"],
81 ["https\\://example.com/name~_", link("https://example.com/name~_")],
82 ["https://example.com/name. _~", `${link("https://example.com/name")}. _~`],
83 ["Mail person@example.com_", "Mail person@example.com_"],
84 ["Mail person@example.com ~_", "Mail [person@example.com](mailto:person@example.com) ~_"],
85 ["[test](https://x.com/dimasike_)", "[test](https://x.com/dimasike_)"],
86 ["[https://x.com/a](https://x.com/a)_", `${link("https://x.com/a")}_`],
87 ["<https://x.com/a>_", `${link("https://x.com/a")}_`],
88 ["[test][ref]_\n\n[ref]: https://x.com/a_", "[test][ref]_"],
89 ["`https://x.com/dimasike_`", "`https://x.com/dimasike_`"],
90 ["```\nhttps://x.com/dimasike_\n```", "```https://x.com/dimasike_```"],
91 ["https://x.com/a. _", `${link("https://x.com/a")}. _`],
92 ["https\\://x.com/dimasike_", link("https://x.com/dimasike_")],
93 ["https://x.com/a\\_", link("https://x.com/a\\_")],
94 ["https://x.com/a&#95;", `${link("https://x.com/a&#95")};`],
95 [
96 "可以看了:**http://localhost:18780**(已重启)。",
97 `可以看了:**${link("http://localhost:18780")}**(已重启)。`,
98 ],
99 ["打开 http://localhost:18780,然后刷新", `打开 ${link("http://localhost:18780")},然后刷新`],
100 ["URL:https://example.com/a。下一句", `URL:${link("https://example.com/a")}。下一句`],
101 ["(预览:http://localhost:18780)", `(预览:${link("http://localhost:18780")})`],
102 [
103 "能打开 http://localhost:3000?见 http://localhost:3001;「www.example.com」",
104 `能打开 ${link("http://localhost:3000")}?见 ${link("http://localhost:3001")};「${link("www.example.com")}」`,
105 ],
106 [
107 "__http://localhost:18780__,*http://localhost:3000*,~~http://localhost:3001~~。",
108 `**${link("http://localhost:18780")}**,*${link("http://localhost:3000")}*,~~${link("http://localhost:3001")}~~。`,
109 ],
110 [
111 "foo**http://localhost:18780**(bar)***http://localhost:3000***。",
112 `foo**${link("http://localhost:18780")}**(bar)***${link("http://localhost:3000")}***。`,
113 ],
114 [
115 "http://a.com。http://b.com,www.c.com、person@example.com。",
116 `${link("http://a.com")}。${link("http://b.com")},${link("www.c.com")}、[person@example.com](mailto:person@example.com)。`,
117 ],
118 [
119 "**http://localhost:18780**(重启)。 https://example.com/a。结束",
120 `**${link("http://localhost:18780")}**(重启)。 ${link("https://example.com/a")}。结束`,
121 ],
122 ["# **http://localhost:3000**,刷新", `**${link("http://localhost:3000")}**,刷新`],
123 [
124 "| a |\n| - |\n| **http://localhost:3000**,刷新 |",
125 `a**${link("http://localhost:3000")}**,刷新`,
126 ],
127 ["www.example.com/preview。下一句", `${link("www.example.com/preview")}。下一句`],
128 ["https://example.com/a_。下一句", `${link("https://example.com/a_")}。下一句`],
129 ["https://example.com/a.,下一句", `${link("https://example.com/a")}.,下一句`],
130 [
131 "https://example.com/search?q=hello,world",
132 `${link("https://example.com/search?q=hello")},world`,
133 ],
134 ["https://example.com/日本語,続き", link("https://example.com/日本語,続き")],
135 [
136 "https://zh.wikipedia.org/wiki/北京(城市)",
137 link("https://zh.wikipedia.org/wiki/北京(城市)"),
138 ],
139 ["https://example.com/a%EF%BC%8Cb", link("https://example.com/a%EF%BC%8Cb")],
140 ["<https://example.com/日本語,続き>", link("https://example.com/日本語,続き")],
141 ["[記事](https://example.com/a,b)", "[記事](https://example.com/a,b)"],
142 ["[http://example.com/a,b](http://example.com/a,b)", link("http://example.com/a,b")],
143 ["`https://example.com/a。次`", "`https://example.com/a。次`"],
144 [
145 "منشور على https://example.com/report، ونسخته",
146 `منشور على ${link("https://example.com/report")}، ونسخته`,
147 ],
148 [
149 "هل يعمل **http://localhost:3000**؟ نعم؛ www.example.com۔",
150 `هل يعمل **${link("http://localhost:3000")}**؟ نعم؛ ${link("www.example.com")}۔`,
151 ],
152 [
153 "https://x.com/a_ https://x.com/b_",
154 `${link("https://x.com/a_")} ${link("https://x.com/b_")}`,
155 ],
156 ] as const) {
157 expect(shape(processor.runSync(processor.parse(markdown), markdown) as Root), markdown).toBe(
158 expected,
159 );
160 }
161 for (const url of ["https://x.com/dimasike_", "www.example.com/name~_"]) {
162 const markdown = `${url}, next`;
163 const boundary = { line: 1, column: url.length + 1, offset: url.length };
164 expect(processor.runSync(processor.parse(markdown), markdown)).toMatchObject({
165 children: [
166 {
167 children: [
168 {
169 type: "link",
170 url: /^www\./i.test(url) ? `http://${url}` : url,
171 position: { end: boundary },
172 children: [{ type: "text", value: url, position: { end: boundary } }],
173 },
174 { type: "text", value: ", next", position: { start: boundary } },
175 ],
176 },
177 ],
178 });
179 }
180 });
181
182 it("ends bare URLs at CJK and Arabic punctuation with source positions", () => {
183 const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS);
184 const parse = (markdown: string) => processor.runSync(processor.parse(markdown), markdown);
185 const at = (line: number, column: number, offset: number) => ({ line, column, offset });
186 const url = "http://localhost:18780";
187 expect(parse(`第一行\n可以看了:**${url}**(已重启)。`)).toMatchObject({
188 children: [
189 {
190 children: [
191 { type: "text", value: "第一行\n可以看了:", position: { end: at(2, 6, 9) } },
192 {
193 type: "strong",
194 position: { start: at(2, 6, 9), end: at(2, 32, 35) },
195 children: [
196 {
197 type: "link",
198 url,
199 position: { start: at(2, 8, 11), end: at(2, 30, 33) },
200 children: [{ type: "text", value: url, position: { end: at(2, 30, 33) } }],
201 },
202 ],
203 },
204 {
205 type: "text",
206 value: "(已重启)。",
207 position: { start: at(2, 32, 35), end: at(2, 38, 41) },
208 },
209 ],
210 },
211 ],
212 });
213 // Markup the URL swallowed comes back once the URL ends.
214 expect(parse(`打开 ${url},然后运行\`pnpm dev\``)).toMatchObject({
215 children: [
216 {
217 children: [
218 { type: "text", value: "打开 " },
219 { type: "link", url },
220 { type: "text", value: ",然后运行" },
221 { type: "inlineCode", value: "pnpm dev" },
222 ],
223 },
224 ],
225 });
226 // Every URL in one paragraph ends at its mark, with positions on its own line.
227 const lines = Array.from({ length: 300 }, (_, index) => `接口:http://localhost:${index},`);
228 const [paragraph] = (parse(lines.join("\n")) as Root).children;
229 const links =
230 paragraph?.type === "paragraph"
231 ? paragraph.children.filter((node): node is Link => node.type === "link")
232 : [];
233 expect(links.map((node) => node.url)).toEqual(
234 lines.map((_, index) => `http://localhost:${index}`),
235 );
236 const lastLine = lines.slice(0, -1).join("\n").length + 1;
237 expect(links.at(-1)).toMatchObject({
238 position: { start: at(300, 4, lastLine + 3), end: at(300, 24, lastLine + 23) },
239 });
240 // Only a table unescapes `\|` in code, so that cell keeps its parse and the URL is cut.
241 const descendants = (node: Root | RootContent): RootContent[] =>
242 "children" in node
243 ? (node.children as RootContent[]).flatMap((child) => [child, ...descendants(child)])
244 : [];
245 const cell = descendants(parse("| a |\n| - |\n| http://example.com, `a\\|b` |") as Root);
246 expect(cell.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
247 "http://example.com",
248 ]);
249 expect(cell.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual(["a|b"]);
250 // A task item's checkbox is not part of its paragraph's text.
251 const task = descendants(parse("- [x] http://example.com,刷新") as Root);
252 expect(task.flatMap((node) => (node.type === "text" ? [node.value] : []))).toEqual([
253 "http://example.com",
254 ",刷新",
255 ]);
256 // A list item's indentation stays out of a code span that crosses lines.
257 const code = descendants(parse("- `echo\n hello` http://example.com,结束") as Root);
258 expect(code.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
259 "http://example.com",
260 ]);
261 expect(code.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual([
262 "echo\nhello",
263 ]);
264 // A list item's continuation line parses the same on its own.
265 expect(parse(`- 第一行\n **${url}**(已重启)`)).toMatchObject({
266 children: [
267 {
268 children: [
269 {
270 children: [
271 {
272 children: [
273 { type: "text", value: "第一行\n" },
274 { type: "strong", children: [{ type: "link", url }] },
275 { type: "text", value: "(已重启)" },
276 ],
277 },
278 ],
279 },
280 ],
281 },
282 ],
283 });
284 // A quote's continuation line does not, so the URL ends and the asterisks stay as text.
285 expect(parse(`> 第一行\n> **${url}**(已重启)`)).toMatchObject({
286 children: [
287 {
288 children: [
289 {
290 children: [
291 { type: "text", value: "第一行\n**" },
292 { type: "link", url, position: { end: at(2, 27, 32) } },
293 { type: "text", value: "**(已重启)", position: { start: at(2, 27, 32) } },
294 ],
295 },
296 ],
297 },
298 ],
299 });
300 });
301
302 const insightRule = "─".repeat(20);
303 const insightFence = (body: string, wrap = "") =>
304 [`${wrap}★ Insight ${insightRule}${wrap}`, body, `${wrap}${insightRule}${wrap}`].join("\n");
305
306 it.each(["", "`"])("groups %j insight fences while preserving surrounding markdown", (wrap) => {
307 const markdown = [
308 "Before **bold** and [reference][src].",
309 "",
310 insightFence("- Also [reference][src].", wrap),
311 "",
312 "After.",
313 "",
314 '[src]:\n https://example.com\n "Source"',
315 ].join("\n");
316 const processor = unified()
317 .use(remarkParse)
318 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
319 .use(remarkAssistantInsights)
320 .use(remarkRehype)
321 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
322 const tree = processor.runSync(processor.parse(markdown), markdown);
323 expect(
324 tree.children.filter((node) => node.type === "element").map((node) => node.tagName),
325 ).toEqual(["p", "blockquote", "p"]);
326 expect(
327 tree.children.find((node) => node.type === "element" && node.tagName === "blockquote"),
328 ).toMatchObject({ properties: { dataAlert: "insight" } });
329 expect(JSON.stringify(tree).match(/"href":"https:\/\/example.com"/g)).toHaveLength(2);
330 expect(JSON.stringify(tree)).toContain('"tagName":"strong"');
331 expect(JSON.stringify(tree)).not.toContain(insightRule);
332 });
333
334 it.each([
335 ["```text", "★ Insight " + insightRule, "literal", insightRule, "```"].join("\n"),
336 ["- ```text", " ★ Insight " + insightRule, " literal", " " + insightRule, " ```"].join("\n"),
337 [" ★ Insight " + insightRule, " literal", " " + insightRule].join("\n"),
338 ["Paragraph.", " ★ Insight " + insightRule, " literal", " " + insightRule].join("\n"),
339 ["`example", "★ Insight " + insightRule, "literal", insightRule, "`"].join("\n"),
340 insightFence("quoted")
341 .split("\n")
342 .map((line) => "> " + line)
343 .join("\n"),
344 ])("leaves literal and nested insight examples unchanged: %s", (markdown) => {
345 const processor = unified().use(remarkParse).use(remarkAssistantInsights);
346 const tree = processor.parse(markdown);
347 expect(processor.runSync(structuredClone(tree), markdown)).toEqual(tree);
348 });
349
350 it("groups streaming bodies and several insights without losing code or headings", () => {
351 const markdown = [
352 "## Setup",
353 "",
354 insightFence("```text\n" + insightRule + "\n```"),
355 "",
356 "## Setup",
357 "",
358 "★ Insight " + insightRule,
359 "Still **streaming**",
360 ].join("\n");
361 const processor = unified().use(remarkParse).use(remarkAssistantInsights);
362 const tree = processor.parse(markdown);
363 processor.runSync(tree, markdown);
364 expect(tree.children.map((node) => node.type)).toEqual([
365 "heading",
366 "blockquote",
367 "heading",
368 "blockquote",
369 ]);
370 expect(tree.children[1]).toMatchObject({ children: [{ type: "code", value: insightRule }] });
371 expect(tree.children[3]).toMatchObject({
372 children: [
373 { type: "paragraph", children: [{ type: "text", value: "Still " }, { type: "strong" }] },
374 ],
375 });
376 });
377
378 it("keeps reference definitions out of unfinished streaming code", () => {
379 const markdown = `[src]: https://example.com\n\n★ Insight ${insightRule}\n\`\`\`ts\nconst value = 1`;
380 const processor = unified().use(remarkParse).use(remarkAssistantInsights);
381 const tree = processor.parse(markdown);
382 processor.runSync(tree, markdown);
383 expect(tree.children.find((node) => node.type === "blockquote")).toMatchObject({
384 children: [{ type: "code", value: "const value = 1" }],
385 });
386 });
387
388 it("preserves footnotes and original code positions across a callout", () => {
389 const code = "```sh\necho hello\n```";
390 const markdown = ["Before[^n].", "", insightFence(code), "", "[^n]: Footnote."].join("\n");
391 const processor = unified()
392 .use(remarkParse)
393 .use(remarkAssistantInsights)
394 .use(CHAT_MARKDOWN_REMARK_PLUGINS);
395 const tree = processor.parse(markdown);
396 processor.runSync(tree, markdown);
397 expect(tree.children[0]).toMatchObject({
398 children: [
399 { type: "text", value: "Before" },
400 { type: "footnoteReference", identifier: "n" },
401 { type: "text", value: "." },
402 ],
403 });
404 expect(tree.children.at(-1)).toMatchObject({ type: "footnoteDefinition", identifier: "n" });
405 const note = tree.children.find((node) => node.type === "blockquote");
406 const block = note?.children[0];
407 expect(markdown.slice(block?.position?.start.offset, block?.position?.end.offset)).toBe(code);
408 });
409
410 it("preserves in-app thread links through the shared Markdown sanitizer", () => {
411 const processor = unified()
412 .use(remarkParse)
413 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
414 .use(remarkRehype, { allowDangerousHtml: true })
415 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
416 const tree = processor.runSync(
417 processor.parse(
418 '[Open thread](t3-thread://v1/thread)\n\n<t3-thread://v2/environment/thread:a%3A1>\n\n<a href="javascript:alert(1)">Unsafe</a>',
419 ),
420 );
421 expect(tree.children[0]).toMatchObject({
422 tagName: "p",
423 children: [
424 {
425 tagName: "a",
426 properties: { href: "t3-thread://v1/thread" },
427 children: [{ type: "text", value: "Open thread" }],
428 },
429 ],
430 });
431 expect(tree.children[2]).toMatchObject({
432 tagName: "p",
433 children: [
434 {
435 tagName: "a",
436 properties: { href: "t3-thread://v2/environment/thread:a%3A1" },
437 children: [{ type: "text", value: "t3-thread://v2/environment/thread:a%3A1" }],
438 },
439 ],
440 });
441 expect(JSON.stringify(tree)).not.toContain("javascript:");
442 });
443
444 it("builds the tree rehype-raw builds for markdown without HTML", () => {
445 type Node = {
446 type: string;
447 tagName?: string;
448 value?: string;
449 position?: unknown;
450 data?: unknown;
451 children?: Node[];
452 };
453 // What the round trip changes that renders the same; see `needsHtmlParser`.
454 const comparable = (node: Node): Node => {
455 const { position, children, data, ...rest } = node;
456 const result: Node =
457 node.type === "text" || node.type === "root" ? rest : { ...rest, position, data };
458 if (!children) return result;
459 const merged: Node[] = [];
460 for (const child of children.map(comparable)) {
461 const last = merged.at(-1);
462 if (child.type === "text" && last?.type === "text") last.value += child.value ?? "";
463 else merged.push(child);
464 }
465 // Blank text inside a table moves in front of it; the JSX runtime drops it in tables.
466 const inTable = /^(?:table|thead|tbody|tfoot|tr)$/.test(node.tagName ?? "");
467 result.children = merged.filter(
468 (child, index) =>
469 child.type !== "text" ||
470 child.value?.trim() ||
471 !(inTable || merged[index + 1]?.tagName === "table"),
472 );
473 return result;
474 };
475 const tree = (rehypePlugins: typeof CHAT_MARKDOWN_REHYPE_PLUGINS, markdown: string) => {
476 const processor = unified()
477 .use(remarkParse)
478 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
479 .use(remarkRehype, { allowDangerousHtml: true })
480 .use(rehypePlugins);
481 return comparable(processor.runSync(processor.parse(markdown)) as Node);
482 };
483 const [first, , ...rest] = CHAT_MARKDOWN_REHYPE_PLUGINS;
484 const withRaw = [first!, rehypeRaw, ...rest];
485 for (const markdown of [
486 "- [ ] open\n- [x] done\n\nNote[^1]\n\n[^1]: The note.",
487 "| a | b |\n| :- | -: |\n| `c` | ![d](./d.png) |\n\n```ts title=x\nconst a = 1;\n```",
488 // The HTML parser moves the footnote's link out of the label's link.
489 "[note[^x]](https://example.com)\n\n[^x]: text",
490 "**a** **b**\r\n> - | a |\r\n> | - |\r\n> | b |\r\n\n-\tx\n\n y",
491 ]) {
492 expect(tree(CHAT_MARKDOWN_REHYPE_PLUGINS, markdown)).toEqual(tree(withRaw, markdown));
493 }
494 });
495
496 it("keeps local editor file links but not other schemes through the shared Markdown sanitizer", () => {
497 const processor = unified()
498 .use(remarkParse)
499 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
500 .use(remarkRehype, { allowDangerousHtml: true })
501 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
502 const tree = processor.runSync(
503 processor.parse(
504 [
505 "[cursor](cursor://file/Users/me/repo/file.md:3)",
506 "[code](vscode://file/Users/me/repo/file.md:3:7)",
507 "[insiders](vscode-insiders://file/Users/me/repo/file.md)",
508 "[unknown](foo://file/Users/me/repo/file.md)",
509 "[zed](zed://file/Users/me/repo/file.md)",
510 ].join(" "),
511 ),
512 );
513 type Node = { tagName?: string; properties?: { href?: string }; children?: Node[] };
514 const hrefs = ((tree.children[0] as Node).children ?? [])
515 .filter((node) => node.tagName === "a")
516 .map((node) => node.properties?.href);
517 expect(hrefs).toEqual([
518 "cursor://file/Users/me/repo/file.md:3",
519 "vscode://file/Users/me/repo/file.md:3:7",
520 "vscode-insiders://file/Users/me/repo/file.md",
521 undefined,
522 undefined,
523 ]);
524 });
525
526 it("keeps mixed-case local editor file links by lowercasing the scheme before sanitizing", () => {
527 const processor = unified()
528 .use(remarkParse)
529 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
530 .use(remarkRehype, { allowDangerousHtml: true })
531 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
532 const tree = processor.runSync(
533 processor.parse(
534 [
535 "[upper](VSCODE://file/Users/me/repo/file.md:3:7)",
536 "[mixed](Cursor://file/Users/me/repo/file.md:3)",
537 "[other](FOO://file/Users/me/repo/file.md)",
538 ].join(" "),
539 ),
540 );
541 type Node = { tagName?: string; properties?: { href?: string }; children?: Node[] };
542 const hrefs = ((tree.children[0] as Node).children ?? [])
543 .filter((node) => node.tagName === "a")
544 .map((node) => node.properties?.href);
545 expect(hrefs).toEqual([
546 "vscode://file/Users/me/repo/file.md:3:7",
547 "cursor://file/Users/me/repo/file.md:3",
548 undefined,
549 ]);
550 });
551
552 it("keeps backslashes that CommonMark would read as escapes in prose Windows paths", () => {
553 const processor = unified()
554 .use(remarkParse)
555 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
556 .use(remarkRehype, { allowDangerousHtml: true })
557 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
558 type Node = { type: string; value?: string; tagName?: string; children?: Node[] };
559 const text = (node: Node): string =>
560 node.type === "text"
561 ? (node.value ?? "")
562 : node.tagName === "code"
563 ? `<code>${(node.children ?? []).map(text).join("")}</code>`
564 : (node.children ?? []).map(text).join("");
565 const render = (markdown: string) => text(processor.runSync(processor.parse(markdown)) as Node);
566
567 expect(
568 render(String.raw`Saved to C:\Users\me\.t3\_build and (C:\.scratch) and C:/work\.config.`),
569 ).toBe(String.raw`Saved to C:\Users\me\.t3\_build and (C:\.scratch) and C:/work\.config.`);
570 expect(render(String.raw`Escaped C:\\Users\\me`)).toBe(String.raw`Escaped C:\\Users\\me`);
571 expect(render(String.raw`See C:\x and \*this\* and Note:\*that\*.`)).toBe(
572 String.raw`See C:\x and *this* and Note:*that*.`,
573 );
574 expect(render(String.raw`${"`"}C:\me\.t3${"`"} and C:\me\.t3`)).toBe(
575 String.raw`<code>C:\me\.t3</code> and C:\me\.t3`,
576 );
577 });
578
579 it("keeps an image on its own line under a label but leaves inline images inline", () => {
580 const processor = unified()
581 .use(remarkParse)
582 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
583 .use(remarkRehype, { allowDangerousHtml: true })
584 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
585 type Node = { type: string; value?: string; tagName?: string; children?: Node[] };
586 const shape = (node: Node): string =>
587 node.type === "text"
588 ? (node.value ?? "")
589 : node.tagName === "br" || node.tagName === "img"
590 ? `<${node.tagName}>`
591 : (node.children ?? []).map(shape).join("");
592 const render = (markdown: string) => shape(processor.runSync(processor.parse(markdown)) as Node);
593
594 expect(
595 render("**Palette**\n![Palette](/tmp/palette.png)\n**Hero** (dark)\n![Hero](/tmp/hero.png)"),
596 ).toBe("Palette<br>\n<img><br>\nHero (dark)<br>\n<img>");
597 expect(render("Built [![CI](/ci.svg)](/ci)\nand wrapped")).toBe("Built <img>\nand wrapped");
598 expect(render("Hard-wrapped\nprose")).toBe("Hard-wrapped\nprose");
599 });
600
601 it("keeps the number written on each ordered list item only when enabled", () => {
602 type Node = { tagName?: string; properties?: { value?: unknown }; children?: Node[] };
603 const values = (node: Node): unknown[] => [
604 ...(node.tagName === "li" ? [node.properties?.value] : []),
605 ...(node.children ?? []).flatMap(values),
606 ];
607 const renderer = (plugins: PluggableList) => {
608 const processor = unified()
609 .use(remarkParse)
610 .use(plugins)
611 .use(remarkRehype, { allowDangerousHtml: true });
612 return (markdown: string) =>
613 values(processor.runSync(processor.parse(markdown), markdown) as Node);
614 };
615 const render = renderer([
616 ...CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
617 remarkEnableOrderedListNumbers,
618 ]);
619
620 expect(render("3. first\n5. second\n6. third")).toEqual([undefined, "5", undefined]);
621 expect(render("1. one\n1. one again")).toEqual([undefined, "1"]);
622 expect(render("> 1. quoted\n> 4. skipped")).toEqual([undefined, "4"]);
623 expect(render("1. outer\n 1. nested\n 7. jump\n9. back")).toEqual([
624 undefined,
625 undefined,
626 "7",
627 "9",
628 ]);
629 expect(render("1. one\n2. two\n- bullet")).toEqual([undefined, undefined, undefined]);
630 expect(render("- first block\n\n 3. first\n 5. second")).toEqual([
631 undefined,
632 undefined,
633 "5",
634 ]);
635 expect(
636 renderer(CHAT_MARKDOWN_REMARK_PLUGINS)(
637 "3. first\n5. second\n\n- first block\n\n 3. first\n 5. second",
638 ),
639 ).toEqual([undefined, undefined, undefined, undefined, undefined]);
640 });
641
642 type ShownNode = {
643 type: string;
644 tagName?: string;
645 value?: string;
646 properties?: Record<string, unknown>;
647 children?: ShownNode[];
648 };
649 const SHOWN_BLOCKS = new Set([
650 "p",
651 "li",
652 "blockquote",
653 "ul",
654 "ol",
655 "pre",
656 "h1",
657 "h2",
658 "table",
659 "tr",
660 ]);
661
662 /** The text a browser shows, by line: whitespace collapses except in `pre` and kept whitespace. */
663 function shownText(tree: ShownNode): string {
664 let out = "";
665 const visit = (node: ShownNode, keep: boolean) => {
666 if (node.type === "text") {
667 const value = node.value ?? "";
668 // Only spaces, tabs and line endings collapse; a no-break space shows.
669 out += keep
670 ? value.replace(/ /g, "\u0001").replace(/\t/g, "\u0002")
671 : value.replace(/[ \t\r\n]/g, " ");
672 }
673 // An image shows as something, which its alt text stands in for.
674 if (node.properties?.alt !== undefined) out += String(node.properties.alt);
675 if (node.tagName === "br" || SHOWN_BLOCKS.has(node.tagName ?? "")) out += "\n";
676 const inner =
677 keep || node.tagName === "pre" || node.properties?.dataLeadingWhitespace !== undefined;
678 node.children?.forEach((child) => visit(child, inner));
679 if (SHOWN_BLOCKS.has(node.tagName ?? "")) out += "\n";
680 };
681 visit(tree, false);
682 return out
683 .split("\n")
684 .map((line) => line.replace(/ +/g, " ").replace(/^ | $/g, ""))
685 .filter(Boolean)
686 .join("\n")
687 .replace(/\u0001/g, " ")
688 .replace(/\u0002/g, "\t");
689 }
690
691 const hastRenderer = (plugins: PluggableList) => {
692 const processor = unified()
693 .use(remarkParse)
694 .use(plugins)
695 .use(remarkRehype, { allowDangerousHtml: true });
696 return (markdown: string) => processor.runSync(processor.parse(markdown), markdown) as ShownNode;
697 };
698 const userMessage = hastRenderer([
699 ...CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
700 ...CHAT_USER_MARKDOWN_REMARK_PLUGINS,
701 ]);
702 // The user pipeline as it was before leading whitespace was kept.
703 const commonMarkUserMessage = hastRenderer([
704 ...CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
705 [remarkGfm, { singleTilde: false }],
706 ]);
707 /** The tree with the restored indentation taken out, with the empty texts and positions it leaves. */
708 const withoutRestoredWhitespace = ({
709 position: _position,
710 ...node
711 }: ShownNode & { position?: unknown }): ShownNode => ({
712 ...node,
713 ...(node.children && {
714 children: node.children
715 .filter(
716 (child) =>
717 child.properties?.dataLeadingWhitespace === undefined &&
718 !(child.type === "text" && child.value === ""),
719 )
720 .map(withoutRestoredWhitespace),
721 }),
722 });
723 /** Each link's text and where it goes. */
724 const linkTargets = (node: ShownNode): [string, unknown][] =>
725 node.properties?.href === undefined
726 ? (node.children ?? []).flatMap(linkTargets)
727 : [[shownText(node), node.properties.href]];
728
729 it("shows the spaces and tabs typed at the start of each line of a user message", () => {
730 const shown = (markdown: string) => shownText(userMessage(markdown));
731 const outline = "Plan\n Step one\n Step two\n Detail a\n Detail b";
732 expect(shown(outline)).toBe(outline);
733 expect(shown(outline.replaceAll("\n", "\r\n"))).toBe(outline);
734 expect(shown(" First line\nnext\n\n New paragraph")).toBe(
735 " First line\nnext\n New paragraph",
736 );
737 expect(shown("Plan\n\tTabbed\n \t Mixed")).toBe("Plan\n\tTabbed\n \t Mixed");
738 expect(shown("Plan \n After a hard break\\\n and an escaped one")).toBe(
739 "Plan\n After a hard break\n and an escaped one",
740 );
741 // In a container, only whitespace past its own structure counts.
742 expect(shown("- item\n continued")).toBe("item\ncontinued");
743 expect(shown("- item\n more")).toBe("item\n more");
744 expect(shown("- item\ncontinued")).toBe("item\ncontinued");
745 expect(shown("- item\n continued")).toBe("item\ncontinued");
746 expect(shown("1. one\n two")).toBe("one\n two");
747 expect(shown("- item\n\tsplit tab")).toBe("item\n split tab");
748 expect(shown("- item\n\n second paragraph")).toBe("item\n second paragraph");
749 // An item's content starts after its marker and the marker's spacing. A task's checkbox is
750 // content, so it does not move where continuation lines start, however the task begins.
751 expect(shown("- [ ] task\n under the task")).toBe("task\n under the task");
752 for (const task of ["task", "**task**", "[task](https://example.com)", "`task`"]) {
753 expect(shown(`- [ ] ${task}\n detail`), task).toBe("task\n detail");
754 }
755 expect(shown("- x\n y")).toBe("x\n y");
756 expect(shown("- x\n y")).toBe("x\n y");
757 expect(shown("-\tx\n\t y")).toBe("x\n y");
758 expect(shown("10. x\n y")).toBe("x\n y");
759 expect(shown("> a\n> b")).toBe("a\n b");
760 expect(shown("> first")).toBe(" first");
761 expect(shown("> a\n lazy")).toBe("a\nlazy");
762 expect(shown("> - item\n> more")).toBe("item\n more");
763 expect(shown("- > a\n > b")).toBe("a\n b");
764 expect(shown("- a\n - b\n c")).toBe("a\nb\n c");
765 // List recovery reparses over-indented content; its first line sets the indentation.
766 expect(shown("- first block\n continued\n deeper")).toBe(
767 "first block\ncontinued\n deeper",
768 );
769
770 // A space or tab written as a character reference after the indentation is dropped, as it is
771 // without the indentation; only the typed indentation is added.
772 for (const reference of ["&#32;", "&#x20;", "&#9;", "&Tab;"]) {
773 expect(shown(`a\n ${reference}b`), reference).toBe("a\n b");
774 expect(shown(`a \n\t${reference}b`), reference).toBe("a\n\tb");
775 expect(shown(`a\\\r\n ${reference}${reference}b`), reference).toBe("a\n b");
776 }
777 // Converting to HTML also drops the spaces that start code or a link's text after a line break.
778 expect(shown("a\n ` npm i`")).toBe("a\n npm i");
779 expect(shown("a\n [ docs](https://example.com)")).toBe("a\n docs");
780 // A line whose text starts with whitespace the browser hides keeps no indentation, as without it.
781 expect(shown(" &#32;b")).toBe("b");
782 expect(shown("a\n ***&#32;b***")).toBe("a\nb");
783
784 // Only the indentation is added: formatting, code, links and alerts render as without it.
785 const inline = "Plan\n **bold** [link](https://example.com) `code` ~/a and ~~gone~~ /tmp/a.ts";
786 const alert = "> [!NOTE]\n> indented body";
787 expect(shown(inline)).toBe("Plan\n bold link code ~/a and gone /tmp/a.ts");
788 expect(shown(alert)).toBe(" indented body");
789 for (const markdown of [inline, alert]) {
790 expect(withoutRestoredWhitespace(userMessage(markdown)), markdown).toEqual(
791 withoutRestoredWhitespace(commonMarkUserMessage(markdown)),
792 );
793 }
794 expect(linkTargets(userMessage(inline))).toEqual(linkTargets(commonMarkUserMessage(inline)));
795 expect(linkTargets(userMessage(inline)).map(([label]) => label)).toEqual(["link", "/tmp/a.ts"]);
796
797 // A line ending written as a character reference is not a source line: it never takes a typed
798 // line's indentation, and no typed line's indentation moves onto it.
799 const decoded = shown("a &#10; b\n c").split("\n");
800 expect(decoded.slice(0, 2)).toEqual(["a", "b"]);
801 expect(["c", " c"]).toContain(decoded[2]);
802
803 // Code, and agent replies, keep CommonMark's reading.
804 for (const markdown of [
805 "```\n code\n more\n```",
806 "Plan\n\n code\n deeper",
807 "- item\n\n code",
808 ]) {
809 expect(userMessage(markdown), markdown).toEqual(commonMarkUserMessage(markdown));
810 }
811 for (const plugins of [CHAT_MARKDOWN_REMARK_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS]) {
812 expect(shownText(hastRenderer(plugins)(outline)).replaceAll("\n", " ")).toBe(
813 "Plan Step one Step two Detail a Detail b",
814 );
815 }
816 });
817
818 it("adds only the typed indentation, and renders what follows it as without it", () => {
819 const starts = [
820 "b",
821 "*b*",
822 "`b`",
823 "~~b~~",
824 "[b](https://example.com)",
825 "![b](https://example.com/b.png)",
826 "https://example.com/b",
827 "/tmp/b.ts",
828 "~/b",
829 "@b",
830 "$b",
831 "&amp;b",
832 "&nbsp;b",
833 // Spaces and tabs written as character references, which a line start drops.
834 "&#32;b",
835 "&#x20;b",
836 "&#9;b",
837 "&Tab;b",
838 "&#32;&#9;b",
839 "&#32;* b",
840 "&#32;`b`",
841 "&#32;**b**",
842 // Content whose first character is a space.
843 "` b`",
844 "[ b](https://example.com)",
845 "**&#32;b**",
846 "***&#32;b***",
847 "[ b]",
848 ];
849 const lines = [
850 (indent: string, start: string, join: string) => `a${join}${indent}${start}`,
851 (indent: string, start: string, join: string) => `> a${join}> ${indent}${start}`,
852 (indent: string, start: string, join: string) => `- a${join} ${indent}${start}`,
853 ];
854 for (const indent of ["", " ", " ", "\t", " \t"]) {
855 for (const start of starts) {
856 // Without the indentation, an image alone on a CRLF line breaks twice: the image-line break
857 // leaves the CR behind, which breaks again.
858 const joins = start.startsWith("!") ? ["\n", " \n", "\\\n"] : ["\n", "\r\n", " \n", "\\\n"];
859 const markdowns = [
860 ...joins.flatMap((join) => lines.map((line) => line(indent, start, join))),
861 // A paragraph's first line takes at most three spaces before it reads as code.
862 ...(indent.includes("\t") ? [] : [`${indent}${start}`, `> ${indent}${start}`]),
863 ];
864 for (const markdown of markdowns) {
865 const label = JSON.stringify(markdown);
866 expect(withoutRestoredWhitespace(userMessage(markdown)), label).toEqual(
867 withoutRestoredWhitespace(commonMarkUserMessage(markdown)),
868 );
869 const shown = shownText(userMessage(markdown)).split("\n");
870 const before = shownText(commonMarkUserMessage(markdown)).split("\n");
871 expect(shown.length, label).toBe(before.length);
872 shown.forEach((line, index) =>
873 expect([before[index], `${indent}${before[index]}`], label).toContain(line),
874 );
875 }
876 }
877 }
878 });
879
880 it("moves a <picture>'s color-scheme sources onto its <img>, which the renderer resolves", () => {
881 const processor = unified()
882 .use(remarkParse)
883 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
884 .use(remarkRehype, { allowDangerousHtml: true })
885 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
886 type Node = { type: string; tagName?: string; properties?: object; children?: Node[] };
887 const tags = (node: Node): string[] => [
888 ...(node.tagName ? [node.tagName] : []),
889 ...(node.children ?? []).flatMap(tags),
890 ];
891 const images = (node: Node): Node[] =>
892 node.tagName === "img" ? [node] : (node.children ?? []).flatMap(images);
893 const tree = processor.runSync(
894 processor.parse(
895 [
896 "<picture>",
897 ' <source media="(prefers-color-scheme: dark)" srcset="docs/dark.webp 1x, docs/dark@2x.webp 2x">',
898 ' <source media="(prefers-color-scheme: dark)" srcset="docs/dark-unused.webp">',
899 ' <source media="(min-width: 600px)" srcset="docs/wide.webp">',
900 ' <img src="docs/light.webp" alt="Studio">',
901 "</picture>",
902 "",
903 '<picture><source media="(prefers-color-scheme: light)" srcset="javascript:alert(1)"><img src="docs/dark.webp"></picture>',
904 "",
905 String.raw`<picture><source media="(prefers-color-scheme: dark)" srcset="C:\shots\dark.png"><img src="C:\shots\light.png"></picture>`,
906 ].join("\n"),
907 ),
908 ) as Node;
909
910 expect(tags(tree)).toEqual(["picture", "img", "p", "picture", "img", "p", "picture", "img"]);
911 expect(images(tree).map((image) => image.properties)).toEqual([
912 { src: "docs/light.webp", alt: "Studio", dataDarkSrc: "docs/dark.webp" },
913 { src: "docs/dark.webp" },
914 { dataLocalSrc: String.raw`C:\shots\light.png`, dataDarkSrc: "file:///C:/shots/dark.png" },
915 ]);
916 });
917
918 it("links complete absolute paths in prose and skips shortened ones", () => {
919 const path =
920 "/tmp/claude-1000/-home-user--t3-worktrees-app-t3-abc123/session/scratchpad/notes.md";
921 const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS);
922 const links = (markdown: string) => {
923 const urls: string[] = [];
924 const visit = (node: Root | RootContent) => {
925 if (node.type === "link") urls.push(node.url);
926 if ("children" in node) node.children.forEach(visit);
927 };
928 visit(processor.runSync(processor.parse(markdown), markdown) as Root);
929 return urls;
930 };
931 expect(links(`Saved to ${path} and /tmp/other.txt.`)).toEqual([path, "/tmp/other.txt"]);
932 expect(links(`See (${path}:12).`)).toEqual([`${path}:12`]);
933 expect(links("See /tmp/report(1).txt. and /tmp/app/[id]/page.tsx.")).toEqual([
934 "/tmp/report(1).txt",
935 "/tmp/app/[id]/page.tsx",
936 ]);
937 expect(links("(see /tmp/claude-1000/...). and /tmp/claude-1000/….")).toEqual([]);
938 expect(links("Served at /index.html; GET /api/users.json; see /tmp/x.md:3.")).toEqual([
939 "/tmp/x.md:3",
940 ]);
941 expect(
942 links(
943 '<a title="</a>" href="https://example.com">/tmp/notes.md</a> and **<a href="x">/tmp/a.md</a>**',
944 ),
945 ).toEqual([]);
946 expect(
947 links("Config lives in /Users/me/Library/Application Support/T3 Code/settings.json now."),
948 ).toEqual([]);
949 expect(links(`/chat/settings and https://example.com${path}`)).toEqual([
950 `https://example.com${path}`,
951 ]);
952 expect(links(`[notes](${path}) and \`${path}\``)).toEqual([path]);
953 expect(links(`\`\`\`\n${path}\n\`\`\``)).toEqual([]);
954 });