@devries-cameron20 / t3code-1 / commits / 4d641633b2
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(web): bare URLs end at Chinese and Japanese punctuation (#35267)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

sanjai balajee (e/acc) committed Nov 12, 2026 at 22:51 UTC 4d641633b2877bad1e3f754ea88389eec293dc7b
2 files changed +375 -3
packages/shared/src/markdownPipeline.test.ts
+171 -2
@@ -3,7 +3,7 @@ import { type PluggableList, unified } from "unified";
3 import rehypeRaw from "rehype-raw";
4 import remarkParse from "remark-parse";
5 import remarkRehype from "remark-rehype";
6 -import type { Root, RootContent } from "mdast";
6 +import type { Link, Root, RootContent } from "mdast";
7 import {
8 CHAT_MARKDOWN_REHYPE_PLUGINS,
9 CHAT_MARKDOWN_REMARK_PLUGINS,
@@ -15,7 +15,7 @@ import {
15 it.each([
16 ["default", CHAT_MARKDOWN_REMARK_PLUGINS],
17 ["with breaks", CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS],
18 -] as const)("preserves trailing underscores and tildes in bare URLs (%s)", (_, plugins) => {
18 +] as const)("preserves bare URL boundaries (%s)", (_, plugins) => {
19 const processor = unified().use(remarkParse).use(plugins);
20 const shape = (node: Root | RootContent): string => {
21 if (node.type === "text") return node.value;
@@ -90,6 +90,55 @@ it.each([
90 ["https\\://x.com/dimasike_", link("https://x.com/dimasike_")],
91 ["https://x.com/a\\_", link("https://x.com/a\\_")],
92 ["https://x.com/a&#95;", `${link("https://x.com/a&#95")};`],
93 + [
94 + "可以看了:**http://localhost:18780**(已重启)。",
95 + `可以看了:**${link("http://localhost:18780")}**(已重启)。`,
96 + ],
97 + ["打开 http://localhost:18780,然后刷新", `打开 ${link("http://localhost:18780")},然后刷新`],
98 + ["URL:https://example.com/a。下一句", `URL:${link("https://example.com/a")}。下一句`],
99 + ["(预览:http://localhost:18780)", `(预览:${link("http://localhost:18780")})`],
100 + [
101 + "能打开 http://localhost:3000?见 http://localhost:3001;「www.example.com」",
102 + `能打开 ${link("http://localhost:3000")}?见 ${link("http://localhost:3001")};「${link("www.example.com")}」`,
103 + ],
104 + [
105 + "__http://localhost:18780__,*http://localhost:3000*,~~http://localhost:3001~~。",
106 + `**${link("http://localhost:18780")}**,*${link("http://localhost:3000")}*,~~${link("http://localhost:3001")}~~。`,
107 + ],
108 + [
109 + "foo**http://localhost:18780**(bar)***http://localhost:3000***。",
110 + `foo**${link("http://localhost:18780")}**(bar)***${link("http://localhost:3000")}***。`,
111 + ],
112 + [
113 + "http://a.com。http://b.com,www.c.com、person@example.com。",
114 + `${link("http://a.com")}。${link("http://b.com")},${link("www.c.com")}、[person@example.com](mailto:person@example.com)。`,
115 + ],
116 + [
117 + "**http://localhost:18780**(重启)。 https://example.com/a。结束",
118 + `**${link("http://localhost:18780")}**(重启)。 ${link("https://example.com/a")}。结束`,
119 + ],
120 + ["# **http://localhost:3000**,刷新", `**${link("http://localhost:3000")}**,刷新`],
121 + [
122 + "| a |\n| - |\n| **http://localhost:3000**,刷新 |",
123 + `a**${link("http://localhost:3000")}**,刷新`,
124 + ],
125 + ["www.example.com/preview。下一句", `${link("www.example.com/preview")}。下一句`],
126 + ["https://example.com/a_。下一句", `${link("https://example.com/a_")}。下一句`],
127 + ["https://example.com/a.,下一句", `${link("https://example.com/a")}.,下一句`],
128 + [
129 + "https://example.com/search?q=hello,world",
130 + `${link("https://example.com/search?q=hello")},world`,
131 + ],
132 + ["https://example.com/日本語,続き", link("https://example.com/日本語,続き")],
133 + [
134 + "https://zh.wikipedia.org/wiki/北京(城市)",
135 + link("https://zh.wikipedia.org/wiki/北京(城市)"),
136 + ],
137 + ["https://example.com/a%EF%BC%8Cb", link("https://example.com/a%EF%BC%8Cb")],
138 + ["<https://example.com/日本語,続き>", link("https://example.com/日本語,続き")],
139 + ["[記事](https://example.com/a,b)", "[記事](https://example.com/a,b)"],
140 + ["[http://example.com/a,b](http://example.com/a,b)", link("http://example.com/a,b")],
141 + ["`https://example.com/a。次`", "`https://example.com/a。次`"],
142 [
143 "https://x.com/a_ https://x.com/b_",
144 `${link("https://x.com/a_")} ${link("https://x.com/b_")}`,
@@ -120,6 +169,126 @@ it.each([
169 }
170 });
171
172 +it("ends bare URLs at CJK punctuation with source positions", () => {
173 + const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS);
174 + const parse = (markdown: string) => processor.runSync(processor.parse(markdown), markdown);
175 + const at = (line: number, column: number, offset: number) => ({ line, column, offset });
176 + const url = "http://localhost:18780";
177 + expect(parse(`第一行\n可以看了:**${url}**(已重启)。`)).toMatchObject({
178 + children: [
179 + {
180 + children: [
181 + { type: "text", value: "第一行\n可以看了:", position: { end: at(2, 6, 9) } },
182 + {
183 + type: "strong",
184 + position: { start: at(2, 6, 9), end: at(2, 32, 35) },
185 + children: [
186 + {
187 + type: "link",
188 + url,
189 + position: { start: at(2, 8, 11), end: at(2, 30, 33) },
190 + children: [{ type: "text", value: url, position: { end: at(2, 30, 33) } }],
191 + },
192 + ],
193 + },
194 + {
195 + type: "text",
196 + value: "(已重启)。",
197 + position: { start: at(2, 32, 35), end: at(2, 38, 41) },
198 + },
199 + ],
200 + },
201 + ],
202 + });
203 + // Markup the URL swallowed comes back once the URL ends.
204 + expect(parse(`打开 ${url},然后运行\`pnpm dev\``)).toMatchObject({
205 + children: [
206 + {
207 + children: [
208 + { type: "text", value: "打开 " },
209 + { type: "link", url },
210 + { type: "text", value: ",然后运行" },
211 + { type: "inlineCode", value: "pnpm dev" },
212 + ],
213 + },
214 + ],
215 + });
216 + // Every URL in one paragraph ends at its mark, with positions on its own line.
217 + const lines = Array.from({ length: 300 }, (_, index) => `接口:http://localhost:${index},`);
218 + const [paragraph] = (parse(lines.join("\n")) as Root).children;
219 + const links =
220 + paragraph?.type === "paragraph"
221 + ? paragraph.children.filter((node): node is Link => node.type === "link")
222 + : [];
223 + expect(links.map((node) => node.url)).toEqual(
224 + lines.map((_, index) => `http://localhost:${index}`),
225 + );
226 + const lastLine = lines.slice(0, -1).join("\n").length + 1;
227 + expect(links.at(-1)).toMatchObject({
228 + position: { start: at(300, 4, lastLine + 3), end: at(300, 24, lastLine + 23) },
229 + });
230 + // Only a table unescapes `\|` in code, so that cell keeps its parse and the URL is cut.
231 + const descendants = (node: Root | RootContent): RootContent[] =>
232 + "children" in node
233 + ? (node.children as RootContent[]).flatMap((child) => [child, ...descendants(child)])
234 + : [];
235 + const cell = descendants(parse("| a |\n| - |\n| http://example.com, `a\\|b` |") as Root);
236 + expect(cell.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
237 + "http://example.com",
238 + ]);
239 + expect(cell.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual(["a|b"]);
240 + // A task item's checkbox is not part of its paragraph's text.
241 + const task = descendants(parse("- [x] http://example.com,刷新") as Root);
242 + expect(task.flatMap((node) => (node.type === "text" ? [node.value] : []))).toEqual([
243 + "http://example.com",
244 + ",刷新",
245 + ]);
246 + // A list item's indentation stays out of a code span that crosses lines.
247 + const code = descendants(parse("- `echo\n hello` http://example.com,结束") as Root);
248 + expect(code.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
249 + "http://example.com",
250 + ]);
251 + expect(code.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual([
252 + "echo\nhello",
253 + ]);
254 + // A list item's continuation line parses the same on its own.
255 + expect(parse(`- 第一行\n **${url}**(已重启)`)).toMatchObject({
256 + children: [
257 + {
258 + children: [
259 + {
260 + children: [
261 + {
262 + children: [
263 + { type: "text", value: "第一行\n" },
264 + { type: "strong", children: [{ type: "link", url }] },
265 + { type: "text", value: "(已重启)" },
266 + ],
267 + },
268 + ],
269 + },
270 + ],
271 + },
272 + ],
273 + });
274 + // A quote's continuation line does not, so the URL ends and the asterisks stay as text.
275 + expect(parse(`> 第一行\n> **${url}**(已重启)`)).toMatchObject({
276 + children: [
277 + {
278 + children: [
279 + {
280 + children: [
281 + { type: "text", value: "第一行\n**" },
282 + { type: "link", url, position: { end: at(2, 27, 32) } },
283 + { type: "text", value: "**(已重启)", position: { start: at(2, 27, 32) } },
284 + ],
285 + },
286 + ],
287 + },
288 + ],
289 + });
290 +});
291 +
292 const insightRule = "─".repeat(20);
293 const insightFence = (body: string, wrap = "") =>
294 [`${wrap}★ Insight ${insightRule}${wrap}`, body, `${wrap}${insightRule}${wrap}`].join("\n");
packages/shared/src/markdownPipeline.ts
+204 -1
@@ -1,5 +1,14 @@
1 import { isLocalEditorFileUrl, LOCAL_FILE_LINK_SCHEMES } from "@t3tools/contracts";
2 -import type { Paragraph, PhrasingContent, Root, RootContent } from "mdast";
2 +import type {
3 + Heading,
4 + Link,
5 + Paragraph,
6 + PhrasingContent,
7 + Root,
8 + RootContent,
9 + TableCell,
10 + Text,
11 +} from "mdast";
12 import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
13 import { type PluggableList, type Processor, type Transformer, unified } from "unified";
14 import remarkParse from "remark-parse";
@@ -236,6 +245,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = {
245
246 export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
247 remarkGfm,
248 + remarkEndAutolinksAtCjkPunctuation,
249 remarkKeepAutolinkSuffixes,
250 remarkLinkAbsoluteFilePaths,
251 remarkKeepWindowsPaths,
@@ -250,6 +260,7 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
260
261 export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
262 remarkGfm,
263 + remarkEndAutolinksAtCjkPunctuation,
264 remarkKeepAutolinkSuffixes,
265 remarkLinkAbsoluteFilePaths,
266 remarkKeepWindowsPaths,
@@ -453,6 +464,198 @@ function remarkKeepAutolinkSuffixes(): Transformer<Root> {
464 };
465 }
466
467 +// Punctuation and symbols from CJK Symbols and Punctuation and from Halfwidth and Fullwidth Forms.
468 +const CJK_PUNCTUATION = /(?=[ -〿＀-￯])[\p{P}\p{S}]/u;
469 +
470 +/**
471 + * GFM runs a bare URL on until whitespace, so Chinese and Japanese punctuation joins it, along with
472 + * the rest of the sentence and any `**` that should have closed around it. A URL that is ASCII up to
473 + * such a mark ends there. Unicode earlier in the URL means the mark may belong to it, so it stays.
474 + */
475 +function cjkPunctuationIndex(link: Link): number {
476 + const label = link.children.length === 1 ? link.children[0] : undefined;
477 + if (label?.type !== "text" || !/^https?:\/\//i.test(link.url)) return -1;
478 + if (link.url !== label.value && link.url !== `http://${label.value}`) return -1;
479 + const index = label.value.search(/[^!-~]/u);
480 + return index > 0 && CJK_PUNCTUATION.test(label.value.charAt(index)) ? index : -1;
481 +}
482 +
483 +/** GFM's text pass links bare URLs without positions; its tokenizer's links span their URL. */
484 +function isBareLink(link: Link, source: string): boolean {
485 + const start = link.position?.start.offset;
486 + const end = link.position?.end.offset;
487 + const label = link.children[0];
488 + return (
489 + !link.position ||
490 + (start !== undefined &&
491 + end !== undefined &&
492 + label?.type === "text" &&
493 + source.slice(start, end) === label.value)
494 + );
495 +}
496 +
497 +/** Source offsets of the marks that tokenized bare URLs run past. */
498 +function cjkPunctuationOffsets(node: RootContent, source: string): number[] {
499 + if (node.type === "link") {
500 + const index = cjkPunctuationIndex(node);
501 + const start = node.position?.start.offset;
502 + return index > 0 && start !== undefined && isBareLink(node, source) ? [start + index] : [];
503 + }
504 + return "children" in node && node.type !== "linkReference"
505 + ? (node.children as RootContent[]).flatMap((child) => cjkPunctuationOffsets(child, source))
506 + : [];
507 +}
508 +
509 +const hasReference = (node: RootContent): boolean =>
510 + node.type === "linkReference" ||
511 + node.type === "imageReference" ||
512 + node.type === "footnoteReference" ||
513 + ("children" in node && (node.children as RootContent[]).some(hasReference));
514 +
515 +/** How many of the ascending numbers are below `value`. */
516 +function countBelow(sorted: readonly number[], value: number): number {
517 + let low = 0;
518 + let high = sorted.length;
519 + while (low < high) {
520 + const middle = (low + high) >> 1;
521 + if (sorted[middle]! < value) low = middle + 1;
522 + else high = middle;
523 + }
524 + return low;
525 +}
526 +
527 +/**
528 + * Parses the block's text again with a tab before each mark. The tab ends the URL as a space would,
529 + * so emphasis around the URL closes and later URLs and code get their own nodes. The tabs are then
530 + * dropped and positions mapped back to the source. Returns false and leaves the block alone when its
531 + * text would parse differently on its own, as a quote's continuation lines do.
532 + */
533 +function reparseAtCjkPunctuation(
534 + processor: Processor,
535 + block: Paragraph | Heading | TableCell,
536 + source: string,
537 + offsets: number[],
538 +): boolean {
539 + // A task item's paragraph starts at its checkbox; its first child starts after it.
540 + const from = block.children[0]?.position?.start;
541 + const to = block.children.at(-1)?.position?.end;
542 + if (from?.offset === undefined || to?.offset === undefined || hasReference(block)) return false;
543 + const base = from.offset;
544 + const text = source.slice(base, to.offset);
545 + // A tab is the marker, a quote's `>` on a later line would read as a new quote, a list's
546 + // indentation would join a code span, and only a table unescapes `\|` in code.
547 + if (
548 + text.includes("\t") ||
549 + /[\r\n][ ]*>/.test(text) ||
550 + (/[\r\n] /.test(text) && text.includes("`")) ||
551 + (block.type === "tableCell" && text.includes("\\|"))
552 + ) {
553 + return false;
554 + }
555 + const marks = offsets.map((offset) => offset - base);
556 + // Each pass finds the next URL in a run with no spaces, like `http://a,http://b,http://c`.
557 + for (let attempt = 0; attempt < 16; attempt++) {
558 + marks.sort((a, b) => a - b);
559 + const tabs = marks.map((mark, index) => mark + index);
560 + const unmarked = (offset: number) => offset - countBelow(tabs, offset);
561 + const marked = [0, ...marks]
562 + .map((start, index) => text.slice(start, marks[index] ?? text.length))
563 + .join("\t");
564 + const [paragraph, ...rest] = (processor.parse(marked) as Root).children;
565 + if (paragraph?.type !== "paragraph" || rest.length > 0) return false;
566 + const more = cjkPunctuationOffsets(paragraph, marked);
567 + if (more.length > 0) {
568 + marks.push(...more.map(unmarked));
569 + continue;
570 + }
571 +
572 + let tabCount = 0;
573 + const moved = new Set<object>();
574 + const restore = (node: RootContent): boolean => {
575 + for (const point of node.position ? [node.position.start, node.position.end] : []) {
576 + const offset = point.offset;
577 + if (offset === undefined || moved.has(point)) continue;
578 + moved.add(point);
579 + const tabsOnLine = countBelow(tabs, offset) - countBelow(tabs, offset - point.column + 1);
580 + point.column = (point.line === 1 ? from.column - 1 : 0) + point.column - tabsOnLine;
581 + point.line += from.line - 1;
582 + point.offset = base + unmarked(offset);
583 + }
584 + if (node.type === "text") {
585 + tabCount += node.value.split("\t").length - 1;
586 + node.value = node.value.replaceAll("\t", "");
587 + } else if (
588 + ("value" in node && node.value.includes("\t")) ||
589 + ("url" in node && node.url.includes("\t"))
590 + ) {
591 + return false;
592 + }
593 + if (!("children" in node)) return true;
594 + const children = node.children as RootContent[];
595 + if (!children.every(restore)) return false;
596 + node.children = children.filter(
597 + (child) => child.type !== "text" || child.value !== "",
598 + ) as typeof node.children;
599 + return true;
600 + };
601 + if (!restore(paragraph) || tabCount !== tabs.length) return false;
602 + block.children = paragraph.children;
603 + return true;
604 + }
605 + return false;
606 +}
607 +
608 +/** Ends the bare URLs a reparse could not, and those GFM's text pass linked after emphasis. */
609 +function splitAtCjkPunctuation(node: RootContent, source: string) {
610 + if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
611 + const children = node.children as RootContent[];
612 + for (let index = 0; index < children.length; index++) {
613 + const child = children[index]!;
614 + if (child.type !== "link") {
615 + splitAtCjkPunctuation(child, source);
616 + continue;
617 + }
618 + const at = isBareLink(child, source) ? cjkPunctuationIndex(child) : -1;
619 + const label = child.children[0];
620 + if (at < 0 || label?.type !== "text") continue;
621 + // GFM drops this punctuation from the end of a URL; remarkKeepAutolinkSuffixes restores `_` and `~`.
622 + const length = label.value.slice(0, at).replace(/[!"'*,.:;?_~]+$/, "").length;
623 + const tail: Text = { type: "text", value: label.value.slice(length) };
624 + const start = child.position?.start;
625 + if (child.position && start?.offset !== undefined) {
626 + const end = {
627 + line: start.line,
628 + column: start.column + length,
629 + offset: start.offset + length,
630 + };
631 + tail.position = { start: end, end: child.position.end };
632 + child.position = { start, end };
633 + label.position = { start, end };
634 + }
635 + child.url = child.url.slice(0, child.url.length - label.value.length + length);
636 + label.value = label.value.slice(0, length);
637 + children.splice(index + 1, 0, tail);
638 + index++;
639 + }
640 +}
641 +
642 +function remarkEndAutolinksAtCjkPunctuation(this: Processor): Transformer<Root> {
643 + return (tree, file) => {
644 + const source = String(file);
645 + if (!CJK_PUNCTUATION.test(source)) return;
646 + const visit = (node: RootContent) => {
647 + if (node.type === "paragraph" || node.type === "heading" || node.type === "tableCell") {
648 + const offsets = cjkPunctuationOffsets(node, source);
649 + if (offsets.length > 0) reparseAtCjkPunctuation(this, node, source, offsets);
650 + splitAtCjkPunctuation(node, source);
651 + } else if ("children" in node) {
652 + (node.children as RootContent[]).forEach(visit);
653 + }
654 + };
655 + tree.children.forEach(visit);
656 + };
657 +}
658 +
659 function remarkLinkAbsoluteFilePaths(): Transformer<Root> {
660 return (tree) => {
661 // Raw `<a>` tags are sibling html nodes, so text inside one is not under a link node.