The best way to control Claude Code, Codex and more from any computer or phone
fix(web): bare URLs end at Chinese and Japanese punctuation (#35267)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
sanjai balajee (e/acc) committed
Nov 12, 2026 at 22:51 UTC
4d641633b2877bad1e3f754ea88389eec293dc7b
2 files changed
+375
-3
packages/shared/src/markdownPipeline.test.ts
+171
-2
@@ -3,7 +3,7 @@ import { type PluggableList, unified } from "unified";
3
import rehypeRaw from "rehype-raw";
4
import remarkParse from "remark-parse";
5
import remarkRehype from "remark-rehype";
6
-import type { Root, RootContent } from "mdast";
6
+import type { Link, Root, RootContent } from "mdast";
7
import {
8
CHAT_MARKDOWN_REHYPE_PLUGINS,
9
CHAT_MARKDOWN_REMARK_PLUGINS,
@@ -15,7 +15,7 @@ import {
15
it.each([
16
["default", CHAT_MARKDOWN_REMARK_PLUGINS],
17
["with breaks", CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS],
18
-] as const)("preserves trailing underscores and tildes in bare URLs (%s)", (_, plugins) => {
18
+] as const)("preserves bare URL boundaries (%s)", (_, plugins) => {
19
const processor = unified().use(remarkParse).use(plugins);
20
const shape = (node: Root | RootContent): string => {
21
if (node.type === "text") return node.value;
@@ -90,6 +90,55 @@ it.each([
90
["https\\://x.com/dimasike_", link("https://x.com/dimasike_")],
91
["https://x.com/a\\_", link("https://x.com/a\\_")],
92
["https://x.com/a_", `${link("https://x.com/a_")};`],
93
+ [
94
+ "可以看了:**http://localhost:18780**(已重启)。",
95
+ `可以看了:**${link("http://localhost:18780")}**(已重启)。`,
96
+ ],
97
+ ["打开 http://localhost:18780,然后刷新", `打开 ${link("http://localhost:18780")},然后刷新`],
98
+ ["URL:https://example.com/a。下一句", `URL:${link("https://example.com/a")}。下一句`],
99
+ ["(预览:http://localhost:18780)", `(预览:${link("http://localhost:18780")})`],
100
+ [
101
+ "能打开 http://localhost:3000?见 http://localhost:3001;「www.example.com」",
102
+ `能打开 ${link("http://localhost:3000")}?见 ${link("http://localhost:3001")};「${link("www.example.com")}」`,
103
+ ],
104
+ [
105
+ "__http://localhost:18780__,*http://localhost:3000*,~~http://localhost:3001~~。",
106
+ `**${link("http://localhost:18780")}**,*${link("http://localhost:3000")}*,~~${link("http://localhost:3001")}~~。`,
107
+ ],
108
+ [
109
+ "foo**http://localhost:18780**(bar)***http://localhost:3000***。",
110
+ `foo**${link("http://localhost:18780")}**(bar)***${link("http://localhost:3000")}***。`,
111
+ ],
112
+ [
113
+ "http://a.com。http://b.com,www.c.com、person@example.com。",
114
+ `${link("http://a.com")}。${link("http://b.com")},${link("www.c.com")}、[person@example.com](mailto:person@example.com)。`,
115
+ ],
116
+ [
117
+ "**http://localhost:18780**(重启)。 https://example.com/a。结束",
118
+ `**${link("http://localhost:18780")}**(重启)。 ${link("https://example.com/a")}。结束`,
119
+ ],
120
+ ["# **http://localhost:3000**,刷新", `**${link("http://localhost:3000")}**,刷新`],
121
+ [
122
+ "| a |\n| - |\n| **http://localhost:3000**,刷新 |",
123
+ `a**${link("http://localhost:3000")}**,刷新`,
124
+ ],
125
+ ["www.example.com/preview。下一句", `${link("www.example.com/preview")}。下一句`],
126
+ ["https://example.com/a_。下一句", `${link("https://example.com/a_")}。下一句`],
127
+ ["https://example.com/a.,下一句", `${link("https://example.com/a")}.,下一句`],
128
+ [
129
+ "https://example.com/search?q=hello,world",
130
+ `${link("https://example.com/search?q=hello")},world`,
131
+ ],
132
+ ["https://example.com/日本語,続き", link("https://example.com/日本語,続き")],
133
+ [
134
+ "https://zh.wikipedia.org/wiki/北京(城市)",
135
+ link("https://zh.wikipedia.org/wiki/北京(城市)"),
136
+ ],
137
+ ["https://example.com/a%EF%BC%8Cb", link("https://example.com/a%EF%BC%8Cb")],
138
+ ["<https://example.com/日本語,続き>", link("https://example.com/日本語,続き")],
139
+ ["[記事](https://example.com/a,b)", "[記事](https://example.com/a,b)"],
140
+ ["[http://example.com/a,b](http://example.com/a,b)", link("http://example.com/a,b")],
141
+ ["`https://example.com/a。次`", "`https://example.com/a。次`"],
142
[
143
"https://x.com/a_ https://x.com/b_",
144
`${link("https://x.com/a_")} ${link("https://x.com/b_")}`,
@@ -120,6 +169,126 @@ it.each([
169
}
170
});
171
172
+it("ends bare URLs at CJK punctuation with source positions", () => {
173
+ const processor = unified().use(remarkParse).use(CHAT_MARKDOWN_REMARK_PLUGINS);
174
+ const parse = (markdown: string) => processor.runSync(processor.parse(markdown), markdown);
175
+ const at = (line: number, column: number, offset: number) => ({ line, column, offset });
176
+ const url = "http://localhost:18780";
177
+ expect(parse(`第一行\n可以看了:**${url}**(已重启)。`)).toMatchObject({
178
+ children: [
179
+ {
180
+ children: [
181
+ { type: "text", value: "第一行\n可以看了:", position: { end: at(2, 6, 9) } },
182
+ {
183
+ type: "strong",
184
+ position: { start: at(2, 6, 9), end: at(2, 32, 35) },
185
+ children: [
186
+ {
187
+ type: "link",
188
+ url,
189
+ position: { start: at(2, 8, 11), end: at(2, 30, 33) },
190
+ children: [{ type: "text", value: url, position: { end: at(2, 30, 33) } }],
191
+ },
192
+ ],
193
+ },
194
+ {
195
+ type: "text",
196
+ value: "(已重启)。",
197
+ position: { start: at(2, 32, 35), end: at(2, 38, 41) },
198
+ },
199
+ ],
200
+ },
201
+ ],
202
+ });
203
+ // Markup the URL swallowed comes back once the URL ends.
204
+ expect(parse(`打开 ${url},然后运行\`pnpm dev\``)).toMatchObject({
205
+ children: [
206
+ {
207
+ children: [
208
+ { type: "text", value: "打开 " },
209
+ { type: "link", url },
210
+ { type: "text", value: ",然后运行" },
211
+ { type: "inlineCode", value: "pnpm dev" },
212
+ ],
213
+ },
214
+ ],
215
+ });
216
+ // Every URL in one paragraph ends at its mark, with positions on its own line.
217
+ const lines = Array.from({ length: 300 }, (_, index) => `接口:http://localhost:${index},`);
218
+ const [paragraph] = (parse(lines.join("\n")) as Root).children;
219
+ const links =
220
+ paragraph?.type === "paragraph"
221
+ ? paragraph.children.filter((node): node is Link => node.type === "link")
222
+ : [];
223
+ expect(links.map((node) => node.url)).toEqual(
224
+ lines.map((_, index) => `http://localhost:${index}`),
225
+ );
226
+ const lastLine = lines.slice(0, -1).join("\n").length + 1;
227
+ expect(links.at(-1)).toMatchObject({
228
+ position: { start: at(300, 4, lastLine + 3), end: at(300, 24, lastLine + 23) },
229
+ });
230
+ // Only a table unescapes `\|` in code, so that cell keeps its parse and the URL is cut.
231
+ const descendants = (node: Root | RootContent): RootContent[] =>
232
+ "children" in node
233
+ ? (node.children as RootContent[]).flatMap((child) => [child, ...descendants(child)])
234
+ : [];
235
+ const cell = descendants(parse("| a |\n| - |\n| http://example.com, `a\\|b` |") as Root);
236
+ expect(cell.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
237
+ "http://example.com",
238
+ ]);
239
+ expect(cell.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual(["a|b"]);
240
+ // A task item's checkbox is not part of its paragraph's text.
241
+ const task = descendants(parse("- [x] http://example.com,刷新") as Root);
242
+ expect(task.flatMap((node) => (node.type === "text" ? [node.value] : []))).toEqual([
243
+ "http://example.com",
244
+ ",刷新",
245
+ ]);
246
+ // A list item's indentation stays out of a code span that crosses lines.
247
+ const code = descendants(parse("- `echo\n hello` http://example.com,结束") as Root);
248
+ expect(code.flatMap((node) => (node.type === "link" ? [node.url] : []))).toEqual([
249
+ "http://example.com",
250
+ ]);
251
+ expect(code.flatMap((node) => (node.type === "inlineCode" ? [node.value] : []))).toEqual([
252
+ "echo\nhello",
253
+ ]);
254
+ // A list item's continuation line parses the same on its own.
255
+ expect(parse(`- 第一行\n **${url}**(已重启)`)).toMatchObject({
256
+ children: [
257
+ {
258
+ children: [
259
+ {
260
+ children: [
261
+ {
262
+ children: [
263
+ { type: "text", value: "第一行\n" },
264
+ { type: "strong", children: [{ type: "link", url }] },
265
+ { type: "text", value: "(已重启)" },
266
+ ],
267
+ },
268
+ ],
269
+ },
270
+ ],
271
+ },
272
+ ],
273
+ });
274
+ // A quote's continuation line does not, so the URL ends and the asterisks stay as text.
275
+ expect(parse(`> 第一行\n> **${url}**(已重启)`)).toMatchObject({
276
+ children: [
277
+ {
278
+ children: [
279
+ {
280
+ children: [
281
+ { type: "text", value: "第一行\n**" },
282
+ { type: "link", url, position: { end: at(2, 27, 32) } },
283
+ { type: "text", value: "**(已重启)", position: { start: at(2, 27, 32) } },
284
+ ],
285
+ },
286
+ ],
287
+ },
288
+ ],
289
+ });
290
+});
291
+
292
const insightRule = "─".repeat(20);
293
const insightFence = (body: string, wrap = "") =>
294
[`${wrap}★ Insight ${insightRule}${wrap}`, body, `${wrap}${insightRule}${wrap}`].join("\n");
packages/shared/src/markdownPipeline.ts
+204
-1
@@ -1,5 +1,14 @@
1
import { isLocalEditorFileUrl, LOCAL_FILE_LINK_SCHEMES } from "@t3tools/contracts";
2
-import type { Paragraph, PhrasingContent, Root, RootContent } from "mdast";
2
+import type {
3
+ Heading,
4
+ Link,
5
+ Paragraph,
6
+ PhrasingContent,
7
+ Root,
8
+ RootContent,
9
+ TableCell,
10
+ Text,
11
+} from "mdast";
12
import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
13
import { type PluggableList, type Processor, type Transformer, unified } from "unified";
14
import remarkParse from "remark-parse";
@@ -236,6 +245,7 @@ const CHAT_MARKDOWN_SANITIZE_SCHEMA = {
245
246
export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
247
remarkGfm,
248
+ remarkEndAutolinksAtCjkPunctuation,
249
remarkKeepAutolinkSuffixes,
250
remarkLinkAbsoluteFilePaths,
251
remarkKeepWindowsPaths,
@@ -250,6 +260,7 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
260
261
export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
262
remarkGfm,
263
+ remarkEndAutolinksAtCjkPunctuation,
264
remarkKeepAutolinkSuffixes,
265
remarkLinkAbsoluteFilePaths,
266
remarkKeepWindowsPaths,
@@ -453,6 +464,198 @@ function remarkKeepAutolinkSuffixes(): Transformer<Root> {
464
};
465
}
466
467
+// Punctuation and symbols from CJK Symbols and Punctuation and from Halfwidth and Fullwidth Forms.
468
+const CJK_PUNCTUATION = /(?=[ -〿-])[\p{P}\p{S}]/u;
469
+
470
+/**
471
+ * GFM runs a bare URL on until whitespace, so Chinese and Japanese punctuation joins it, along with
472
+ * the rest of the sentence and any `**` that should have closed around it. A URL that is ASCII up to
473
+ * such a mark ends there. Unicode earlier in the URL means the mark may belong to it, so it stays.
474
+ */
475
+function cjkPunctuationIndex(link: Link): number {
476
+ const label = link.children.length === 1 ? link.children[0] : undefined;
477
+ if (label?.type !== "text" || !/^https?:\/\//i.test(link.url)) return -1;
478
+ if (link.url !== label.value && link.url !== `http://${label.value}`) return -1;
479
+ const index = label.value.search(/[^!-~]/u);
480
+ return index > 0 && CJK_PUNCTUATION.test(label.value.charAt(index)) ? index : -1;
481
+}
482
+
483
+/** GFM's text pass links bare URLs without positions; its tokenizer's links span their URL. */
484
+function isBareLink(link: Link, source: string): boolean {
485
+ const start = link.position?.start.offset;
486
+ const end = link.position?.end.offset;
487
+ const label = link.children[0];
488
+ return (
489
+ !link.position ||
490
+ (start !== undefined &&
491
+ end !== undefined &&
492
+ label?.type === "text" &&
493
+ source.slice(start, end) === label.value)
494
+ );
495
+}
496
+
497
+/** Source offsets of the marks that tokenized bare URLs run past. */
498
+function cjkPunctuationOffsets(node: RootContent, source: string): number[] {
499
+ if (node.type === "link") {
500
+ const index = cjkPunctuationIndex(node);
501
+ const start = node.position?.start.offset;
502
+ return index > 0 && start !== undefined && isBareLink(node, source) ? [start + index] : [];
503
+ }
504
+ return "children" in node && node.type !== "linkReference"
505
+ ? (node.children as RootContent[]).flatMap((child) => cjkPunctuationOffsets(child, source))
506
+ : [];
507
+}
508
+
509
+const hasReference = (node: RootContent): boolean =>
510
+ node.type === "linkReference" ||
511
+ node.type === "imageReference" ||
512
+ node.type === "footnoteReference" ||
513
+ ("children" in node && (node.children as RootContent[]).some(hasReference));
514
+
515
+/** How many of the ascending numbers are below `value`. */
516
+function countBelow(sorted: readonly number[], value: number): number {
517
+ let low = 0;
518
+ let high = sorted.length;
519
+ while (low < high) {
520
+ const middle = (low + high) >> 1;
521
+ if (sorted[middle]! < value) low = middle + 1;
522
+ else high = middle;
523
+ }
524
+ return low;
525
+}
526
+
527
+/**
528
+ * Parses the block's text again with a tab before each mark. The tab ends the URL as a space would,
529
+ * so emphasis around the URL closes and later URLs and code get their own nodes. The tabs are then
530
+ * dropped and positions mapped back to the source. Returns false and leaves the block alone when its
531
+ * text would parse differently on its own, as a quote's continuation lines do.
532
+ */
533
+function reparseAtCjkPunctuation(
534
+ processor: Processor,
535
+ block: Paragraph | Heading | TableCell,
536
+ source: string,
537
+ offsets: number[],
538
+): boolean {
539
+ // A task item's paragraph starts at its checkbox; its first child starts after it.
540
+ const from = block.children[0]?.position?.start;
541
+ const to = block.children.at(-1)?.position?.end;
542
+ if (from?.offset === undefined || to?.offset === undefined || hasReference(block)) return false;
543
+ const base = from.offset;
544
+ const text = source.slice(base, to.offset);
545
+ // A tab is the marker, a quote's `>` on a later line would read as a new quote, a list's
546
+ // indentation would join a code span, and only a table unescapes `\|` in code.
547
+ if (
548
+ text.includes("\t") ||
549
+ /[\r\n][ ]*>/.test(text) ||
550
+ (/[\r\n] /.test(text) && text.includes("`")) ||
551
+ (block.type === "tableCell" && text.includes("\\|"))
552
+ ) {
553
+ return false;
554
+ }
555
+ const marks = offsets.map((offset) => offset - base);
556
+ // Each pass finds the next URL in a run with no spaces, like `http://a,http://b,http://c`.
557
+ for (let attempt = 0; attempt < 16; attempt++) {
558
+ marks.sort((a, b) => a - b);
559
+ const tabs = marks.map((mark, index) => mark + index);
560
+ const unmarked = (offset: number) => offset - countBelow(tabs, offset);
561
+ const marked = [0, ...marks]
562
+ .map((start, index) => text.slice(start, marks[index] ?? text.length))
563
+ .join("\t");
564
+ const [paragraph, ...rest] = (processor.parse(marked) as Root).children;
565
+ if (paragraph?.type !== "paragraph" || rest.length > 0) return false;
566
+ const more = cjkPunctuationOffsets(paragraph, marked);
567
+ if (more.length > 0) {
568
+ marks.push(...more.map(unmarked));
569
+ continue;
570
+ }
571
+
572
+ let tabCount = 0;
573
+ const moved = new Set<object>();
574
+ const restore = (node: RootContent): boolean => {
575
+ for (const point of node.position ? [node.position.start, node.position.end] : []) {
576
+ const offset = point.offset;
577
+ if (offset === undefined || moved.has(point)) continue;
578
+ moved.add(point);
579
+ const tabsOnLine = countBelow(tabs, offset) - countBelow(tabs, offset - point.column + 1);
580
+ point.column = (point.line === 1 ? from.column - 1 : 0) + point.column - tabsOnLine;
581
+ point.line += from.line - 1;
582
+ point.offset = base + unmarked(offset);
583
+ }
584
+ if (node.type === "text") {
585
+ tabCount += node.value.split("\t").length - 1;
586
+ node.value = node.value.replaceAll("\t", "");
587
+ } else if (
588
+ ("value" in node && node.value.includes("\t")) ||
589
+ ("url" in node && node.url.includes("\t"))
590
+ ) {
591
+ return false;
592
+ }
593
+ if (!("children" in node)) return true;
594
+ const children = node.children as RootContent[];
595
+ if (!children.every(restore)) return false;
596
+ node.children = children.filter(
597
+ (child) => child.type !== "text" || child.value !== "",
598
+ ) as typeof node.children;
599
+ return true;
600
+ };
601
+ if (!restore(paragraph) || tabCount !== tabs.length) return false;
602
+ block.children = paragraph.children;
603
+ return true;
604
+ }
605
+ return false;
606
+}
607
+
608
+/** Ends the bare URLs a reparse could not, and those GFM's text pass linked after emphasis. */
609
+function splitAtCjkPunctuation(node: RootContent, source: string) {
610
+ if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
611
+ const children = node.children as RootContent[];
612
+ for (let index = 0; index < children.length; index++) {
613
+ const child = children[index]!;
614
+ if (child.type !== "link") {
615
+ splitAtCjkPunctuation(child, source);
616
+ continue;
617
+ }
618
+ const at = isBareLink(child, source) ? cjkPunctuationIndex(child) : -1;
619
+ const label = child.children[0];
620
+ if (at < 0 || label?.type !== "text") continue;
621
+ // GFM drops this punctuation from the end of a URL; remarkKeepAutolinkSuffixes restores `_` and `~`.
622
+ const length = label.value.slice(0, at).replace(/[!"'*,.:;?_~]+$/, "").length;
623
+ const tail: Text = { type: "text", value: label.value.slice(length) };
624
+ const start = child.position?.start;
625
+ if (child.position && start?.offset !== undefined) {
626
+ const end = {
627
+ line: start.line,
628
+ column: start.column + length,
629
+ offset: start.offset + length,
630
+ };
631
+ tail.position = { start: end, end: child.position.end };
632
+ child.position = { start, end };
633
+ label.position = { start, end };
634
+ }
635
+ child.url = child.url.slice(0, child.url.length - label.value.length + length);
636
+ label.value = label.value.slice(0, length);
637
+ children.splice(index + 1, 0, tail);
638
+ index++;
639
+ }
640
+}
641
+
642
+function remarkEndAutolinksAtCjkPunctuation(this: Processor): Transformer<Root> {
643
+ return (tree, file) => {
644
+ const source = String(file);
645
+ if (!CJK_PUNCTUATION.test(source)) return;
646
+ const visit = (node: RootContent) => {
647
+ if (node.type === "paragraph" || node.type === "heading" || node.type === "tableCell") {
648
+ const offsets = cjkPunctuationOffsets(node, source);
649
+ if (offsets.length > 0) reparseAtCjkPunctuation(this, node, source, offsets);
650
+ splitAtCjkPunctuation(node, source);
651
+ } else if ("children" in node) {
652
+ (node.children as RootContent[]).forEach(visit);
653
+ }
654
+ };
655
+ tree.children.forEach(visit);
656
+ };
657
+}
658
+
659
function remarkLinkAbsoluteFilePaths(): Transformer<Root> {
660
return (tree) => {
661
// Raw `<a>` tags are sibling html nodes, so text inside one is not under a link node.