@devries-cameron20 / t3code Mirror · read-only / commits / 0b3ebcf268
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 08:30 UTC

fix(web): sent messages keep the spaces typed at the start of a line (#38079)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

cosier committed Nov 17, 2026 at 22:20 UTC 0b3ebcf268e65a84d797939a2b8c334db973fc8a
11 files changed +773 -2
apps/web/src/components/ChatMarkdown.browser.test.tsx
+43 -1
@@ -2,6 +2,7 @@ import { MISSING_FILE_ACCESS_MESSAGE } from "@t3tools/client-runtime/state/asset
2 import { EnvironmentId, ThreadId } from "@t3tools/contracts";
3 import { AsyncResult } from "effect/reactivity";
4 import { type ReactElement } from "react";
5 +import remarkGfm from "remark-gfm";
6 import { describe, expect, it, vi } from "vite-plus/test";
7 import { page, userEvent } from "vite-plus/test/browser";
8
@@ -10,7 +11,10 @@ import "../index.css";
11
12 import { filesystemEnvironment } from "../state/filesystem";
13 import { useRightPanelStore } from "../rightPanelStore";
13 -import { remarkEnableOrderedListNumbers } from "@t3tools/shared/markdownPipeline";
14 +import {
15 + CHAT_USER_MARKDOWN_REMARK_PLUGINS,
16 + remarkEnableOrderedListNumbers,
17 +} from "@t3tools/shared/markdownPipeline";
18 import { setMarkdownTaskChecked } from "@t3tools/shared/markdownTaskList";
19 import { HostImageUrlContext } from "./chat/hostImageUrl";
20 import { getSyntaxHighlighterPromise } from "~/lib/syntaxHighlighting";
@@ -703,6 +707,44 @@ describe("ChatMarkdown copy", () => {
707 expect(select(labelText, bodyText, "Back up".length)).toBe("Warning\n\nBack up");
708 });
709
710 + it.each([
711 + { text: " Plan\nnext" },
712 + { text: "Plan\n Deep\n Deeper" },
713 + { text: "Plan\n\tTabbed\n \t Mixed" },
714 + { text: "- item\n more\n deeper\n- next" },
715 + { text: "Plan\n Step one\n Detail a", from: "Step one" },
716 + ])(
717 + "copies and selects user lines the same whether their typed indentation shows: $text",
718 + async (copy) => {
719 + setup();
720 + const copied = async (extraRemarkPlugins: typeof CHAT_USER_MARKDOWN_REMARK_PLUGINS) => {
721 + await using screen = await renderText(copy.text, {
722 + className: "user-message-markdown",
723 + lineBreaks: true,
724 + parseRawHtml: false,
725 + extraRemarkPlugins,
726 + });
727 + const range = document.createRange();
728 + range.selectNodeContents(screen.container);
729 + if (copy.from) {
730 + const walker = document.createTreeWalker(screen.container, NodeFilter.SHOW_TEXT);
731 + let line = walker.nextNode();
732 + while (line && !line.textContent!.includes(copy.from)) line = walker.nextNode();
733 + // Start mid-line, three characters into the line's text.
734 + range.setStart(line!, line!.textContent!.indexOf(copy.from) + 3);
735 + }
736 + const selection = window.getSelection()!;
737 + selection.removeAllRanges();
738 + selection.addRange(range);
739 + // A drag or anything else that reads the selection gets its text, not the copy payload.
740 + return { payload: chatMarkdownClipboardPayload(selection), selected: selection.toString() };
741 + };
742 + const shown = await copied(CHAT_USER_MARKDOWN_REMARK_PLUGINS);
743 + // The user pipeline as it was before typed indentation was shown.
744 + expect(shown).toEqual(await copied([[remarkGfm, { singleTilde: false }]]));
745 + },
746 + );
747 +
748 it("copies part of inline code without prose escapes", async () => {
749 setup();
750 await using screen = await renderText("Read `items[0]*_x` and a*b*c.");
apps/web/src/components/chat/MessagesTimeline.browser.test.tsx
+26
@@ -49,6 +49,8 @@ import { ViewedImagesTimelineRow } from "./TimelineWorkEntryRow";
49 import { remarkThoughtPreview, THOUGHT_PREVIEW_MAX_LENGTH } from "./TimelineWorkEntryRow.logic";
50 import { TimelineRowCtx } from "./timelineRowContext";
51 import { parseMarkdownWithMath } from "@t3tools/shared/markdownMath";
52 +import { searchableMessageSegments } from "@t3tools/shared/threadFindText";
53 +import { countThreadSearchOccurrences } from "@t3tools/shared/threadSearch";
54 import { collectThreadFindRanges } from "./threadFindHighlights";
55 import { toastManager } from "../ui/toast";
56 import type { CitedHistory } from "./citationProcedure";
@@ -865,6 +867,30 @@ describe("MessagesTimeline messages", () => {
867 expect((globalThis as { __t3Xss?: number }).__t3Xss).toBeUndefined();
868 });
869
870 + it("shows the spaces typed at the start of user lines, where find also matches them", async () => {
871 + const source = "Plan\n Step one\n\tDetail";
872 + await using screen = await renderTimeline([
873 + message(source),
874 + message(source, { role: "assistant" }, "assistant-1"),
875 + ]);
876 + const [user, assistant] = screen.container.querySelectorAll<HTMLElement>(
877 + "[data-timeline-row-kind=message]",
878 + );
879 + expect(user?.querySelector<HTMLElement>("[data-user-message-body]")?.innerText).toBe(source);
880 + expect(assistant?.querySelector<HTMLElement>(".chat-markdown")?.innerText).toBe(
881 + "Plan Step one Detail",
882 + );
883 + const query = " Step one";
884 + const indexed = searchableMessageSegments({ role: "user", text: source, streaming: false });
885 + const ranges = collectThreadFindRanges(screen.container, query);
886 + expect(ranges.map(({ rowId, range }) => [rowId, range.toString()])).toEqual([
887 + [user?.closest("[data-timeline-row-id]")?.getAttribute("data-timeline-row-id"), query],
888 + ]);
889 + expect(
890 + indexed?.reduce((count, segment) => count + countThreadSearchOccurrences(segment, query), 0),
891 + ).toBe(ranges.length);
892 + });
893 +
894 it("keeps single tildes in user paths literal without changing double tildes or assistant text", async () => {
895 const source =
896 "Install the skill as ~/folder-a/skills/x, then follow the guide (~/folder-a/guide.md).";
apps/web/src/index.css
+8
@@ -2015,6 +2015,14 @@ code {
2015 list-style-type: decimal;
2016 }
2017
2018 +/* Spaces a user typed at the start of a line, which Markdown would otherwise drop. pre-wrap, not
2019 + pre, so an indent wider than the bubble wraps instead of pushing the line's text out of view.
2020 + Not selectable, so a selection, a drag or a copy carries the text as it did before. */
2021 +.chat-markdown.user-message-markdown [data-leading-whitespace] {
2022 + white-space: pre-wrap;
2023 + user-select: none;
2024 +}
2025 +
2026 .chat-markdown li + li {
2027 margin-top: 0.25rem;
2028 }
apps/web/src/markdown-clipboard.ts
+4
@@ -545,6 +545,10 @@ export function chatMarkdownClipboardPayload(
545 const container = document.createElement("div");
546 container.appendChild(range.cloneContents());
547 keepStartListNumbers(range, container);
548 + // The indentation typed at a line start only shows; pasted, it would turn lines into code.
549 + for (const indentation of container.querySelectorAll("[data-leading-whitespace]")) {
550 + indentation.remove();
551 + }
552 const ancestor = range.commonAncestorContainer;
553 const ancestorElement =
554 ancestor.nodeType === Node.ELEMENT_NODE ? (ancestor as Element) : ancestor.parentElement;
packages/shared/src/markdownGithubAlerts.ts
+1 -1
@@ -7,7 +7,7 @@ interface MarkdownAstNode {
7 children?: MarkdownAstNode[];
8 }
9
10 -const GITHUB_ALERT_MARKER = /^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\](?:\r?\n|$)/i;
10 +export const GITHUB_ALERT_MARKER = /^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\](?:\r?\n|$)/i;
11
12 function readGithubAlert(node: MarkdownAstNode): void {
13 if (node.type !== "blockquote") return;
packages/shared/src/markdownLeadingWhitespace.ts new
+381
@@ -0,0 +1,381 @@
1 +import type { Root } from "mdast";
2 +import type { Transformer } from "unified";
3 +
4 +import { GITHUB_ALERT_MARKER } from "./markdownGithubAlerts.ts";
5 +
6 +interface MarkdownPoint {
7 + readonly line: number;
8 + readonly column: number;
9 + readonly offset?: number | undefined;
10 +}
11 +
12 +interface MarkdownNode {
13 + readonly type: string;
14 + value?: string;
15 + position?: { readonly start: MarkdownPoint; readonly end: MarkdownPoint } | undefined;
16 + children?: MarkdownNode[];
17 + data?: Record<string, unknown> | undefined;
18 +}
19 +
20 +type Container =
21 + | { readonly kind: "quote" }
22 + | { readonly kind: "item"; readonly node: MarkdownNode; indent?: number | null }
23 + // Recovered list content starts past the item's indentation, so its lines measure from there.
24 + | { readonly kind: "indent"; readonly indent: number };
25 +
26 +interface Cursor {
27 + offset: number;
28 + column: number;
29 + /** The character at `offset` is a tab whose first columns are already taken. */
30 + insideTab: boolean;
31 +}
32 +
33 +interface Source {
34 + readonly text: string;
35 + readonly lineStarts: readonly number[];
36 +}
37 +
38 +const LINE_ENDING = /\r\n|\r|\n/g;
39 +// Whitespace at a line start, or past a quote marker and its one optional space.
40 +const MAY_KEEP_WHITESPACE = /(?:^|[\r\n])[ \t]|>(?:\t|[ \t]{2})/;
41 +const LIST_MARKER = /[-*+]|\d{1,9}[.)]/y;
42 +const INLINE_CONTAINERS = new Set(["emphasis", "strong", "delete", "link", "linkReference"]);
43 +const nextTabStop = (column: number) => (Math.floor(column / 4) + 1) * 4;
44 +const isBlank = (character: string | undefined) => character === " " || character === "\t";
45 +
46 +function columnAt(source: Source, line: number, offset: number): number {
47 + let column = 0;
48 + for (let index = source.lineStarts[line - 1]!; index < offset; index++) {
49 + column = source.text[index] === "\t" ? nextTabStop(column) : column + 1;
50 + }
51 + return column;
52 +}
53 +
54 +/** Moves `width` columns through spaces and tabs, splitting a tab where it must. */
55 +function skipColumns(source: Source, cursor: Cursor, width: number): boolean {
56 + let left = width;
57 + while (left > 0) {
58 + const character = source.text[cursor.offset];
59 + if (!isBlank(character)) return false;
60 + const step = character === " " ? 1 : nextTabStop(cursor.column) - cursor.column;
61 + if (step > left) {
62 + cursor.column += left;
63 + cursor.insideTab = true;
64 + return true;
65 + }
66 + cursor.offset += 1;
67 + cursor.column += step;
68 + cursor.insideTab = false;
69 + left -= step;
70 + }
71 + return true;
72 +}
73 +
74 +/**
75 + * Where the item's content starts on its own first line, and its column, measured from the marker
76 + * as CommonMark does. A task's checkbox is content, whatever GFM makes the paragraph's position.
77 + */
78 +function itemContent(source: Source, item: MarkdownNode): Cursor | undefined {
79 + const start = item.position?.start;
80 + if (start?.offset === undefined) return undefined;
81 + LIST_MARKER.lastIndex = start.offset;
82 + const marker = LIST_MARKER.exec(source.text)?.[0];
83 + if (!marker) return undefined;
84 + const end = start.offset + marker.length;
85 + const cursor: Cursor = {
86 + offset: end,
87 + column: columnAt(source, start.line, end),
88 + insideTab: false,
89 + };
90 + let offset = end;
91 + let column = cursor.column;
92 + while (isBlank(source.text[offset])) {
93 + column = source.text[offset] === "\t" ? nextTabStop(column) : column + 1;
94 + offset += 1;
95 + }
96 + const blankLine = offset >= source.text.length || /[\r\n]/.test(source.text[offset]!);
97 + if (!blankLine && column - cursor.column <= 4) return { offset, column, insideTab: false };
98 + // A blank first line, or code after five or more columns, starts one column past the marker.
99 + if (blankLine) return { ...cursor, column: cursor.column + 1 };
100 + skipColumns(source, cursor, 1);
101 + return cursor;
102 +}
103 +
104 +/** The end of the container markers and indentation on `line`, or nothing for a lazy line. */
105 +function containerEnd(
106 + source: Source,
107 + line: number,
108 + chain: readonly Container[],
109 + depth = chain.length,
110 +): Cursor | undefined {
111 + const cursor: Cursor = { offset: source.lineStarts[line - 1]!, column: 0, insideTab: false };
112 + for (let index = 0; index < depth; index++) {
113 + const container = chain[index]!;
114 + if (container.kind === "quote") {
115 + // Up to three columns of indentation, the marker, then one optional space.
116 + const indentation = { ...cursor };
117 + while (isBlank(source.text[cursor.offset]) && skipColumns(source, cursor, 1)) {
118 + if (cursor.column - indentation.column > 3) return undefined;
119 + }
120 + if (cursor.insideTab || source.text[cursor.offset] !== ">") return undefined;
121 + cursor.offset += 1;
122 + cursor.column += 1;
123 + if (isBlank(source.text[cursor.offset])) skipColumns(source, cursor, 1);
124 + continue;
125 + }
126 + if (container.kind === "item" && container.node.position?.start.line === line) {
127 + const content = itemContent(source, container.node);
128 + if (!content) return undefined;
129 + Object.assign(cursor, content);
130 + continue;
131 + }
132 + const indent =
133 + container.kind === "indent" ? container.indent : itemIndent(source, chain, index);
134 + if (indent === undefined || !skipColumns(source, cursor, indent)) return undefined;
135 + }
136 + return cursor;
137 +}
138 +
139 +/** The columns an item's continuation lines need past its parent's own markers. */
140 +function itemIndent(
141 + source: Source,
142 + chain: readonly Container[],
143 + index: number,
144 +): number | undefined {
145 + const item = chain[index];
146 + if (item?.kind !== "item") return undefined;
147 + if (item.indent === undefined) {
148 + const line = item.node.position?.start.line;
149 + const parent = line === undefined ? undefined : containerEnd(source, line, chain, index);
150 + const content = itemContent(source, item.node);
151 + item.indent = parent && content ? content.column - parent.column : null;
152 + }
153 + return item.indent ?? undefined;
154 +}
155 +
156 +/** The spaces and tabs typed on `line` that CommonMark drops, past what is structure. */
157 +function typedWhitespace(
158 + source: Source,
159 + line: number,
160 + paragraph: MarkdownNode,
161 + chain: readonly Container[],
162 +): string {
163 + const cursor = containerEnd(source, line, chain);
164 + if (!cursor) return "";
165 + const start = paragraph.position!.start;
166 + let end = cursor.offset;
167 + if (line === start.line) end = start.offset!;
168 + else while (isBlank(source.text[end])) end++;
169 + const typed = source.text.slice(cursor.offset, end);
170 + if (end <= cursor.offset || !/^[ \t]+$/.test(typed)) return "";
171 + // A tab split by the container keeps the columns left of it, as spaces.
172 + return cursor.insideTab
173 + ? " ".repeat(nextTabStop(cursor.column) - cursor.column) + typed.slice(1)
174 + : typed;
175 +}
176 +
177 +function whitespaceNode(value: string): MarkdownNode {
178 + return {
179 + type: "text",
180 + value,
181 + // Converting to HTML trims a whitespace-only text, and the first text after a line break;
182 + // the empty text takes that trim.
183 + data: {
184 + hName: "span",
185 + hProperties: { dataLeadingWhitespace: "" },
186 + hChildren: [
187 + { type: "text", value: "" },
188 + { type: "text", value },
189 + ],
190 + },
191 + };
192 +}
193 +
194 +/** Whether `node` starts with a space, tab or line ending, which a line start drops or hides. */
195 +function startsBlank(node: MarkdownNode): boolean {
196 + if (typeof node.value === "string") return /^[ \t\r\n]/.test(node.value);
197 + const first = node.children?.[0];
198 + return first !== undefined && startsBlank(first);
199 +}
200 +
201 +// Split or trimmed texts drop their position: nothing later reads a text node's.
202 +function withValue({ position: _position, ...text }: MarkdownNode, value: string): MarkdownNode {
203 + return { ...text, value };
204 +}
205 +
206 +/** `node` with the start trim that converting to HTML gives whatever follows a line break. */
207 +function trimmedAfterBreak(node: MarkdownNode): MarkdownNode {
208 + const value = node.value ?? "";
209 + if (node.type === "text" && /^[ \t]/.test(value)) {
210 + return withValue(node, value.replace(/^[ \t]+/, ""));
211 + }
212 + // Code turns its line endings into spaces first.
213 + if (node.type === "inlineCode" && /^[ \t\r\n]/.test(value)) {
214 + return { ...node, value: value.replace(/^(?:[ \t]|\r?\n|\r)+/, "") };
215 + }
216 + const head = node.children?.[0];
217 + if (INLINE_CONTAINERS.has(node.type) && node.type !== "linkReference" && head?.type === "text") {
218 + return { ...node, children: [trimmedAfterBreak(head), ...node.children!.slice(1)] };
219 + }
220 + return node;
221 +}
222 +
223 +/**
224 + * Converting to HTML trims the start of the node after a line break, and the span now takes that
225 + * place, so the node after the span gets the same trim. A line whose content still starts with
226 + * whitespace, which the browser hides at a line start, keeps no span, so it shows as without one.
227 + */
228 +function settle(children: MarkdownNode[], at: number, afterBreak: boolean): void {
229 + if (afterBreak && children[at + 1]) children[at + 1] = trimmedAfterBreak(children[at + 1]!);
230 + let index = at + 1;
231 + while (children[index]?.type === "text" && children[index]!.value === "") index++;
232 + const content = children[index];
233 + if (!content || content.type === "break" || startsBlank(content)) children.splice(at, 1);
234 +}
235 +
236 +function restoreParagraph(
237 + source: Source,
238 + paragraph: MarkdownNode,
239 + chain: readonly Container[],
240 + skipFirstLine: boolean,
241 +): void {
242 + const start = paragraph.position?.start;
243 + if (start?.offset === undefined || !paragraph.children) return;
244 + const typed = (at: number) => typedWhitespace(source, at, paragraph, chain);
245 + const inserted = new Set<MarkdownNode>();
246 + const indentation = (value: string) => {
247 + const node = whitespaceNode(value);
248 + inserted.add(node);
249 + return node;
250 + };
251 + // Nodes a plugin made without a position follow on from their siblings.
252 + let line = start.line;
253 +
254 + const split = (text: MarkdownNode): MarkdownNode[] => {
255 + const value = text.value ?? "";
256 + const endings = [...value.matchAll(LINE_ENDING)];
257 + // A character reference such as `&#10;` adds a line ending the source does not have.
258 + if (
259 + endings.length === 0 ||
260 + (text.position && text.position.end.line - text.position.start.line !== endings.length)
261 + ) {
262 + return [text];
263 + }
264 + const parts: MarkdownNode[] = [];
265 + let cut = 0;
266 + for (const ending of endings) {
267 + line += 1;
268 + const whitespace = typed(line);
269 + if (!whitespace) continue;
270 + const at = ending.index + ending[0].length;
271 + parts.push(withValue(text, value.slice(cut, at)), indentation(whitespace));
272 + cut = at;
273 + }
274 + if (parts.length === 0) return [text];
275 + if (cut < value.length) parts.push(withValue(text, value.slice(cut)));
276 + return parts;
277 + };
278 +
279 + const walk = (parent: MarkdownNode) => {
280 + const children: MarkdownNode[] = [];
281 + for (const child of parent.children ?? []) {
282 + line = child.position?.start.line ?? line;
283 + if (child.type === "text") {
284 + children.push(...split(child));
285 + } else {
286 + children.push(child);
287 + if (child.type === "break") {
288 + line = child.position?.end.line ?? line + 1;
289 + const whitespace = typed(line);
290 + if (whitespace) children.push(indentation(whitespace));
291 + } else if (INLINE_CONTAINERS.has(child.type)) {
292 + walk(child);
293 + }
294 + }
295 + line = child.position?.end.line ?? line;
296 + }
297 + // From the end, so settling one span leaves the earlier ones' places as they are.
298 + for (let index = children.length - 1; index >= 0; index--) {
299 + if (inserted.has(children[index]!)) settle(children, index, true);
300 + }
301 + parent.children = children;
302 + };
303 +
304 + const first = skipFirstLine ? "" : typed(start.line);
305 + walk(paragraph);
306 + if (first) {
307 + paragraph.children.unshift(indentation(first));
308 + settle(paragraph.children, 0, false);
309 + }
310 +}
311 +
312 +// The alert plugin reads the marker from the first text, and drops the marker's line.
313 +function startsWithAlertMarker(paragraph: MarkdownNode): boolean {
314 + const first = paragraph.children?.[0];
315 + return first?.type === "text" && GITHUB_ALERT_MARKER.test(first.value ?? "");
316 +}
317 +
318 +function visit(
319 + source: Source,
320 + node: MarkdownNode,
321 + chain: readonly Container[],
322 + firstInQuote: boolean,
323 +): void {
324 + if (node.type === "paragraph") {
325 + restoreParagraph(source, node, chain, firstInQuote && startsWithAlertMarker(node));
326 + return;
327 + }
328 + // Other containers, such as footnote definitions, keep CommonMark's reading.
329 + const inner =
330 + node.type === "root" || node.type === "list"
331 + ? chain
332 + : node.type === "blockquote"
333 + ? [...chain, { kind: "quote" } as const]
334 + : node.type === "listItem"
335 + ? [...chain, { kind: "item", node } as const]
336 + : undefined;
337 + if (!inner) return;
338 + node.children?.forEach((child, index) =>
339 + visit(source, child, inner, node.type === "blockquote" && index === 0),
340 + );
341 +}
342 +
343 +/**
344 + * CommonMark drops the spaces and tabs at the start of each line of a paragraph. This puts back
345 + * what was typed past the structure CommonMark reads, such as a list item's indentation or a
346 + * quote's marker, in spans the stylesheet keeps whitespace in. `source` is the text the node's
347 + * offsets point into; `indent` is how far its lines are already indented.
348 + */
349 +export function keepLeadingWhitespace(
350 + node: { readonly type: string; readonly children?: readonly object[] | undefined },
351 + source: string,
352 + indent = 0,
353 +): void {
354 + if (!MAY_KEEP_WHITESPACE.test(source)) return;
355 + const lineStarts = [0];
356 + for (const ending of source.matchAll(LINE_ENDING))
357 + lineStarts.push(ending.index + ending[0].length);
358 + const chain: Container[] = indent > 0 ? [{ kind: "indent", indent }] : [];
359 + visit({ text: source, lineStarts }, node as MarkdownNode, chain, false);
360 +}
361 +
362 +const enabledProcessors = new WeakSet<object>();
363 +
364 +/**
365 + * Opts a processor in. User messages use it so the sent bubble shows each line as it was typed
366 + * and the agent received it; agent replies and files keep CommonMark's reading.
367 + */
368 +export function remarkEnableLeadingWhitespace(this: object): undefined {
369 + enabledProcessors.add(this);
370 +}
371 +
372 +export function keepsLeadingWhitespace(processor: object): boolean {
373 + return enabledProcessors.has(processor);
374 +}
375 +
376 +/** Sits before plugins that split text without positions, and before list recovery. */
377 +export function remarkKeepLeadingWhitespace(this: object): Transformer<Root> {
378 + return (tree, file) => {
379 + if (enabledProcessors.has(this)) keepLeadingWhitespace(tree, String(file));
380 + };
381 +}
packages/shared/src/markdownListIndentation.ts
+10
@@ -1,3 +1,4 @@
1 +import { keepLeadingWhitespace, keepsLeadingWhitespace } from "./markdownLeadingWhitespace.ts";
2 import { keepOrderedListNumbers, keepsOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
3
4 interface MarkdownPosition {
@@ -96,6 +97,15 @@ function blocksFromIndentedCode(node: MarkdownAstNode, parser: MarkdownParser):
97 if (keepsOrderedListNumbers(parser)) {
98 for (const block of recovered.blocks) keepOrderedListNumbers(block, recovered.source);
99 }
100 + if (keepsLeadingWhitespace(parser)) {
101 + // The trimmed first line sets where the item's content starts.
102 + const indent = /^[\t ]*/.exec(String(node.value))![0];
103 + const columns = [...indent].reduce(
104 + (column, character) => (character === "\t" ? (Math.floor(column / 4) + 1) * 4 : column + 1),
105 + 0,
106 + );
107 + keepLeadingWhitespace({ type: "root", children: recovered.blocks }, recovered.source, columns);
108 + }
109 const first = recovered.blocks[0];
110 return {
111 ...recovered,
packages/shared/src/markdownPipeline.test.ts
+240
@@ -1,6 +1,7 @@
1 import { expect, it } from "vite-plus/test";
2 import { type PluggableList, unified } from "unified";
3 import rehypeRaw from "rehype-raw";
4 +import remarkGfm from "remark-gfm";
5 import remarkParse from "remark-parse";
6 import remarkRehype from "remark-rehype";
7 import type { Link, Root, RootContent } from "mdast";
@@ -8,6 +9,7 @@ import {
9 CHAT_MARKDOWN_REHYPE_PLUGINS,
10 CHAT_MARKDOWN_REMARK_PLUGINS,
11 CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
12 + CHAT_USER_MARKDOWN_REMARK_PLUGINS,
13 remarkAssistantInsights,
14 remarkEnableOrderedListNumbers,
15 } from "./markdownPipeline.ts";
@@ -637,6 +639,244 @@ it("keeps the number written on each ordered list item only when enabled", () =>
639 ).toEqual([undefined, undefined, undefined, undefined, undefined]);
640 });
641
642 +type ShownNode = {
643 + type: string;
644 + tagName?: string;
645 + value?: string;
646 + properties?: Record<string, unknown>;
647 + children?: ShownNode[];
648 +};
649 +const SHOWN_BLOCKS = new Set([
650 + "p",
651 + "li",
652 + "blockquote",
653 + "ul",
654 + "ol",
655 + "pre",
656 + "h1",
657 + "h2",
658 + "table",
659 + "tr",
660 +]);
661 +
662 +/** The text a browser shows, by line: whitespace collapses except in `pre` and kept whitespace. */
663 +function shownText(tree: ShownNode): string {
664 + let out = "";
665 + const visit = (node: ShownNode, keep: boolean) => {
666 + if (node.type === "text") {
667 + const value = node.value ?? "";
668 + // Only spaces, tabs and line endings collapse; a no-break space shows.
669 + out += keep
670 + ? value.replace(/ /g, "\u0001").replace(/\t/g, "\u0002")
671 + : value.replace(/[ \t\r\n]/g, " ");
672 + }
673 + // An image shows as something, which its alt text stands in for.
674 + if (node.properties?.alt !== undefined) out += String(node.properties.alt);
675 + if (node.tagName === "br" || SHOWN_BLOCKS.has(node.tagName ?? "")) out += "\n";
676 + const inner =
677 + keep || node.tagName === "pre" || node.properties?.dataLeadingWhitespace !== undefined;
678 + node.children?.forEach((child) => visit(child, inner));
679 + if (SHOWN_BLOCKS.has(node.tagName ?? "")) out += "\n";
680 + };
681 + visit(tree, false);
682 + return out
683 + .split("\n")
684 + .map((line) => line.replace(/ +/g, " ").replace(/^ | $/g, ""))
685 + .filter(Boolean)
686 + .join("\n")
687 + .replace(/\u0001/g, " ")
688 + .replace(/\u0002/g, "\t");
689 +}
690 +
691 +const hastRenderer = (plugins: PluggableList) => {
692 + const processor = unified()
693 + .use(remarkParse)
694 + .use(plugins)
695 + .use(remarkRehype, { allowDangerousHtml: true });
696 + return (markdown: string) => processor.runSync(processor.parse(markdown), markdown) as ShownNode;
697 +};
698 +const userMessage = hastRenderer([
699 + ...CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
700 + ...CHAT_USER_MARKDOWN_REMARK_PLUGINS,
701 +]);
702 +// The user pipeline as it was before leading whitespace was kept.
703 +const commonMarkUserMessage = hastRenderer([
704 + ...CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
705 + [remarkGfm, { singleTilde: false }],
706 +]);
707 +/** The tree with the restored indentation taken out, with the empty texts and positions it leaves. */
708 +const withoutRestoredWhitespace = ({
709 + position: _position,
710 + ...node
711 +}: ShownNode & { position?: unknown }): ShownNode => ({
712 + ...node,
713 + ...(node.children && {
714 + children: node.children
715 + .filter(
716 + (child) =>
717 + child.properties?.dataLeadingWhitespace === undefined &&
718 + !(child.type === "text" && child.value === ""),
719 + )
720 + .map(withoutRestoredWhitespace),
721 + }),
722 +});
723 +/** Each link's text and where it goes. */
724 +const linkTargets = (node: ShownNode): [string, unknown][] =>
725 + node.properties?.href === undefined
726 + ? (node.children ?? []).flatMap(linkTargets)
727 + : [[shownText(node), node.properties.href]];
728 +
729 +it("shows the spaces and tabs typed at the start of each line of a user message", () => {
730 + const shown = (markdown: string) => shownText(userMessage(markdown));
731 + const outline = "Plan\n Step one\n Step two\n Detail a\n Detail b";
732 + expect(shown(outline)).toBe(outline);
733 + expect(shown(outline.replaceAll("\n", "\r\n"))).toBe(outline);
734 + expect(shown(" First line\nnext\n\n New paragraph")).toBe(
735 + " First line\nnext\n New paragraph",
736 + );
737 + expect(shown("Plan\n\tTabbed\n \t Mixed")).toBe("Plan\n\tTabbed\n \t Mixed");
738 + expect(shown("Plan \n After a hard break\\\n and an escaped one")).toBe(
739 + "Plan\n After a hard break\n and an escaped one",
740 + );
741 + // In a container, only whitespace past its own structure counts.
742 + expect(shown("- item\n continued")).toBe("item\ncontinued");
743 + expect(shown("- item\n more")).toBe("item\n more");
744 + expect(shown("- item\ncontinued")).toBe("item\ncontinued");
745 + expect(shown("- item\n continued")).toBe("item\ncontinued");
746 + expect(shown("1. one\n two")).toBe("one\n two");
747 + expect(shown("- item\n\tsplit tab")).toBe("item\n split tab");
748 + expect(shown("- item\n\n second paragraph")).toBe("item\n second paragraph");
749 + // An item's content starts after its marker and the marker's spacing. A task's checkbox is
750 + // content, so it does not move where continuation lines start, however the task begins.
751 + expect(shown("- [ ] task\n under the task")).toBe("task\n under the task");
752 + for (const task of ["task", "**task**", "[task](https://example.com)", "`task`"]) {
753 + expect(shown(`- [ ] ${task}\n detail`), task).toBe("task\n detail");
754 + }
755 + expect(shown("- x\n y")).toBe("x\n y");
756 + expect(shown("- x\n y")).toBe("x\n y");
757 + expect(shown("-\tx\n\t y")).toBe("x\n y");
758 + expect(shown("10. x\n y")).toBe("x\n y");
759 + expect(shown("> a\n> b")).toBe("a\n b");
760 + expect(shown("> first")).toBe(" first");
761 + expect(shown("> a\n lazy")).toBe("a\nlazy");
762 + expect(shown("> - item\n> more")).toBe("item\n more");
763 + expect(shown("- > a\n > b")).toBe("a\n b");
764 + expect(shown("- a\n - b\n c")).toBe("a\nb\n c");
765 + // List recovery reparses over-indented content; its first line sets the indentation.
766 + expect(shown("- first block\n continued\n deeper")).toBe(
767 + "first block\ncontinued\n deeper",
768 + );
769 +
770 + // A space or tab written as a character reference after the indentation is dropped, as it is
771 + // without the indentation; only the typed indentation is added.
772 + for (const reference of ["&#32;", "&#x20;", "&#9;", "&Tab;"]) {
773 + expect(shown(`a\n ${reference}b`), reference).toBe("a\n b");
774 + expect(shown(`a \n\t${reference}b`), reference).toBe("a\n\tb");
775 + expect(shown(`a\\\r\n ${reference}${reference}b`), reference).toBe("a\n b");
776 + }
777 + // Converting to HTML also drops the spaces that start code or a link's text after a line break.
778 + expect(shown("a\n ` npm i`")).toBe("a\n npm i");
779 + expect(shown("a\n [ docs](https://example.com)")).toBe("a\n docs");
780 + // A line whose text starts with whitespace the browser hides keeps no indentation, as without it.
781 + expect(shown(" &#32;b")).toBe("b");
782 + expect(shown("a\n ***&#32;b***")).toBe("a\nb");
783 +
784 + // Only the indentation is added: formatting, code, links and alerts render as without it.
785 + const inline = "Plan\n **bold** [link](https://example.com) `code` ~/a and ~~gone~~ /tmp/a.ts";
786 + const alert = "> [!NOTE]\n> indented body";
787 + expect(shown(inline)).toBe("Plan\n bold link code ~/a and gone /tmp/a.ts");
788 + expect(shown(alert)).toBe(" indented body");
789 + for (const markdown of [inline, alert]) {
790 + expect(withoutRestoredWhitespace(userMessage(markdown)), markdown).toEqual(
791 + withoutRestoredWhitespace(commonMarkUserMessage(markdown)),
792 + );
793 + }
794 + expect(linkTargets(userMessage(inline))).toEqual(linkTargets(commonMarkUserMessage(inline)));
795 + expect(linkTargets(userMessage(inline)).map(([label]) => label)).toEqual(["link", "/tmp/a.ts"]);
796 +
797 + // A line ending written as a character reference is not a source line: it never takes a typed
798 + // line's indentation, and no typed line's indentation moves onto it.
799 + const decoded = shown("a &#10; b\n c").split("\n");
800 + expect(decoded.slice(0, 2)).toEqual(["a", "b"]);
801 + expect(["c", " c"]).toContain(decoded[2]);
802 +
803 + // Code, and agent replies, keep CommonMark's reading.
804 + for (const markdown of [
805 + "```\n code\n more\n```",
806 + "Plan\n\n code\n deeper",
807 + "- item\n\n code",
808 + ]) {
809 + expect(userMessage(markdown), markdown).toEqual(commonMarkUserMessage(markdown));
810 + }
811 + for (const plugins of [CHAT_MARKDOWN_REMARK_PLUGINS, CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS]) {
812 + expect(shownText(hastRenderer(plugins)(outline)).replaceAll("\n", " ")).toBe(
813 + "Plan Step one Step two Detail a Detail b",
814 + );
815 + }
816 +});
817 +
818 +it("adds only the typed indentation, and renders what follows it as without it", () => {
819 + const starts = [
820 + "b",
821 + "*b*",
822 + "`b`",
823 + "~~b~~",
824 + "[b](https://example.com)",
825 + "![b](https://example.com/b.png)",
826 + "https://example.com/b",
827 + "/tmp/b.ts",
828 + "~/b",
829 + "@b",
830 + "$b",
831 + "&amp;b",
832 + "&nbsp;b",
833 + // Spaces and tabs written as character references, which a line start drops.
834 + "&#32;b",
835 + "&#x20;b",
836 + "&#9;b",
837 + "&Tab;b",
838 + "&#32;&#9;b",
839 + "&#32;* b",
840 + "&#32;`b`",
841 + "&#32;**b**",
842 + // Content whose first character is a space.
843 + "` b`",
844 + "[ b](https://example.com)",
845 + "**&#32;b**",
846 + "***&#32;b***",
847 + "[ b]",
848 + ];
849 + const lines = [
850 + (indent: string, start: string, join: string) => `a${join}${indent}${start}`,
851 + (indent: string, start: string, join: string) => `> a${join}> ${indent}${start}`,
852 + (indent: string, start: string, join: string) => `- a${join} ${indent}${start}`,
853 + ];
854 + for (const indent of ["", " ", " ", "\t", " \t"]) {
855 + for (const start of starts) {
856 + // Without the indentation, an image alone on a CRLF line breaks twice: the image-line break
857 + // leaves the CR behind, which breaks again.
858 + const joins = start.startsWith("!") ? ["\n", " \n", "\\\n"] : ["\n", "\r\n", " \n", "\\\n"];
859 + const markdowns = [
860 + ...joins.flatMap((join) => lines.map((line) => line(indent, start, join))),
861 + // A paragraph's first line takes at most three spaces before it reads as code.
862 + ...(indent.includes("\t") ? [] : [`${indent}${start}`, `> ${indent}${start}`]),
863 + ];
864 + for (const markdown of markdowns) {
865 + const label = JSON.stringify(markdown);
866 + expect(withoutRestoredWhitespace(userMessage(markdown)), label).toEqual(
867 + withoutRestoredWhitespace(commonMarkUserMessage(markdown)),
868 + );
869 + const shown = shownText(userMessage(markdown)).split("\n");
870 + const before = shownText(commonMarkUserMessage(markdown)).split("\n");
871 + expect(shown.length, label).toBe(before.length);
872 + shown.forEach((line, index) =>
873 + expect([before[index], `${indent}${before[index]}`], label).toContain(line),
874 + );
875 + }
876 + }
877 + }
878 +});
879 +
880 it("moves a <picture>'s color-scheme sources onto its <img>, which the renderer resolves", () => {
881 const processor = unified()
882 .use(remarkParse)
packages/shared/src/markdownPipeline.ts
+8
@@ -21,6 +21,10 @@ import remarkGfm from "remark-gfm";
21 import { remarkGithubAlerts } from "./markdownGithubAlerts.ts";
22 import { remarkNormalizeListItemIndentation } from "./markdownListIndentation.ts";
23 import { remarkKeepOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
24 +import {
25 + remarkEnableLeadingWhitespace,
26 + remarkKeepLeadingWhitespace,
27 +} from "./markdownLeadingWhitespace.ts";
28 import {
29 CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES,
30 remarkCodexDirectives,
@@ -259,10 +263,12 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
263 remarkNormalizeLinksAndTagInlineCode,
264 ];
265
266 +// Leading whitespace is kept only where each line renders as a line.
267 export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
268 remarkGfm,
269 remarkEndAutolinksAtScriptPunctuation,
270 remarkKeepAutolinkSuffixes,
271 + remarkKeepLeadingWhitespace,
272 remarkLinkAbsoluteFilePaths,
273 remarkKeepWindowsPaths,
274 remarkGithubAlerts,
@@ -277,8 +283,10 @@ export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
283
284 // Users type paths like ~/a and (~/b, which GFM's single-tilde strikethrough hides as <del>.
285 // Appended after the base list, this reconfigures its remark-gfm entry; ~~double~~ still strikes.
286 +// Users also indent lines with spaces, which CommonMark drops from a paragraph.
287 export const CHAT_USER_MARKDOWN_REMARK_PLUGINS: PluggableList = [
288 [remarkGfm, { singleTilde: false }],
289 + remarkEnableLeadingWhitespace,
290 ];
291
292 // Start tags an HTML parser closes an open `<p>` for.
packages/shared/src/threadFindText.test.ts
+17
@@ -29,6 +29,23 @@ it("indexes single tildes in user paths as typed", () => {
29 ).toBe("Drop old");
30 });
31
32 +it("indexes the spaces typed at the start of a user's lines, as the bubble shows them", () => {
33 + const text = "Plan\n Step one\n\tDetail";
34 + // Each line ending reads as one space, as the text after a rendered line break does.
35 + expect(searchableMessageSegments({ role: "user", text, streaming: false })).toEqual([
36 + "Plan",
37 + " Step one",
38 + " \tDetail",
39 + ]);
40 + const indexed = (text: string) =>
41 + searchableMessageSegments({ role: "user", text, streaming: false });
42 + // A task's checkbox does not move where its continuation lines start.
43 + expect(indexed("- [ ] task\n detail")).toEqual([" task", " detail"]);
44 + // A space or tab written as a character reference is dropped there, as without the indentation.
45 + expect(indexed("a\n &#32;b")).toEqual(["a", " b"]);
46 + expect(indexed("a \n\t&Tab;b")).toEqual(["a", " \tb"]);
47 +});
48 +
49 it("excludes structured context chips without joining text across them", () => {
50 expect(
51 searchableMessageSegments({
scripts/scenarios/sent-message-keeps-leading-spaces.e2e.ts new
+35
@@ -0,0 +1,35 @@
1 +/// <reference lib="dom" />
2 +// The DOM lib types the function this scenario runs in the page.
3 +import { expect } from "e2e";
4 +
5 +import { test, WEB } from "./t3.ts";
6 +
7 +const LINES = ["Plan", " Step one", " Step two", " Detail a", " Detail b"];
8 +
9 +test(
10 + "A sent message shows the spaces typed at the start of its lines",
11 + WEB,
12 + async ({ t3, app, browser, screen }) => {
13 + const project = await t3.project("demo");
14 + const thread = await t3.thread({ project, title: "Outline", messages: ["Start"] });
15 + await t3.open(thread);
16 +
17 + const composer = screen.getByRole("textbox");
18 + for (const [index, line] of LINES.entries()) {
19 + if (index > 0) await composer.press("Shift+Enter");
20 + await composer.pressSequentially(line);
21 + }
22 + await composer.press("Enter");
23 + await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible();
24 + await app.screenshot("sent");
25 +
26 + // The visible text, which keeps whitespace only where the page shows it.
27 + const shown = await browser.evaluate(
28 + () =>
29 + [...document.querySelectorAll<HTMLElement>("[data-user-message-body]")]
30 + .map((body) => body.innerText)
31 + .find((text) => text.includes("Detail b")) ?? null,
32 + );
33 + expect(shown).toBe(LINES.join("\n"));
34 + },
35 +);