The best way to control Claude Code, Codex and more from any computer or phone

main
ts 497 lines 17.8 KB
Raw
1 import { tokenizeCode, type HighlightScheme, type TokenizedCode } from "../../lib/highlights";
2 import type { NativeReviewDiffFile, NativeReviewDiffSource } from "./nativeReviewDiffTypes";
3 import type { NativeReviewDiffRow, NativeReviewDiffToken } from "./nativeReviewDiffSurface";
4 import { diffHighlightTraceId, traceDiffHighlight } from "./nativeReviewDiffTrace";
5
6 interface NativeReviewDiffLineRow extends NativeReviewDiffRow {
7 readonly kind: "line";
8 readonly fileId: string;
9 readonly content: string;
10 }
11
12 export interface HighlightNativeReviewDiffVisibleRowsInput {
13 readonly rows: ReadonlyArray<NativeReviewDiffRow>;
14 readonly files: ReadonlyArray<NativeReviewDiffFile>;
15 readonly scheme: HighlightScheme;
16 readonly firstRowIndex: number;
17 readonly lastRowIndex: number;
18 readonly overscanRows?: number;
19 readonly maxRows?: number;
20 readonly alreadyHighlightedRowIds?: ReadonlySet<string>;
21 readonly collapsedFileIds?: ReadonlySet<string>;
22 readonly signal?: AbortSignal;
23 readonly onChunk?: (chunk: NativeReviewDiffHighlightChunk) => void;
24 }
25
26 export interface NativeReviewDiffHighlightChunk {
27 readonly tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>>;
28 readonly retryableRowIds: ReadonlyArray<string>;
29 readonly rowCount: number;
30 readonly durationMs: number;
31 }
32
33 const NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS = 160;
34 const NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS = 360;
35 function isHighlightableLineRow(row: NativeReviewDiffRow): row is NativeReviewDiffLineRow {
36 return row.kind === "line" && typeof row.fileId === "string" && typeof row.content === "string";
37 }
38
39 function hasConsecutiveLineNumbers(
40 previous: number | null | undefined,
41 next: number | null | undefined,
42 ): boolean {
43 return typeof previous === "number" && typeof next === "number" && next === previous + 1;
44 }
45
46 // Native lexing is non-incremental. Cache theme-independent raw records, never
47 // expanded token objects for an entire large file. Bound both entry count and bytes.
48 const MAX_TOKEN_CACHE_BYTES = 64 * 1024 * 1024;
49 const MAX_TOKEN_CACHE_ENTRIES = 8;
50 // LRU entries own only token data; source contexts and diff rows can collect independently.
51 const tokenCacheKeys = new WeakMap<object, object>();
52 const tokenCache = new Map<object, { language: string; tokens: TokenizedCode }>();
53 let tokenCacheBytes = 0;
54
55 function getTokenCacheKey(context: object): object {
56 let key = tokenCacheKeys.get(context);
57 if (!key) {
58 key = {};
59 tokenCacheKeys.set(context, key);
60 }
61 return key;
62 }
63
64 function getCachedTokens(
65 cacheKey: object,
66 contents: string | (() => string),
67 language: string,
68 ): TokenizedCode {
69 const cached = tokenCache.get(cacheKey);
70 if (cached) {
71 tokenCache.delete(cacheKey);
72 tokenCacheBytes -= cached.tokens.byteLength;
73 if (cached.language === language) {
74 tokenCache.set(cacheKey, cached);
75 tokenCacheBytes += cached.tokens.byteLength;
76 return cached.tokens;
77 }
78 }
79 const tokens = tokenizeCode(typeof contents === "string" ? contents : contents(), language);
80 if (tokens.byteLength <= MAX_TOKEN_CACHE_BYTES) {
81 tokenCache.set(cacheKey, { language, tokens });
82 tokenCacheBytes += tokens.byteLength;
83 }
84 while (tokenCacheBytes > MAX_TOKEN_CACHE_BYTES || tokenCache.size > MAX_TOKEN_CACHE_ENTRIES) {
85 const oldest = tokenCache.keys().next().value!;
86 tokenCacheBytes -= tokenCache.get(oldest)!.tokens.byteLength;
87 tokenCache.delete(oldest);
88 }
89 return tokens;
90 }
91
92 const fullSourceContexts = new WeakMap<
93 NativeReviewDiffSource,
94 { old: FullSourceContext; new: FullSourceContext }
95 >();
96
97 interface FullSourceContext {
98 readonly contents: string;
99 readonly comparisonLines: ReadonlyArray<string>;
100 readonly validation: WeakMap<ReadonlyArray<NativeReviewDiffLineRow>, boolean>;
101 }
102
103 const sourceRowsByFileCache = new WeakMap<
104 ReadonlyArray<NativeReviewDiffRow>,
105 Map<string, NativeReviewDiffLineRow[]>
106 >();
107
108 function getSourceRowsByFile(rows: ReadonlyArray<NativeReviewDiffRow>) {
109 let byFile = sourceRowsByFileCache.get(rows);
110 if (!byFile) {
111 byFile = new Map();
112 for (const row of rows) {
113 if (!isHighlightableLineRow(row)) continue;
114 const fileRows = byFile.get(row.fileId);
115 if (fileRows) fileRows.push(row);
116 else byFile.set(row.fileId, [row]);
117 }
118 sourceRowsByFileCache.set(rows, byFile);
119 }
120 return byFile;
121 }
122
123 function matchesSourceSide(
124 context: FullSourceContext,
125 rows: ReadonlyArray<NativeReviewDiffLineRow>,
126 side: DiffSide,
127 ) {
128 const cached = context.validation.get(rows);
129 if (cached !== undefined) return cached;
130 // Context participates on both sides, even though it displays NEW tokens.
131 // A known mismatch anywhere invalidates grammar from the entire source side.
132 const matches = rows.every((row) => {
133 if (row.change === (side === "old" ? "add" : "delete")) return true;
134 const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
135 return (
136 typeof lineNumber === "number" && context.comparisonLines[lineNumber - 1] === row.content
137 );
138 });
139 context.validation.set(rows, matches);
140 return matches;
141 }
142
143 function createFullSourceContext(contents: string): FullSourceContext {
144 return {
145 contents,
146 // The diff parser retains CR on patch rows, unlike the token output.
147 comparisonLines: contents.split("\n"),
148 validation: new WeakMap(),
149 };
150 }
151
152 function getFullSourceContexts(source: NativeReviewDiffSource) {
153 let contexts = fullSourceContexts.get(source);
154 if (!contexts) {
155 contexts = {
156 old: createFullSourceContext(source.oldContents),
157 new: createFullSourceContext(source.newContents),
158 };
159 fullSourceContexts.set(source, contexts);
160 }
161 return contexts;
162 }
163
164 type DiffSide = "old" | "new";
165
166 interface SourceSegment {
167 readonly fileId: string;
168 readonly side: DiffSide;
169 readonly rows: NativeReviewDiffLineRow[];
170 }
171
172 // Row arrays are immutable snapshots; comments and hidden hunks retain source context.
173 const sourceSegmentsCache = new WeakMap<
174 ReadonlyArray<NativeReviewDiffRow>,
175 Map<string, { readonly segment: SourceSegment; readonly index: number }>
176 >();
177
178 function getSourceSegments(rows: ReadonlyArray<NativeReviewDiffRow>) {
179 const cached = sourceSegmentsCache.get(rows);
180 if (cached) return cached;
181
182 const segmentsByRowId = new Map<
183 string,
184 { readonly segment: SourceSegment; readonly index: number }
185 >();
186 for (const side of ["old", "new"] as const) {
187 let segment: SourceSegment | undefined;
188 for (const row of rows) {
189 if (row.kind === "comment" || row.kind === "hunk") continue;
190 if (!isHighlightableLineRow(row)) {
191 segment = undefined;
192 continue;
193 }
194 if (row.change === (side === "old" ? "add" : "delete")) continue;
195
196 const previous = segment?.rows.at(-1);
197 const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
198 const previousLineNumber = side === "old" ? previous?.oldLineNumber : previous?.newLineNumber;
199 if (
200 !segment ||
201 segment.fileId !== row.fileId ||
202 !hasConsecutiveLineNumbers(previousLineNumber, lineNumber)
203 ) {
204 // A gap has no known incoming grammar state. Highlight the available
205 // fragment independently; never join it to an earlier, incomplete hunk.
206 segment = { fileId: row.fileId, side, rows: [] };
207 }
208 if (displaySide(row) === side) {
209 segmentsByRowId.set(row.id, { segment, index: segment.rows.length });
210 }
211 segment.rows.push(row);
212 }
213 }
214 sourceSegmentsCache.set(rows, segmentsByRowId);
215 return segmentsByRowId;
216 }
217
218 function displaySide(row: NativeReviewDiffLineRow): DiffSide {
219 // Unified context rows have one token array. Match the NEW side, while OLD
220 // context still advances deletion grammar independently.
221 return row.change === "delete" ? "old" : "new";
222 }
223
224 function createFileMap(files: ReadonlyArray<NativeReviewDiffFile>) {
225 return new Map(files.map((file) => [file.id, file]));
226 }
227
228 function clampRowIndex(index: number, rows: ReadonlyArray<NativeReviewDiffRow>) {
229 if (rows.length === 0) {
230 return 0;
231 }
232
233 return Math.min(rows.length - 1, Math.max(0, Math.floor(index)));
234 }
235
236 function makePlainTokenFallback(
237 row: NativeReviewDiffLineRow,
238 ): ReadonlyArray<NativeReviewDiffToken> {
239 return [{ content: row.content || " ", color: null, fontStyle: null }];
240 }
241
242 function settlesWithinTask(promise: Promise<unknown>): Promise<boolean> {
243 return Promise.race([
244 promise.then(
245 () => true,
246 () => true,
247 ),
248 new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 0)),
249 ]);
250 }
251
252 export async function highlightNativeReviewDiffVisibleRows(
253 input: HighlightNativeReviewDiffVisibleRowsInput,
254 ): Promise<{
255 readonly tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>>;
256 // Deliver plain tokens for these rows without marking them permanently highlighted.
257 readonly retryableRowIds: ReadonlyArray<string>;
258 readonly rowCount: number;
259 readonly durationMs: number;
260 }> {
261 const request = diffHighlightTraceId(input);
262 traceDiffHighlight("highlight-start", {
263 request,
264 first: input.firstRowIndex,
265 last: input.lastRowIndex,
266 overscan: input.overscanRows ?? NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS,
267 });
268 if (input.signal?.aborted) {
269 return {
270 tokensByRowId: {},
271 retryableRowIds: [],
272 rowCount: 0,
273 durationMs: 0,
274 };
275 }
276
277 const startedAt = performance.now();
278 const fileMap = createFileMap(input.files);
279 const overscanRows = input.overscanRows ?? NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS;
280 const maxRows = input.maxRows ?? NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS;
281 const startIndex = clampRowIndex(input.firstRowIndex - overscanRows, input.rows);
282 const endIndex = clampRowIndex(input.lastRowIndex + overscanRows, input.rows);
283 const visibleStartIndex = clampRowIndex(input.firstRowIndex, input.rows);
284 const visibleEndIndex = clampRowIndex(input.lastRowIndex, input.rows);
285 const visibleFileIds = new Set<string>();
286 // Overscan warms visible files without loading unrelated neighboring sources.
287 // Headers and already-highlighted rows still identify files in the viewport.
288 for (let rowIndex = visibleStartIndex; rowIndex <= visibleEndIndex; rowIndex += 1) {
289 const fileId = input.rows[rowIndex]?.fileId;
290 if (typeof fileId === "string") visibleFileIds.add(fileId);
291 }
292 const selectedRows: NativeReviewDiffLineRow[] = [];
293
294 for (
295 let rowIndex = startIndex;
296 rowIndex <= endIndex && selectedRows.length < maxRows;
297 rowIndex += 1
298 ) {
299 const row = input.rows[rowIndex];
300 if (
301 row &&
302 isHighlightableLineRow(row) &&
303 visibleFileIds.has(row.fileId) &&
304 !input.collapsedFileIds?.has(row.fileId) &&
305 !input.alreadyHighlightedRowIds?.has(row.id) &&
306 fileMap.has(row.fileId)
307 ) {
308 selectedRows.push(row);
309 }
310 }
311
312 const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
313 const retryableRowIds: string[] = [];
314 const selectedRowIds = new Set(selectedRows.map((row) => row.id));
315 const tokenize = (context: object, contents: string | (() => string), language: string) => {
316 const cacheKey = getTokenCacheKey(context);
317 const cacheHit = tokenCache.get(cacheKey)?.language === language;
318 const cached = getCachedTokens(cacheKey, contents, language);
319 traceDiffHighlight("tokenized", {
320 request,
321 characters: typeof contents === "string" ? contents.length : undefined,
322 cacheHit,
323 retainedBytes: cached.byteLength,
324 lines: cached.lineCount,
325 aborted: input.signal?.aborted ?? false,
326 });
327 return cached;
328 };
329
330 // Patch-only grammar: each run of consecutive lines is tokenized on its own.
331 const highlightSegmentRows = (
332 rowIds: ReadonlySet<string>,
333 out: Record<string, ReadonlyArray<NativeReviewDiffToken>>,
334 ) => {
335 const sourceSegments = getSourceSegments(input.rows);
336 const selectedSegments = new Map<
337 SourceSegment,
338 { firstSelected: number; lastSelected: number }
339 >();
340 for (const row of selectedRows) {
341 if (!rowIds.has(row.id)) continue;
342 const source = sourceSegments.get(row.id);
343 if (!source) continue;
344 const selection = selectedSegments.get(source.segment);
345 if (selection) selection.lastSelected = source.index;
346 else
347 selectedSegments.set(source.segment, {
348 firstSelected: source.index,
349 lastSelected: source.index,
350 });
351 }
352 for (const [segment, { firstSelected, lastSelected }] of selectedSegments) {
353 if (input.signal?.aborted) return;
354 const file = fileMap.get(segment.fileId);
355 if (!file) continue;
356 const tokens = tokenize(
357 segment,
358 () => segment.rows.map((row) => row.content).join("\n"),
359 file.language,
360 );
361 for (let index = firstSelected; index <= lastSelected; index += 1) {
362 const row = segment.rows[index]!;
363 if (rowIds.has(row.id) && displaySide(row) === segment.side) {
364 out[row.id] = tokens.highlightLine(index, input.scheme) ?? makePlainTokenFallback(row);
365 }
366 }
367 }
368 };
369
370 const sourceFiles = new Map<string, NativeReviewDiffLineRow[]>();
371 for (const row of selectedRows) {
372 if (!fileMap.get(row.fileId)?.loadContents) continue;
373 const fileRows = sourceFiles.get(row.fileId);
374 if (fileRows) fileRows.push(row);
375 else sourceFiles.set(row.fileId, [row]);
376 }
377 const emitChunk = (
378 chunkTokens: Record<string, ReadonlyArray<NativeReviewDiffToken>>,
379 chunkRetryableRowIds: ReadonlyArray<string>,
380 ) => {
381 const rowCount = Object.keys(chunkTokens).length;
382 if (!input.onChunk || input.signal?.aborted || rowCount === 0) return;
383 input.onChunk({
384 tokensByRowId: chunkTokens,
385 retryableRowIds: chunkRetryableRowIds,
386 rowCount,
387 durationMs: Math.round(performance.now() - startedAt),
388 });
389 };
390 // Full sources arrive over the network. Until they do, show patch-only colors
391 // as retryable rows, which the full-source tokens then replace.
392 const emitProvisionalChunk = (rows: ReadonlyArray<NativeReviewDiffLineRow>) => {
393 if (!input.onChunk) return;
394 const provisional: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
395 highlightSegmentRows(new Set(rows.map((row) => row.id)), provisional);
396 traceDiffHighlight("highlight-provisional", { request, rows: rows.length });
397 emitChunk(provisional, Object.keys(provisional));
398 };
399 // The file filling the viewport should not wait for a sliver of its neighbor.
400 const sourceEntries = [...sourceFiles].sort((a, b) => b[1].length - a[1].length);
401 let provisionalFrom = sourceEntries.length;
402 for (const [entryIndex, [fileId, fileRows]] of sourceEntries.entries()) {
403 if (input.signal?.aborted) break;
404 const file = fileMap.get(fileId)!;
405 const fileTraceId = diffHighlightTraceId(fileId);
406 traceDiffHighlight("highlight-source", { request, file: fileTraceId, rows: fileRows.length });
407 const loading = file.loadContents!(input.signal);
408 if (entryIndex < provisionalFrom && !(await settlesWithinTask(loading))) {
409 if (input.signal?.aborted) break;
410 provisionalFrom = entryIndex;
411 emitProvisionalChunk(sourceEntries.slice(entryIndex).flatMap(([, rows]) => rows));
412 }
413 let source: NativeReviewDiffSource | null;
414 try {
415 source = await loading;
416 } catch {
417 traceDiffHighlight("highlight-source-error", { request, file: fileTraceId });
418 // Patch-only colors below, retried with the source on a later viewport.
419 for (const row of fileRows) retryableRowIds.push(row.id);
420 continue;
421 }
422 if (input.signal?.aborted) break;
423 const contexts = source ? getFullSourceContexts(source) : null;
424 const fileTokens: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
425 for (const side of ["old", "new"] as const) {
426 if (input.signal?.aborted) break;
427 const candidate = contexts?.[side];
428 const context =
429 candidate &&
430 matchesSourceSide(candidate, getSourceRowsByFile(input.rows).get(fileId)!, side)
431 ? candidate
432 : null;
433 traceDiffHighlight("source-validation", {
434 request,
435 file: fileTraceId,
436 side,
437 present: !!candidate,
438 matches: !!context,
439 knownLines: context?.comparisonLines.length,
440 });
441 if (!context) continue;
442 // Rows the source cannot vouch for keep patch-only grammar below.
443 const matchingRows: { row: NativeReviewDiffLineRow; lineIndex: number }[] = [];
444 for (const row of fileRows) {
445 if (displaySide(row) !== side) continue;
446 const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
447 const lineIndex = typeof lineNumber === "number" ? lineNumber - 1 : -1;
448 if (
449 lineIndex >= 0 &&
450 lineIndex < context.comparisonLines.length &&
451 context.comparisonLines[lineIndex] === row.content
452 ) {
453 matchingRows.push({ row, lineIndex });
454 }
455 }
456 if (matchingRows.length === 0) continue;
457 const tokens = tokenize(context, context.contents, file.language);
458 for (const { row, lineIndex } of matchingRows) {
459 selectedRowIds.delete(row.id);
460 fileTokens[row.id] = tokensByRowId[row.id] =
461 tokens.highlightLine(lineIndex, input.scheme) ?? makePlainTokenFallback(row);
462 }
463 }
464 emitChunk(fileTokens, []);
465 await new Promise<void>((resolve) => setTimeout(resolve, 0));
466 }
467
468 if (selectedRowIds.size > 0 && !input.signal?.aborted) {
469 highlightSegmentRows(selectedRowIds, tokensByRowId);
470 }
471
472 if (input.signal?.aborted) {
473 traceDiffHighlight("highlight-aborted", { request });
474 return {
475 tokensByRowId: {},
476 retryableRowIds: [],
477 rowCount: 0,
478 durationMs: 0,
479 };
480 }
481
482 traceDiffHighlight("highlight-result", {
483 request,
484 selected: selectedRows.length,
485 delivered: Object.keys(tokensByRowId).length,
486 retryable: retryableRowIds.length,
487 colored: Object.values(tokensByRowId).filter((tokens) =>
488 tokens.some((token) => token.color !== null),
489 ).length,
490 });
491 return {
492 tokensByRowId,
493 retryableRowIds,
494 rowCount: selectedRows.length,
495 durationMs: Math.round(performance.now() - startedAt),
496 };
497 }