The best way to control Claude Code, Codex and more from any computer or phone

main
ts 481 lines 16.8 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 // Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files.
3 import * as NodeCrypto from "node:crypto";
4 import * as NodeFS from "node:fs";
5 import * as NodeFSP from "node:fs/promises";
6 import * as NodePath from "node:path";
7 import * as NodeStringDecoder from "node:string_decoder";
8 import * as NodeUtil from "node:util";
9 import * as NodeZlib from "node:zlib";
10
11 import type {
12 SelectedFields,
13 TranscriptUsageFormat,
14 UsageRecord,
15 } from "@t3tools/provider-core/server/usage";
16 import * as Option from "effect/Option";
17 import * as Schema from "effect/Schema";
18
19 import { createTranscriptJsonReader } from "../project/AgentSessionJson.ts";
20
21 export interface TranscriptFile {
22 readonly path: string;
23 readonly size: number;
24 readonly mtimeMs: number;
25 }
26
27 /** Resume only if bytes before resumeOffset match the guard hash; rewritten files would corrupt totals. */
28 export interface TranscriptParsePosition {
29 readonly resumeOffset: number;
30 readonly guardLength: number;
31 readonly guardHash: number;
32 readonly state: unknown;
33 }
34
35 export interface TranscriptParseResult {
36 readonly records: readonly UsageRecord[];
37 /** Unterminated tail records stay outside records/position so the next scan rereads them. */
38 readonly tailRecords: readonly UsageRecord[];
39 /** Includes tentative tail diagnostics; position.state only includes committed lines. */
40 readonly malformedRecords: number;
41 readonly position: TranscriptParsePosition;
42 readonly resumed: boolean;
43 }
44
45 export const GUARD_LENGTH = 64;
46 const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024;
47 const READ_CHUNK_BYTES = 256 * 1024;
48 const NEWLINE = 0x0a;
49 const STAT_CONCURRENCY = 32;
50 const WALK_CONCURRENCY = 16;
51 // Measured about 3x faster than the promise API's stat across 20,000 transcripts.
52 const statFile = NodeUtil.promisify(NodeFS.stat);
53 const CARRIAGE_RETURN = 0x0d;
54
55 function selectUsageFields(fields: SelectedFields) {
56 return (path: ReadonlyArray<string | number | null>): boolean => {
57 let selected: true | SelectedFields = fields;
58 for (const key of path) {
59 if (selected === true) return true;
60 if (typeof key !== "string" || !Object.hasOwn(selected, key)) return false;
61 selected = selected[key]!;
62 }
63 return true;
64 };
65 }
66
67 function fnv1a(buffer: Buffer): number {
68 let hash = 0x811c9dc5;
69 for (let index = 0; index < buffer.length; index += 1) {
70 hash ^= buffer[index]!;
71 hash = Math.imul(hash, 0x01000193);
72 }
73 return hash >>> 0;
74 }
75
76 /** Skips Grok logs without usage; preserves readdir order for first-seen deduplication. */
77 export async function listTranscriptFiles(
78 root: string,
79 sinceMs: number,
80 options?: { readonly fileName?: string },
81 ): Promise<{
82 readonly files: readonly TranscriptFile[];
83 readonly walked: readonly string[];
84 readonly failedPaths: number;
85 }> {
86 const fileName = options?.fileName;
87 let failedPaths = 0;
88 let reading = 0;
89 const waiting: Array<() => void> = [];
90 // Sibling directories read together; each returns its own list so the result keeps readdir order.
91 const walk = async (dir: string): Promise<readonly string[]> => {
92 if (reading < WALK_CONCURRENCY) reading += 1;
93 else await new Promise<void>((resume) => waiting.push(resume));
94 let entries;
95 try {
96 entries = await NodeFSP.readdir(dir, { withFileTypes: true });
97 } catch {
98 failedPaths += 1;
99 return [];
100 } finally {
101 // A waiting directory takes over this slot, so the count only drops when none waits.
102 const next = waiting.shift();
103 if (next === undefined) reading -= 1;
104 else next();
105 }
106 const found = await Promise.all(
107 entries.map((entry) => {
108 const child = NodePath.join(dir, entry.name);
109 if (entry.isDirectory()) return walk(child);
110 return (
111 fileName !== undefined
112 ? entry.name === fileName
113 : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst")
114 )
115 ? [child]
116 : [];
117 }),
118 );
119 return found.flat();
120 };
121 const candidates = await walk(root);
122
123 const found: Array<TranscriptFile | undefined> = Array.from({ length: candidates.length });
124 const queue = candidates.entries();
125 const statQueued = async (): Promise<void> => {
126 for (const [index, path] of queue) {
127 try {
128 const stats = await statFile(path);
129 if (stats.mtimeMs >= sinceMs) {
130 found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };
131 }
132 } catch (error) {
133 if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
134 failedPaths += 1;
135 }
136 }
137 }
138 };
139 await Promise.all(
140 Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),
141 );
142 return { files: found.filter((file) => file !== undefined), walked: candidates, failedPaths };
143 }
144
145 export async function readDirectoryVolumeId(path: string): Promise<string> {
146 try {
147 const stats = await NodeFSP.stat(path);
148 return `${stats.dev}:${stats.ino}`;
149 } catch {
150 return "";
151 }
152 }
153
154 /**
155 * Set by the desktop app, from a value it persists, on its Windows server and
156 * the WSL server it manages, so both can name one Windows directory alike
157 * without merging separate machines that share a hostname and path.
158 */
159 export const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE";
160
161 const MOUNT_FIELD_ESCAPE = /\\([0-7]{3})/g;
162 const WINDOWS_ROOT = /^(?:[A-Za-z]:|\\\\[^\\]+\\[^\\]+)\\?$/;
163
164 const unescapeMountField = (field: string) =>
165 field.replace(MOUNT_FIELD_ESCAPE, (_, octal: string) =>
166 String.fromCharCode(Number.parseInt(octal, 8)),
167 );
168
169 const normalizeWindowsPath = (path: string) =>
170 (/^[a-z]:/.test(path) ? path[0]!.toUpperCase() + path.slice(1) : path).replace(/\\$/, "");
171
172 /**
173 * The Windows directory behind the deepest mount holding `dir`, read from
174 * `/proc/self/mountinfo` so a custom automount root resolves and an ext4
175 * mount under `/mnt` never passes for a drive.
176 */
177 function windowsMountFor(
178 mountInfo: string,
179 dir: string,
180 ): { readonly mountPoint: string; readonly windowsRoot: string } | undefined {
181 let deepest: { mountPoint: string; windowsRoot: string | undefined } | undefined;
182 for (const line of mountInfo.split("\n")) {
183 const fields = line.split(" ");
184 const separator = fields.indexOf("-", 6);
185 if (separator === -1) continue;
186 const mountPoint = unescapeMountField(fields[4] ?? "");
187 const contains =
188 mountPoint === "/" ||
189 dir === mountPoint ||
190 dir.startsWith(mountPoint.endsWith("/") ? mountPoint : `${mountPoint}/`);
191 if (!contains || (deepest !== undefined && deepest.mountPoint.length > mountPoint.length)) {
192 continue;
193 }
194 // WSL2 mounts drives over 9p with `aname=drvfs;path=C:\`; WSL1 as `drvfs`.
195 const options = (fields[separator + 3] ?? "").split(/[,;]/);
196 const source = options.includes("aname=drvfs")
197 ? options.find((option) => option.startsWith("path="))?.slice("path=".length)
198 : fields[separator + 1] === "drvfs"
199 ? unescapeMountField(fields[separator + 2] ?? "")
200 : undefined;
201 const root = unescapeMountField(fields[3] ?? "/")
202 .split("/")
203 .filter(Boolean);
204 deepest = {
205 mountPoint,
206 windowsRoot:
207 source !== undefined && WINDOWS_ROOT.test(source)
208 ? [normalizeWindowsPath(source), ...root].join("\\")
209 : undefined,
210 };
211 }
212 return deepest?.windowsRoot === undefined
213 ? undefined
214 : { mountPoint: deepest.mountPoint, windowsRoot: deepest.windowsRoot };
215 }
216
217 /**
218 * drvfs keeps the casing a path was typed with, while Windows reports the
219 * on-disk names. Read those names rather than folding case, which would join
220 * distinct entries of a case-sensitive directory.
221 */
222 async function onDiskNames(
223 mountPoint: string,
224 parts: readonly string[],
225 ): Promise<string[] | undefined> {
226 const names: string[] = [];
227 let current = mountPoint;
228 for (const part of parts) {
229 let entries: string[];
230 try {
231 entries = await NodeFSP.readdir(current);
232 } catch {
233 return undefined;
234 }
235 const folded = part.toUpperCase();
236 const [name, ...others] = entries.includes(part)
237 ? [part]
238 : entries.filter((entry) => entry.toUpperCase() === folded);
239 if (name === undefined || others.length > 0) return undefined;
240 names.push(name);
241 current = NodePath.posix.join(current, name);
242 }
243 return names;
244 }
245
246 export function physicalSourceIdOf(scope: string, windowsPath: string): string {
247 return NodeCrypto.createHash("sha256")
248 .update(`${scope}\0${windowsPath}`)
249 .digest("hex")
250 .slice(0, 32);
251 }
252
253 /**
254 * Identifies a Windows directory the same way from the Windows server and from
255 * a WSL server reading it through drvfs. Directories that are not Windows
256 * backed, and servers the desktop app did not start, get none.
257 */
258 export async function readPhysicalSourceId(
259 dir: string,
260 scope: string | undefined,
261 platform: NodeJS.Platform,
262 mountInfoPath = "/proc/self/mountinfo",
263 ): Promise<string | undefined> {
264 if (scope === undefined || scope.length === 0) return undefined;
265 try {
266 if (platform === "win32") {
267 // The native realpath reports on-disk casing; the JS one keeps the input's.
268 return physicalSourceIdOf(scope, normalizeWindowsPath(await NodeFSP.realpath(dir)));
269 }
270 if (platform !== "linux") return undefined;
271 const mount = windowsMountFor(await NodeFSP.readFile(mountInfoPath, "utf8"), dir);
272 if (mount === undefined) return undefined;
273 const names = await onDiskNames(
274 mount.mountPoint,
275 dir.slice(mount.mountPoint.length).split("/").filter(Boolean),
276 );
277 return names === undefined
278 ? undefined
279 : physicalSourceIdOf(scope, [mount.windowsRoot, ...names].join("\\"));
280 } catch {
281 return undefined;
282 }
283 }
284
285 async function guardMatches(
286 handle: NodeFSP.FileHandle,
287 position: TranscriptParsePosition,
288 ): Promise<boolean> {
289 if (position.guardLength <= 0 || position.guardLength > GUARD_LENGTH) return false;
290 try {
291 const window = Buffer.alloc(position.guardLength);
292 const { bytesRead } = await handle.read(
293 window,
294 0,
295 position.guardLength,
296 position.resumeOffset - position.guardLength,
297 );
298 return bytesRead === position.guardLength && fnv1a(window) === position.guardHash;
299 } catch {
300 return false;
301 }
302 }
303
304 /** Plain files reuse one buffer, so a chunk is only valid until the next one is requested. */
305 async function* readChunks(
306 handle: NodeFSP.FileHandle,
307 start: number,
308 compressed: boolean,
309 ): AsyncGenerator<Buffer> {
310 if (compressed) {
311 // Compressed snapshots cache a full parse; decoded byte offsets cannot
312 // resume the compressed input.
313 yield* handle
314 .createReadStream({ start, autoClose: true, highWaterMark: READ_CHUNK_BYTES })
315 .compose(NodeZlib.createZstdDecompress());
316 return;
317 }
318 const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES);
319 for (let position = start; ;) {
320 const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
321 if (bytesRead === 0) return;
322 position += bytesRead;
323 yield buffer.subarray(0, bytesRead);
324 }
325 }
326
327 // Match ASCII markers as raw bytes: UTF-8 cannot false-match them mid-sequence, and no line string is allocated.
328 export function transcriptLineGate(markers: readonly [string, ...string[]]) {
329 const needles = markers.map((marker) => Buffer.from(marker));
330 return (line: Buffer) => needles.some((needle) => line.includes(needle));
331 }
332
333 /** Returns null on read failure; caching it as empty would hide usage until size/mtime changes. */
334 export async function readTranscriptRecords<State>(
335 filePath: string,
336 format: TranscriptUsageFormat<State>,
337 resumeFrom?: TranscriptParsePosition,
338 options?: { readonly streamingThresholdBytes?: number },
339 ): Promise<TranscriptParseResult | null> {
340 const streamingThresholdBytes = options?.streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES;
341 const compressed = filePath.endsWith(".jsonl.zst");
342 let handle: NodeFSP.FileHandle;
343 try {
344 handle = await NodeFSP.open(filePath, "r");
345 } catch {
346 return null;
347 }
348
349 try {
350 const codec = format.state;
351 const decodeState = codec === undefined ? undefined : Schema.decodeUnknownOption(codec.schema);
352 const encodeState = codec === undefined ? undefined : Schema.encodeSync(codec.schema);
353 let state = codec === undefined ? (undefined as State) : codec.initial();
354 let resumed = false;
355 let start = 0;
356 const savedState =
357 resumeFrom === undefined || decodeState === undefined || resumeFrom.state === null
358 ? Option.none()
359 : decodeState(resumeFrom.state);
360 if (
361 !compressed &&
362 resumeFrom !== undefined &&
363 resumeFrom.resumeOffset > 0 &&
364 (codec === undefined || Option.isSome(savedState)) &&
365 (await guardMatches(handle, resumeFrom))
366 ) {
367 if (Option.isSome(savedState)) state = savedState.value;
368 start = resumeFrom.resumeOffset;
369 resumed = true;
370 }
371
372 const toLineString = (lineBuffer: Buffer): string => {
373 const content =
374 lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN
375 ? lineBuffer.subarray(0, -1)
376 : lineBuffer;
377 return content.toString("utf8");
378 };
379
380 const mightCarryUsage = transcriptLineGate(format.usageMarkers);
381 const parseLine = (lineBuffer: Buffer, lineState: State, out: UsageRecord[]): void => {
382 if (!mightCarryUsage(lineBuffer)) return;
383 for (const record of format.parseLine(toLineString(lineBuffer), lineState)) out.push(record);
384 };
385
386 const records: UsageRecord[] = [];
387 let resumeOffset = start;
388 let scanOffset = start;
389 let pendingChunks: Buffer[] = [];
390 let pendingBytes = 0;
391 let streaming: ReturnType<typeof createTranscriptJsonReader> | undefined;
392 let decoder: NodeStringDecoder.StringDecoder | undefined;
393 const selectPath = selectUsageFields(format.selectFields);
394
395 const append = (segment: Buffer) => {
396 if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) {
397 // Copy: the next read overwrites the chunk this segment points into.
398 if (segment.length > 0) pendingChunks.push(Buffer.from(segment));
399 pendingBytes += segment.length;
400 return;
401 }
402 if (!streaming) {
403 // Usage has no import-history budget; retain selected fields regardless of surrounding tool payload size.
404 streaming = createTranscriptJsonReader(() => {}, selectPath, { maxDepth: Infinity });
405 decoder = new NodeStringDecoder.StringDecoder("utf8");
406 for (const pending of pendingChunks) streaming.write(decoder.write(pending));
407 pendingChunks = [];
408 pendingBytes = 0;
409 }
410 streaming.write(decoder!.write(segment));
411 };
412 const finish = (lineState: State, out: UsageRecord[]) => {
413 if (streaming) {
414 streaming.write(decoder!.end());
415 const projected = streaming.finish();
416 for (const record of format.parseProjected(projected, lineState)) out.push(record);
417 } else if (pendingBytes > 0) {
418 const line =
419 pendingChunks.length === 1
420 ? pendingChunks[0]!
421 : Buffer.concat(pendingChunks, pendingBytes);
422 parseLine(line, lineState, out);
423 }
424 pendingChunks = [];
425 pendingBytes = 0;
426 streaming = undefined;
427 decoder = undefined;
428 };
429 for await (const chunk of readChunks(handle, start, compressed)) {
430 let lineStart = 0;
431 while (lineStart < chunk.length) {
432 const newlineIndex = chunk.indexOf(NEWLINE, lineStart);
433 if (newlineIndex === -1) {
434 append(chunk.subarray(lineStart));
435 break;
436 }
437 if (!streaming && pendingBytes === 0) {
438 parseLine(chunk.subarray(lineStart, newlineIndex), state, records);
439 } else {
440 append(chunk.subarray(lineStart, newlineIndex));
441 finish(state, records);
442 }
443 lineStart = newlineIndex + 1;
444 resumeOffset = scanOffset + lineStart;
445 }
446 scanOffset += chunk.length;
447 }
448
449 // Parse unfinished tails against copied state so the next scan can replay from resumeOffset.
450 const encodedState = encodeState === undefined ? null : encodeState(state);
451 const tailRecords: UsageRecord[] = [];
452 const tailState =
453 decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState));
454 finish(tailState, tailRecords);
455
456 const guardLength = compressed ? 0 : Math.min(GUARD_LENGTH, resumeOffset);
457 let guardHash = 0;
458 if (guardLength > 0) {
459 const window = Buffer.alloc(guardLength);
460 await handle.read(window, 0, guardLength, resumeOffset - guardLength);
461 guardHash = fnv1a(window);
462 }
463
464 return {
465 records,
466 tailRecords,
467 malformedRecords: format.readMalformedRecords?.(tailState) ?? 0,
468 position: {
469 resumeOffset: compressed ? 0 : resumeOffset,
470 guardLength,
471 guardHash,
472 state: encodedState,
473 },
474 resumed,
475 };
476 } catch {
477 return null;
478 } finally {
479 await handle.close().catch(() => undefined);
480 }
481 }