The best way to control Claude Code, Codex and more from any computer or phone
| 1 | // @effect-diagnostics nodeBuiltinImport:off |
| 2 | // Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files. |
| 3 | import * as NodeCrypto from "node:crypto"; |
| 4 | import * as NodeFS from "node:fs"; |
| 5 | import * as NodeFSP from "node:fs/promises"; |
| 6 | import * as NodePath from "node:path"; |
| 7 | import * as NodeStringDecoder from "node:string_decoder"; |
| 8 | import * as NodeUtil from "node:util"; |
| 9 | import * as NodeZlib from "node:zlib"; |
| 10 | |
| 11 | import type { |
| 12 | SelectedFields, |
| 13 | TranscriptUsageFormat, |
| 14 | UsageRecord, |
| 15 | } from "@t3tools/provider-core/server/usage"; |
| 16 | import * as Option from "effect/Option"; |
| 17 | import * as Schema from "effect/Schema"; |
| 18 | |
| 19 | import { createTranscriptJsonReader } from "../project/AgentSessionJson.ts"; |
| 20 | |
| 21 | export interface TranscriptFile { |
| 22 | readonly path: string; |
| 23 | readonly size: number; |
| 24 | readonly mtimeMs: number; |
| 25 | } |
| 26 | |
| 27 | /** Resume only if bytes before resumeOffset match the guard hash; rewritten files would corrupt totals. */ |
| 28 | export interface TranscriptParsePosition { |
| 29 | readonly resumeOffset: number; |
| 30 | readonly guardLength: number; |
| 31 | readonly guardHash: number; |
| 32 | readonly state: unknown; |
| 33 | } |
| 34 | |
| 35 | export interface TranscriptParseResult { |
| 36 | readonly records: readonly UsageRecord[]; |
| 37 | /** Unterminated tail records stay outside records/position so the next scan rereads them. */ |
| 38 | readonly tailRecords: readonly UsageRecord[]; |
| 39 | /** Includes tentative tail diagnostics; position.state only includes committed lines. */ |
| 40 | readonly malformedRecords: number; |
| 41 | readonly position: TranscriptParsePosition; |
| 42 | readonly resumed: boolean; |
| 43 | } |
| 44 | |
| 45 | export const GUARD_LENGTH = 64; |
| 46 | const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024; |
| 47 | const READ_CHUNK_BYTES = 256 * 1024; |
| 48 | const NEWLINE = 0x0a; |
| 49 | const STAT_CONCURRENCY = 32; |
| 50 | const WALK_CONCURRENCY = 16; |
| 51 | // Measured about 3x faster than the promise API's stat across 20,000 transcripts. |
| 52 | const statFile = NodeUtil.promisify(NodeFS.stat); |
| 53 | const CARRIAGE_RETURN = 0x0d; |
| 54 | |
| 55 | function selectUsageFields(fields: SelectedFields) { |
| 56 | return (path: ReadonlyArray<string | number | null>): boolean => { |
| 57 | let selected: true | SelectedFields = fields; |
| 58 | for (const key of path) { |
| 59 | if (selected === true) return true; |
| 60 | if (typeof key !== "string" || !Object.hasOwn(selected, key)) return false; |
| 61 | selected = selected[key]!; |
| 62 | } |
| 63 | return true; |
| 64 | }; |
| 65 | } |
| 66 | |
| 67 | function fnv1a(buffer: Buffer): number { |
| 68 | let hash = 0x811c9dc5; |
| 69 | for (let index = 0; index < buffer.length; index += 1) { |
| 70 | hash ^= buffer[index]!; |
| 71 | hash = Math.imul(hash, 0x01000193); |
| 72 | } |
| 73 | return hash >>> 0; |
| 74 | } |
| 75 | |
| 76 | /** Skips Grok logs without usage; preserves readdir order for first-seen deduplication. */ |
| 77 | export async function listTranscriptFiles( |
| 78 | root: string, |
| 79 | sinceMs: number, |
| 80 | options?: { readonly fileName?: string }, |
| 81 | ): Promise<{ |
| 82 | readonly files: readonly TranscriptFile[]; |
| 83 | readonly walked: readonly string[]; |
| 84 | readonly failedPaths: number; |
| 85 | }> { |
| 86 | const fileName = options?.fileName; |
| 87 | let failedPaths = 0; |
| 88 | let reading = 0; |
| 89 | const waiting: Array<() => void> = []; |
| 90 | // Sibling directories read together; each returns its own list so the result keeps readdir order. |
| 91 | const walk = async (dir: string): Promise<readonly string[]> => { |
| 92 | if (reading < WALK_CONCURRENCY) reading += 1; |
| 93 | else await new Promise<void>((resume) => waiting.push(resume)); |
| 94 | let entries; |
| 95 | try { |
| 96 | entries = await NodeFSP.readdir(dir, { withFileTypes: true }); |
| 97 | } catch { |
| 98 | failedPaths += 1; |
| 99 | return []; |
| 100 | } finally { |
| 101 | // A waiting directory takes over this slot, so the count only drops when none waits. |
| 102 | const next = waiting.shift(); |
| 103 | if (next === undefined) reading -= 1; |
| 104 | else next(); |
| 105 | } |
| 106 | const found = await Promise.all( |
| 107 | entries.map((entry) => { |
| 108 | const child = NodePath.join(dir, entry.name); |
| 109 | if (entry.isDirectory()) return walk(child); |
| 110 | return ( |
| 111 | fileName !== undefined |
| 112 | ? entry.name === fileName |
| 113 | : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst") |
| 114 | ) |
| 115 | ? [child] |
| 116 | : []; |
| 117 | }), |
| 118 | ); |
| 119 | return found.flat(); |
| 120 | }; |
| 121 | const candidates = await walk(root); |
| 122 | |
| 123 | const found: Array<TranscriptFile | undefined> = Array.from({ length: candidates.length }); |
| 124 | const queue = candidates.entries(); |
| 125 | const statQueued = async (): Promise<void> => { |
| 126 | for (const [index, path] of queue) { |
| 127 | try { |
| 128 | const stats = await statFile(path); |
| 129 | if (stats.mtimeMs >= sinceMs) { |
| 130 | found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs }; |
| 131 | } |
| 132 | } catch (error) { |
| 133 | if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) { |
| 134 | failedPaths += 1; |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | }; |
| 139 | await Promise.all( |
| 140 | Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued), |
| 141 | ); |
| 142 | return { files: found.filter((file) => file !== undefined), walked: candidates, failedPaths }; |
| 143 | } |
| 144 | |
| 145 | export async function readDirectoryVolumeId(path: string): Promise<string> { |
| 146 | try { |
| 147 | const stats = await NodeFSP.stat(path); |
| 148 | return `${stats.dev}:${stats.ino}`; |
| 149 | } catch { |
| 150 | return ""; |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * Set by the desktop app, from a value it persists, on its Windows server and |
| 156 | * the WSL server it manages, so both can name one Windows directory alike |
| 157 | * without merging separate machines that share a hostname and path. |
| 158 | */ |
| 159 | export const DESKTOP_HOST_SCOPE_ENV = "T3CODE_DESKTOP_HOST_SCOPE"; |
| 160 | |
| 161 | const MOUNT_FIELD_ESCAPE = /\\([0-7]{3})/g; |
| 162 | const WINDOWS_ROOT = /^(?:[A-Za-z]:|\\\\[^\\]+\\[^\\]+)\\?$/; |
| 163 | |
| 164 | const unescapeMountField = (field: string) => |
| 165 | field.replace(MOUNT_FIELD_ESCAPE, (_, octal: string) => |
| 166 | String.fromCharCode(Number.parseInt(octal, 8)), |
| 167 | ); |
| 168 | |
| 169 | const normalizeWindowsPath = (path: string) => |
| 170 | (/^[a-z]:/.test(path) ? path[0]!.toUpperCase() + path.slice(1) : path).replace(/\\$/, ""); |
| 171 | |
| 172 | /** |
| 173 | * The Windows directory behind the deepest mount holding `dir`, read from |
| 174 | * `/proc/self/mountinfo` so a custom automount root resolves and an ext4 |
| 175 | * mount under `/mnt` never passes for a drive. |
| 176 | */ |
| 177 | function windowsMountFor( |
| 178 | mountInfo: string, |
| 179 | dir: string, |
| 180 | ): { readonly mountPoint: string; readonly windowsRoot: string } | undefined { |
| 181 | let deepest: { mountPoint: string; windowsRoot: string | undefined } | undefined; |
| 182 | for (const line of mountInfo.split("\n")) { |
| 183 | const fields = line.split(" "); |
| 184 | const separator = fields.indexOf("-", 6); |
| 185 | if (separator === -1) continue; |
| 186 | const mountPoint = unescapeMountField(fields[4] ?? ""); |
| 187 | const contains = |
| 188 | mountPoint === "/" || |
| 189 | dir === mountPoint || |
| 190 | dir.startsWith(mountPoint.endsWith("/") ? mountPoint : `${mountPoint}/`); |
| 191 | if (!contains || (deepest !== undefined && deepest.mountPoint.length > mountPoint.length)) { |
| 192 | continue; |
| 193 | } |
| 194 | // WSL2 mounts drives over 9p with `aname=drvfs;path=C:\`; WSL1 as `drvfs`. |
| 195 | const options = (fields[separator + 3] ?? "").split(/[,;]/); |
| 196 | const source = options.includes("aname=drvfs") |
| 197 | ? options.find((option) => option.startsWith("path="))?.slice("path=".length) |
| 198 | : fields[separator + 1] === "drvfs" |
| 199 | ? unescapeMountField(fields[separator + 2] ?? "") |
| 200 | : undefined; |
| 201 | const root = unescapeMountField(fields[3] ?? "/") |
| 202 | .split("/") |
| 203 | .filter(Boolean); |
| 204 | deepest = { |
| 205 | mountPoint, |
| 206 | windowsRoot: |
| 207 | source !== undefined && WINDOWS_ROOT.test(source) |
| 208 | ? [normalizeWindowsPath(source), ...root].join("\\") |
| 209 | : undefined, |
| 210 | }; |
| 211 | } |
| 212 | return deepest?.windowsRoot === undefined |
| 213 | ? undefined |
| 214 | : { mountPoint: deepest.mountPoint, windowsRoot: deepest.windowsRoot }; |
| 215 | } |
| 216 | |
| 217 | /** |
| 218 | * drvfs keeps the casing a path was typed with, while Windows reports the |
| 219 | * on-disk names. Read those names rather than folding case, which would join |
| 220 | * distinct entries of a case-sensitive directory. |
| 221 | */ |
| 222 | async function onDiskNames( |
| 223 | mountPoint: string, |
| 224 | parts: readonly string[], |
| 225 | ): Promise<string[] | undefined> { |
| 226 | const names: string[] = []; |
| 227 | let current = mountPoint; |
| 228 | for (const part of parts) { |
| 229 | let entries: string[]; |
| 230 | try { |
| 231 | entries = await NodeFSP.readdir(current); |
| 232 | } catch { |
| 233 | return undefined; |
| 234 | } |
| 235 | const folded = part.toUpperCase(); |
| 236 | const [name, ...others] = entries.includes(part) |
| 237 | ? [part] |
| 238 | : entries.filter((entry) => entry.toUpperCase() === folded); |
| 239 | if (name === undefined || others.length > 0) return undefined; |
| 240 | names.push(name); |
| 241 | current = NodePath.posix.join(current, name); |
| 242 | } |
| 243 | return names; |
| 244 | } |
| 245 | |
| 246 | export function physicalSourceIdOf(scope: string, windowsPath: string): string { |
| 247 | return NodeCrypto.createHash("sha256") |
| 248 | .update(`${scope}\0${windowsPath}`) |
| 249 | .digest("hex") |
| 250 | .slice(0, 32); |
| 251 | } |
| 252 | |
| 253 | /** |
| 254 | * Identifies a Windows directory the same way from the Windows server and from |
| 255 | * a WSL server reading it through drvfs. Directories that are not Windows |
| 256 | * backed, and servers the desktop app did not start, get none. |
| 257 | */ |
| 258 | export async function readPhysicalSourceId( |
| 259 | dir: string, |
| 260 | scope: string | undefined, |
| 261 | platform: NodeJS.Platform, |
| 262 | mountInfoPath = "/proc/self/mountinfo", |
| 263 | ): Promise<string | undefined> { |
| 264 | if (scope === undefined || scope.length === 0) return undefined; |
| 265 | try { |
| 266 | if (platform === "win32") { |
| 267 | // The native realpath reports on-disk casing; the JS one keeps the input's. |
| 268 | return physicalSourceIdOf(scope, normalizeWindowsPath(await NodeFSP.realpath(dir))); |
| 269 | } |
| 270 | if (platform !== "linux") return undefined; |
| 271 | const mount = windowsMountFor(await NodeFSP.readFile(mountInfoPath, "utf8"), dir); |
| 272 | if (mount === undefined) return undefined; |
| 273 | const names = await onDiskNames( |
| 274 | mount.mountPoint, |
| 275 | dir.slice(mount.mountPoint.length).split("/").filter(Boolean), |
| 276 | ); |
| 277 | return names === undefined |
| 278 | ? undefined |
| 279 | : physicalSourceIdOf(scope, [mount.windowsRoot, ...names].join("\\")); |
| 280 | } catch { |
| 281 | return undefined; |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | async function guardMatches( |
| 286 | handle: NodeFSP.FileHandle, |
| 287 | position: TranscriptParsePosition, |
| 288 | ): Promise<boolean> { |
| 289 | if (position.guardLength <= 0 || position.guardLength > GUARD_LENGTH) return false; |
| 290 | try { |
| 291 | const window = Buffer.alloc(position.guardLength); |
| 292 | const { bytesRead } = await handle.read( |
| 293 | window, |
| 294 | 0, |
| 295 | position.guardLength, |
| 296 | position.resumeOffset - position.guardLength, |
| 297 | ); |
| 298 | return bytesRead === position.guardLength && fnv1a(window) === position.guardHash; |
| 299 | } catch { |
| 300 | return false; |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | /** Plain files reuse one buffer, so a chunk is only valid until the next one is requested. */ |
| 305 | async function* readChunks( |
| 306 | handle: NodeFSP.FileHandle, |
| 307 | start: number, |
| 308 | compressed: boolean, |
| 309 | ): AsyncGenerator<Buffer> { |
| 310 | if (compressed) { |
| 311 | // Compressed snapshots cache a full parse; decoded byte offsets cannot |
| 312 | // resume the compressed input. |
| 313 | yield* handle |
| 314 | .createReadStream({ start, autoClose: true, highWaterMark: READ_CHUNK_BYTES }) |
| 315 | .compose(NodeZlib.createZstdDecompress()); |
| 316 | return; |
| 317 | } |
| 318 | const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); |
| 319 | for (let position = start; ;) { |
| 320 | const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); |
| 321 | if (bytesRead === 0) return; |
| 322 | position += bytesRead; |
| 323 | yield buffer.subarray(0, bytesRead); |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | // Match ASCII markers as raw bytes: UTF-8 cannot false-match them mid-sequence, and no line string is allocated. |
| 328 | export function transcriptLineGate(markers: readonly [string, ...string[]]) { |
| 329 | const needles = markers.map((marker) => Buffer.from(marker)); |
| 330 | return (line: Buffer) => needles.some((needle) => line.includes(needle)); |
| 331 | } |
| 332 | |
| 333 | /** Returns null on read failure; caching it as empty would hide usage until size/mtime changes. */ |
| 334 | export async function readTranscriptRecords<State>( |
| 335 | filePath: string, |
| 336 | format: TranscriptUsageFormat<State>, |
| 337 | resumeFrom?: TranscriptParsePosition, |
| 338 | options?: { readonly streamingThresholdBytes?: number }, |
| 339 | ): Promise<TranscriptParseResult | null> { |
| 340 | const streamingThresholdBytes = options?.streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES; |
| 341 | const compressed = filePath.endsWith(".jsonl.zst"); |
| 342 | let handle: NodeFSP.FileHandle; |
| 343 | try { |
| 344 | handle = await NodeFSP.open(filePath, "r"); |
| 345 | } catch { |
| 346 | return null; |
| 347 | } |
| 348 | |
| 349 | try { |
| 350 | const codec = format.state; |
| 351 | const decodeState = codec === undefined ? undefined : Schema.decodeUnknownOption(codec.schema); |
| 352 | const encodeState = codec === undefined ? undefined : Schema.encodeSync(codec.schema); |
| 353 | let state = codec === undefined ? (undefined as State) : codec.initial(); |
| 354 | let resumed = false; |
| 355 | let start = 0; |
| 356 | const savedState = |
| 357 | resumeFrom === undefined || decodeState === undefined || resumeFrom.state === null |
| 358 | ? Option.none() |
| 359 | : decodeState(resumeFrom.state); |
| 360 | if ( |
| 361 | !compressed && |
| 362 | resumeFrom !== undefined && |
| 363 | resumeFrom.resumeOffset > 0 && |
| 364 | (codec === undefined || Option.isSome(savedState)) && |
| 365 | (await guardMatches(handle, resumeFrom)) |
| 366 | ) { |
| 367 | if (Option.isSome(savedState)) state = savedState.value; |
| 368 | start = resumeFrom.resumeOffset; |
| 369 | resumed = true; |
| 370 | } |
| 371 | |
| 372 | const toLineString = (lineBuffer: Buffer): string => { |
| 373 | const content = |
| 374 | lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN |
| 375 | ? lineBuffer.subarray(0, -1) |
| 376 | : lineBuffer; |
| 377 | return content.toString("utf8"); |
| 378 | }; |
| 379 | |
| 380 | const mightCarryUsage = transcriptLineGate(format.usageMarkers); |
| 381 | const parseLine = (lineBuffer: Buffer, lineState: State, out: UsageRecord[]): void => { |
| 382 | if (!mightCarryUsage(lineBuffer)) return; |
| 383 | for (const record of format.parseLine(toLineString(lineBuffer), lineState)) out.push(record); |
| 384 | }; |
| 385 | |
| 386 | const records: UsageRecord[] = []; |
| 387 | let resumeOffset = start; |
| 388 | let scanOffset = start; |
| 389 | let pendingChunks: Buffer[] = []; |
| 390 | let pendingBytes = 0; |
| 391 | let streaming: ReturnType<typeof createTranscriptJsonReader> | undefined; |
| 392 | let decoder: NodeStringDecoder.StringDecoder | undefined; |
| 393 | const selectPath = selectUsageFields(format.selectFields); |
| 394 | |
| 395 | const append = (segment: Buffer) => { |
| 396 | if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) { |
| 397 | // Copy: the next read overwrites the chunk this segment points into. |
| 398 | if (segment.length > 0) pendingChunks.push(Buffer.from(segment)); |
| 399 | pendingBytes += segment.length; |
| 400 | return; |
| 401 | } |
| 402 | if (!streaming) { |
| 403 | // Usage has no import-history budget; retain selected fields regardless of surrounding tool payload size. |
| 404 | streaming = createTranscriptJsonReader(() => {}, selectPath, { maxDepth: Infinity }); |
| 405 | decoder = new NodeStringDecoder.StringDecoder("utf8"); |
| 406 | for (const pending of pendingChunks) streaming.write(decoder.write(pending)); |
| 407 | pendingChunks = []; |
| 408 | pendingBytes = 0; |
| 409 | } |
| 410 | streaming.write(decoder!.write(segment)); |
| 411 | }; |
| 412 | const finish = (lineState: State, out: UsageRecord[]) => { |
| 413 | if (streaming) { |
| 414 | streaming.write(decoder!.end()); |
| 415 | const projected = streaming.finish(); |
| 416 | for (const record of format.parseProjected(projected, lineState)) out.push(record); |
| 417 | } else if (pendingBytes > 0) { |
| 418 | const line = |
| 419 | pendingChunks.length === 1 |
| 420 | ? pendingChunks[0]! |
| 421 | : Buffer.concat(pendingChunks, pendingBytes); |
| 422 | parseLine(line, lineState, out); |
| 423 | } |
| 424 | pendingChunks = []; |
| 425 | pendingBytes = 0; |
| 426 | streaming = undefined; |
| 427 | decoder = undefined; |
| 428 | }; |
| 429 | for await (const chunk of readChunks(handle, start, compressed)) { |
| 430 | let lineStart = 0; |
| 431 | while (lineStart < chunk.length) { |
| 432 | const newlineIndex = chunk.indexOf(NEWLINE, lineStart); |
| 433 | if (newlineIndex === -1) { |
| 434 | append(chunk.subarray(lineStart)); |
| 435 | break; |
| 436 | } |
| 437 | if (!streaming && pendingBytes === 0) { |
| 438 | parseLine(chunk.subarray(lineStart, newlineIndex), state, records); |
| 439 | } else { |
| 440 | append(chunk.subarray(lineStart, newlineIndex)); |
| 441 | finish(state, records); |
| 442 | } |
| 443 | lineStart = newlineIndex + 1; |
| 444 | resumeOffset = scanOffset + lineStart; |
| 445 | } |
| 446 | scanOffset += chunk.length; |
| 447 | } |
| 448 | |
| 449 | // Parse unfinished tails against copied state so the next scan can replay from resumeOffset. |
| 450 | const encodedState = encodeState === undefined ? null : encodeState(state); |
| 451 | const tailRecords: UsageRecord[] = []; |
| 452 | const tailState = |
| 453 | decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState)); |
| 454 | finish(tailState, tailRecords); |
| 455 | |
| 456 | const guardLength = compressed ? 0 : Math.min(GUARD_LENGTH, resumeOffset); |
| 457 | let guardHash = 0; |
| 458 | if (guardLength > 0) { |
| 459 | const window = Buffer.alloc(guardLength); |
| 460 | await handle.read(window, 0, guardLength, resumeOffset - guardLength); |
| 461 | guardHash = fnv1a(window); |
| 462 | } |
| 463 | |
| 464 | return { |
| 465 | records, |
| 466 | tailRecords, |
| 467 | malformedRecords: format.readMalformedRecords?.(tailState) ?? 0, |
| 468 | position: { |
| 469 | resumeOffset: compressed ? 0 : resumeOffset, |
| 470 | guardLength, |
| 471 | guardHash, |
| 472 | state: encodedState, |
| 473 | }, |
| 474 | resumed, |
| 475 | }; |
| 476 | } catch { |
| 477 | return null; |
| 478 | } finally { |
| 479 | await handle.close().catch(() => undefined); |
| 480 | } |
| 481 | } |