The best way to control Claude Code, Codex and more from any computer or phone

main
ts 472 lines 15 KB
Raw
1 import type {
2 ServerTraceDiagnosticsErrorKind,
3 ServerTraceDiagnosticsFailureSummary,
4 ServerTraceDiagnosticsLogEvent,
5 ServerTraceDiagnosticsRecentFailure,
6 ServerTraceDiagnosticsResult,
7 ServerTraceDiagnosticsSpanOccurrence,
8 ServerTraceDiagnosticsSpanSummary,
9 } from "@t3tools/contracts";
10 import * as Context from "effect/Context";
11 import * as DateTime from "effect/DateTime";
12 import * as Effect from "effect/Effect";
13 import * as FileSystem from "effect/FileSystem";
14 import * as Layer from "effect/Layer";
15 import * as Option from "effect/Option";
16 import * as PlatformError from "effect/PlatformError";
17 import * as Result from "effect/Result";
18 import * as Schema from "effect/Schema";
19 import * as Stream from "effect/Stream";
20
21 interface TraceEventLike {
22 readonly name?: unknown;
23 readonly timeUnixNano?: unknown;
24 readonly attributes?: unknown;
25 }
26
27 export interface TraceDiagnosticsOptions {
28 readonly traceFilePath: string;
29 readonly maxFiles: number;
30 readonly slowSpanThresholdMs?: number;
31 readonly readAt?: DateTime.Utc;
32 }
33
34 export class TraceFileReadError extends Schema.TaggedError<TraceFileReadError>()(
35 "TraceFileReadError",
36 {
37 traceFilePath: Schema.String,
38 causeTag: Schema.String,
39 cause: Schema.Defect(),
40 },
41 ) {
42 override get message(): string {
43 return `Failed to read local trace file '${this.traceFilePath}'.`;
44 }
45 }
46
47 export class TraceDiagnostics extends Context.Service<
48 TraceDiagnostics,
49 {
50 readonly read: (
51 options: TraceDiagnosticsOptions,
52 ) => Effect.Effect<ServerTraceDiagnosticsResult>;
53 }
54 >()("t3/diagnostics/TraceDiagnostics") {}
55
56 interface TraceDiagnosticsErrorSummary {
57 readonly kind: ServerTraceDiagnosticsErrorKind;
58 readonly message: string;
59 }
60
61 const DEFAULT_SLOW_SPAN_THRESHOLD_MS = 1_000;
62 const TOP_LIMIT = 10;
63 const RECENT_LIMIT = 20;
64
65 export function toRotatedTracePaths(
66 traceFilePath: string,
67 maxFiles: number,
68 ): ReadonlyArray<string> {
69 const backupCount = Math.max(0, Math.floor(maxFiles));
70 const backups = Array.from(
71 { length: backupCount },
72 (_, index) => `${traceFilePath}.${backupCount - index}`,
73 );
74 return [...backups, traceFilePath];
75 }
76
77 function isRecordObject(value: unknown): value is Readonly<Record<string, unknown>> {
78 return typeof value === "object" && value !== null;
79 }
80
81 function toStringValue(value: unknown): string | null {
82 return typeof value === "string" && value.trim().length > 0 ? value : null;
83 }
84
85 function toNumberValue(value: unknown): number | null {
86 return typeof value === "number" && Number.isFinite(value) ? value : null;
87 }
88
89 function unixNanoToDateTime(value: unknown): DateTime.Utc | null {
90 const text = toStringValue(value);
91 if (!text) return null;
92 try {
93 const millis = Number(BigInt(text) / 1_000_000n);
94 return Option.getOrNull(DateTime.make(millis));
95 } catch {
96 return null;
97 }
98 }
99
100 function readExitTag(exit: unknown): string | null {
101 if (!isRecordObject(exit) || !("_tag" in exit)) return null;
102 return toStringValue(exit._tag);
103 }
104
105 function readExitCause(exit: unknown): string {
106 if (!isRecordObject(exit) || !("cause" in exit)) return "Failure";
107 return toStringValue(exit.cause)?.trim() ?? "Failure";
108 }
109
110 function isTraceEvent(value: unknown): value is TraceEventLike {
111 return typeof value === "object" && value !== null;
112 }
113
114 function readEventAttributes(event: TraceEventLike): Readonly<Record<string, unknown>> {
115 return typeof event.attributes === "object" && event.attributes !== null
116 ? (event.attributes as Readonly<Record<string, unknown>>)
117 : {};
118 }
119
120 function makeEmptyDiagnostics(input: {
121 readonly traceFilePath: string;
122 readonly scannedFilePaths: ReadonlyArray<string>;
123 readonly readAt: DateTime.Utc;
124 readonly slowSpanThresholdMs: number;
125 readonly error?: TraceDiagnosticsErrorSummary;
126 readonly partialFailure?: boolean;
127 }): ServerTraceDiagnosticsResult {
128 return {
129 traceFilePath: input.traceFilePath,
130 scannedFilePaths: [...input.scannedFilePaths],
131 readAt: input.readAt,
132 recordCount: 0,
133 parseErrorCount: 0,
134 firstSpanAt: Option.none(),
135 lastSpanAt: Option.none(),
136 failureCount: 0,
137 interruptionCount: 0,
138 slowSpanThresholdMs: input.slowSpanThresholdMs,
139 slowSpanCount: 0,
140 logLevelCounts: {},
141 topSpansByCount: [],
142 slowestSpans: [],
143 commonFailures: [],
144 latestFailures: [],
145 latestWarningAndErrorLogs: [],
146 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
147 error: Option.fromNullishOr(input.error),
148 };
149 }
150
151 function isNotFoundError(error: PlatformError.PlatformError): boolean {
152 return error.reason._tag === "NotFound";
153 }
154
155 function insertBounded<A>(
156 items: A[],
157 item: A,
158 limit: number,
159 order: (left: A, right: A) => number,
160 ): void {
161 if (items.length >= limit && order(item, items[items.length - 1]!) >= 0) {
162 return;
163 }
164
165 items.push(item);
166 items.sort(order);
167 if (items.length > limit) {
168 items.length = limit;
169 }
170 }
171
172 const slowestFirst = (
173 left: ServerTraceDiagnosticsSpanOccurrence,
174 right: ServerTraceDiagnosticsSpanOccurrence,
175 ) => right.durationMs - left.durationMs;
176
177 const latestEndedFirst = (
178 left: ServerTraceDiagnosticsRecentFailure,
179 right: ServerTraceDiagnosticsRecentFailure,
180 ) => DateTime.toEpochMillis(right.endedAt) - DateTime.toEpochMillis(left.endedAt);
181
182 const latestSeenFirst = (
183 left: ServerTraceDiagnosticsLogEvent,
184 right: ServerTraceDiagnosticsLogEvent,
185 ) => DateTime.toEpochMillis(right.seenAt) - DateTime.toEpochMillis(left.seenAt);
186
187 export function makeTraceDiagnosticsAggregator(
188 slowSpanThresholdMs = DEFAULT_SLOW_SPAN_THRESHOLD_MS,
189 ) {
190 let parseErrorCount = 0;
191 let recordCount = 0;
192 let failureCount = 0;
193 let interruptionCount = 0;
194 let slowSpanCount = 0;
195 let firstSpanAt: DateTime.Utc | null = null;
196 let lastSpanAt: DateTime.Utc | null = null;
197
198 const spansByName = new Map<
199 string,
200 { count: number; failureCount: number; totalDurationMs: number; maxDurationMs: number }
201 >();
202 const failuresByKey = new Map<string, ServerTraceDiagnosticsFailureSummary>();
203 const latestFailures: ServerTraceDiagnosticsRecentFailure[] = [];
204 const slowestSpans: ServerTraceDiagnosticsSpanOccurrence[] = [];
205 const latestWarningAndErrorLogs: ServerTraceDiagnosticsLogEvent[] = [];
206 const logLevelCounts: Record<string, number> = {};
207
208 const addLine = (line: string) => {
209 if (line.trim().length === 0) return;
210
211 let parsed: unknown;
212 try {
213 parsed = JSON.parse(line);
214 } catch {
215 parseErrorCount += 1;
216 return;
217 }
218
219 if (!isRecordObject(parsed)) {
220 parseErrorCount += 1;
221 return;
222 }
223
224 const name = toStringValue(parsed.name);
225 const traceId = toStringValue(parsed.traceId);
226 const spanId = toStringValue(parsed.spanId);
227 const durationMs = toNumberValue(parsed.durationMs);
228 const endedAt = unixNanoToDateTime(parsed.endTimeUnixNano);
229 const startedAt = unixNanoToDateTime(parsed.startTimeUnixNano);
230
231 if (!name || !traceId || !spanId || durationMs === null || !endedAt) {
232 parseErrorCount += 1;
233 return;
234 }
235
236 recordCount += 1;
237 firstSpanAt =
238 startedAt && (firstSpanAt === null || DateTime.isLessThan(startedAt, firstSpanAt))
239 ? startedAt
240 : firstSpanAt;
241 lastSpanAt =
242 lastSpanAt === null || DateTime.isGreaterThan(endedAt, lastSpanAt) ? endedAt : lastSpanAt;
243
244 const exitTag = readExitTag(parsed.exit);
245 const status = isRecordObject(parsed.status) ? parsed.status : {};
246 const attributes = isRecordObject(parsed.attributes) ? parsed.attributes : {};
247 const isFailure = exitTag === "Failure" || status.code === "2";
248 const isInterrupted =
249 exitTag === "Interrupted" ||
250 attributes["effect.fiber.interrupted"] === true ||
251 status.message === "Interrupted";
252 if (isFailure) failureCount += 1;
253 if (isInterrupted) interruptionCount += 1;
254
255 const spanSummary = spansByName.get(name) ?? {
256 count: 0,
257 failureCount: 0,
258 totalDurationMs: 0,
259 maxDurationMs: 0,
260 };
261 spanSummary.count += 1;
262 spanSummary.totalDurationMs += durationMs;
263 spanSummary.maxDurationMs = Math.max(spanSummary.maxDurationMs, durationMs);
264 if (isFailure) spanSummary.failureCount += 1;
265 spansByName.set(name, spanSummary);
266
267 const spanItem = { name, durationMs, endedAt, traceId, spanId };
268 if (durationMs >= slowSpanThresholdMs) {
269 slowSpanCount += 1;
270 }
271 insertBounded(slowestSpans, spanItem, TOP_LIMIT, slowestFirst);
272
273 if (isFailure) {
274 const cause =
275 exitTag === "Failure"
276 ? readExitCause(parsed.exit)
277 : (toStringValue(status.message)?.trim() ?? "Failure");
278 insertBounded(latestFailures, { ...spanItem, cause }, RECENT_LIMIT, latestEndedFirst);
279
280 const failureKey = `${name}\0${cause}`;
281 const existing = failuresByKey.get(failureKey);
282 const isLatestFailure = !existing || DateTime.isGreaterThan(endedAt, existing.lastSeenAt);
283 failuresByKey.set(failureKey, {
284 name,
285 cause,
286 count: (existing?.count ?? 0) + 1,
287 lastSeenAt: isLatestFailure ? endedAt : existing!.lastSeenAt,
288 traceId: isLatestFailure ? traceId : existing!.traceId,
289 spanId: isLatestFailure ? spanId : existing!.spanId,
290 });
291 }
292
293 if (Array.isArray(parsed.events)) {
294 for (const rawEvent of parsed.events) {
295 if (!isTraceEvent(rawEvent)) continue;
296 const attributes = readEventAttributes(rawEvent);
297 const level = toStringValue(attributes["effect.logLevel"]);
298 if (!level) continue;
299
300 logLevelCounts[level] = (logLevelCounts[level] ?? 0) + 1;
301 const normalizedLevel = level.toLowerCase();
302 if (
303 normalizedLevel !== "warning" &&
304 normalizedLevel !== "warn" &&
305 normalizedLevel !== "error" &&
306 normalizedLevel !== "fatal"
307 ) {
308 continue;
309 }
310
311 const seenAt = unixNanoToDateTime(rawEvent.timeUnixNano) ?? endedAt;
312 const message = toStringValue(rawEvent.name)?.trim() ?? "Log event";
313 insertBounded(
314 latestWarningAndErrorLogs,
315 { spanName: name, level, message, seenAt, traceId, spanId },
316 RECENT_LIMIT,
317 latestSeenFirst,
318 );
319 }
320 }
321 };
322
323 const finish = (input: {
324 readonly traceFilePath: string;
325 readonly scannedFilePaths: ReadonlyArray<string>;
326 readonly readAt: DateTime.Utc;
327 readonly error?: TraceDiagnosticsErrorSummary;
328 readonly partialFailure?: boolean;
329 }): ServerTraceDiagnosticsResult => {
330 const topSpansByCount: ServerTraceDiagnosticsSpanSummary[] = [...spansByName.entries()]
331 .map(([name, span]) => ({
332 name,
333 count: span.count,
334 failureCount: span.failureCount,
335 totalDurationMs: span.totalDurationMs,
336 averageDurationMs: span.count > 0 ? span.totalDurationMs / span.count : 0,
337 maxDurationMs: span.maxDurationMs,
338 }))
339 .toSorted(
340 (left, right) => right.count - left.count || right.maxDurationMs - left.maxDurationMs,
341 )
342 .slice(0, TOP_LIMIT);
343
344 return {
345 traceFilePath: input.traceFilePath,
346 scannedFilePaths: input.scannedFilePaths,
347 readAt: input.readAt,
348 recordCount,
349 parseErrorCount,
350 firstSpanAt: Option.fromNullishOr(firstSpanAt),
351 lastSpanAt: Option.fromNullishOr(lastSpanAt),
352 failureCount,
353 interruptionCount,
354 slowSpanThresholdMs,
355 slowSpanCount,
356 logLevelCounts,
357 topSpansByCount,
358 slowestSpans,
359 commonFailures: [...failuresByKey.values()]
360 .toSorted(
361 (left, right) =>
362 right.count - left.count ||
363 DateTime.toEpochMillis(right.lastSeenAt) - DateTime.toEpochMillis(left.lastSeenAt),
364 )
365 .slice(0, TOP_LIMIT),
366 latestFailures,
367 latestWarningAndErrorLogs,
368 partialFailure: input.partialFailure ? Option.some(true) : Option.none(),
369 error: Option.fromNullishOr(input.error),
370 };
371 };
372
373 return { addLine, finish };
374 }
375
376 /** Succeeds with false if the file is absent. */
377 export function streamTraceFileLines(
378 fileSystem: FileSystem.FileSystem,
379 path: string,
380 onLine: (line: string) => void,
381 ): Effect.Effect<boolean, TraceFileReadError> {
382 return fileSystem.stream(path).pipe(
383 Stream.decodeText,
384 Stream.splitLines,
385 Stream.runForEachArray((lines) => Effect.sync(() => lines.forEach(onLine))),
386 Effect.as(true),
387 Effect.catchTags({
388 PlatformError: (cause) =>
389 isNotFoundError(cause)
390 ? Effect.succeed(false)
391 : Effect.fail(
392 new TraceFileReadError({
393 traceFilePath: path,
394 causeTag: cause.reason._tag,
395 cause,
396 }),
397 ),
398 }),
399 );
400 }
401
402 /** @public */
403 export const make = Effect.gen(function* () {
404 const fileSystem = yield* FileSystem.FileSystem;
405
406 const read: TraceDiagnostics["Service"]["read"] = Effect.fn("TraceDiagnostics.read")(
407 function* (options) {
408 const readAt = options.readAt ?? (yield* DateTime.now);
409 const slowSpanThresholdMs = options.slowSpanThresholdMs ?? DEFAULT_SLOW_SPAN_THRESHOLD_MS;
410 const paths = toRotatedTracePaths(options.traceFilePath, options.maxFiles);
411 const aggregator = makeTraceDiagnosticsAggregator(slowSpanThresholdMs);
412 const results = yield* Effect.forEach(
413 paths,
414 (path) =>
415 streamTraceFileLines(fileSystem, path, aggregator.addLine).pipe(
416 Effect.tapError((cause) =>
417 Effect.logWarning("Failed to read local trace file.").pipe(
418 Effect.annotateLogs({
419 traceFilePath: cause.traceFilePath,
420 errorTag: cause._tag,
421 causeTag: cause.causeTag,
422 }),
423 ),
424 ),
425 Effect.result,
426 ),
427 { concurrency: 1 },
428 );
429 const foundFile = results.some((result) => Result.isSuccess(result) && result.success);
430 const readFailure = results.find(Result.isFailure);
431 const readFailureError = readFailure
432 ? ({
433 kind: "trace-file-read-failed",
434 message: readFailure.failure.message,
435 } satisfies TraceDiagnosticsErrorSummary)
436 : undefined;
437
438 if (!foundFile) {
439 return makeEmptyDiagnostics({
440 traceFilePath: options.traceFilePath,
441 scannedFilePaths: paths,
442 readAt,
443 slowSpanThresholdMs,
444 error:
445 readFailureError ??
446 ({
447 kind: "trace-file-not-found",
448 message: "No local trace files were found.",
449 } satisfies TraceDiagnosticsErrorSummary),
450 });
451 }
452
453 return aggregator.finish({
454 traceFilePath: options.traceFilePath,
455 scannedFilePaths: paths,
456 readAt,
457 ...(readFailureError ? { partialFailure: true, error: readFailureError } : {}),
458 });
459 },
460 );
461
462 return TraceDiagnostics.of({ read });
463 });
464
465 export const layer = Layer.effect(TraceDiagnostics, make);
466
467 export const readTraceDiagnostics = Effect.fn("TraceDiagnostics.readTraceDiagnostics")(function* (
468 options: TraceDiagnosticsOptions,
469 ): Effect.fn.Return<ServerTraceDiagnosticsResult, never, TraceDiagnostics> {
470 const diagnostics = yield* TraceDiagnostics;
471 return yield* diagnostics.read(options);
472 });