The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off - only node:perf_hooks exposes the event loop delay histogram, and node:os the load average. |
| 2 | import * as NodeOS from "node:os"; |
| 3 | import * as NodePerfHooks from "node:perf_hooks"; |
| 4 | |
| 5 | import type { ServerEventLoopStall } from "@t3tools/contracts"; |
| 6 | import { RotatingFileSink, RotatingFileSinkError } from "@t3tools/shared/logging"; |
| 7 | import * as Context from "effect/Context"; |
| 8 | import * as DateTime from "effect/DateTime"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import * as Layer from "effect/Layer"; |
| 11 | import * as Metric from "effect/Metric"; |
| 12 | import * as Path from "effect/Path"; |
| 13 | import * as Ref from "effect/Ref"; |
| 14 | import * as Schema from "effect/Schema"; |
| 15 | import type * as Scope from "effect/Scope"; |
| 16 | import * as Semaphore from "effect/Semaphore"; |
| 17 | |
| 18 | import * as ServerConfig from "../config.ts"; |
| 19 | import * as Metrics from "./Metrics.ts"; |
| 20 | |
| 21 | // Subtract the histogram's timer resolution; this can undercount stalls by up to RESOLUTION_MS. |
| 22 | const RESOLUTION_MS = 1000; |
| 23 | const STALL_THRESHOLD_MS = 2000; |
| 24 | const SAMPLE_INTERVAL = "30 seconds"; |
| 25 | const STALL_LOG_MAX_BYTES = 256 * 1024; |
| 26 | // Long enough that users felt it. Shorter stalls stay in the log only. |
| 27 | const NOTICEABLE_STALL_MS = 5000; |
| 28 | // Clients show what they get, so an old stall must not explain a new drop. |
| 29 | const NOTICEABLE_STALL_MAX_AGE_MS = 5 * 60_000; |
| 30 | |
| 31 | export class EventLoopMonitor extends Context.Reference<{ |
| 32 | /** |
| 33 | * The latest stall of 5 s or more, if seen in the last 5 minutes. It samples |
| 34 | * first, so a stall that ended just before a client reconnected is included. |
| 35 | */ |
| 36 | readonly latestNoticeableStall: Effect.Effect<ServerEventLoopStall | undefined>; |
| 37 | }>("t3/observability/EventLoopMonitor", { |
| 38 | defaultValue: () => ({ latestNoticeableStall: Effect.undefined }), |
| 39 | }) {} |
| 40 | |
| 41 | /** Delay in ns, active time in ms, CPU in µs. */ |
| 42 | export interface EventLoopReadings { |
| 43 | readonly delayMaxNs: number; |
| 44 | readonly activeMs: number; |
| 45 | readonly utilization: number; |
| 46 | readonly usage: Pick< |
| 47 | NodeJS.ResourceUsage, |
| 48 | | "userCPUTime" |
| 49 | | "systemCPUTime" |
| 50 | | "majorPageFault" |
| 51 | | "minorPageFault" |
| 52 | | "involuntaryContextSwitches" |
| 53 | >; |
| 54 | readonly rssBytes: number; |
| 55 | readonly heapUsedBytes: number; |
| 56 | /** Wall time the readings cover. */ |
| 57 | readonly elapsedMs: number; |
| 58 | /** One-minute load average, undefined on Windows. */ |
| 59 | readonly loadAverage: number | undefined; |
| 60 | readonly cpuCount: number; |
| 61 | } |
| 62 | |
| 63 | // Node skips the first gap after a reset, so a stall right at a sample boundary can be missed. |
| 64 | const makeNodeSampler = Effect.gen(function* () { |
| 65 | const histogram = yield* Effect.acquireRelease( |
| 66 | Effect.sync(() => { |
| 67 | const histogram = NodePerfHooks.monitorEventLoopDelay({ resolution: RESOLUTION_MS }); |
| 68 | histogram.enable(); |
| 69 | return histogram; |
| 70 | }), |
| 71 | (histogram) => Effect.sync(() => histogram.disable()), |
| 72 | ); |
| 73 | let elu = NodePerfHooks.performance.eventLoopUtilization(); |
| 74 | let usage = process.resourceUsage(); |
| 75 | let sampledAt = NodePerfHooks.performance.now(); |
| 76 | |
| 77 | return Effect.sync(() => { |
| 78 | const nextElu = NodePerfHooks.performance.eventLoopUtilization(); |
| 79 | const nextUsage = process.resourceUsage(); |
| 80 | const nextSampledAt = NodePerfHooks.performance.now(); |
| 81 | const memory = process.memoryUsage(); |
| 82 | const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu); |
| 83 | const readings: EventLoopReadings = { |
| 84 | delayMaxNs: histogram.max, |
| 85 | activeMs: loop.active, |
| 86 | utilization: loop.utilization, |
| 87 | usage: { |
| 88 | userCPUTime: nextUsage.userCPUTime - usage.userCPUTime, |
| 89 | systemCPUTime: nextUsage.systemCPUTime - usage.systemCPUTime, |
| 90 | majorPageFault: nextUsage.majorPageFault - usage.majorPageFault, |
| 91 | minorPageFault: nextUsage.minorPageFault - usage.minorPageFault, |
| 92 | involuntaryContextSwitches: |
| 93 | nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches, |
| 94 | }, |
| 95 | rssBytes: memory.rss, |
| 96 | heapUsedBytes: memory.heapUsed, |
| 97 | elapsedMs: nextSampledAt - sampledAt, |
| 98 | // Windows has no load average and reports 0, which a running process never sees elsewhere. |
| 99 | loadAverage: NodeOS.loadavg()[0] || undefined, |
| 100 | cpuCount: NodeOS.availableParallelism(), |
| 101 | }; |
| 102 | histogram.reset(); |
| 103 | elu = nextElu; |
| 104 | usage = nextUsage; |
| 105 | sampledAt = nextSampledAt; |
| 106 | return readings; |
| 107 | }); |
| 108 | }); |
| 109 | |
| 110 | export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => { |
| 111 | const delayMs = Math.round(delayMaxNs / 1e6) - RESOLUTION_MS; |
| 112 | // libuv clocks run during macOS/Windows sleep; idle poll time distinguishes it from stalls. |
| 113 | if (delayMs <= STALL_THRESHOLD_MS || activeMs < delayMs) return undefined; |
| 114 | return delayMs; |
| 115 | }; |
| 116 | |
| 117 | // Untraced: it runs inside every 30-second tick, which already has the monitor's own span. |
| 118 | const recordGauges = Effect.fnUntraced(function* (readings: EventLoopReadings) { |
| 119 | const cpuMs = (readings.usage.userCPUTime + readings.usage.systemCPUTime) / 1000; |
| 120 | yield* Metric.update(Metrics.serverResidentBytes, readings.rssBytes); |
| 121 | yield* Metric.update(Metrics.serverHeapUsedBytes, readings.heapUsedBytes); |
| 122 | yield* Metric.update(Metrics.serverEventLoopUtilization, readings.utilization); |
| 123 | yield* Metric.update( |
| 124 | Metrics.serverEventLoopDelayMax, |
| 125 | Math.max(0, Math.round(readings.delayMaxNs / 1e6) - RESOLUTION_MS), |
| 126 | ); |
| 127 | if (readings.elapsedMs > 0) { |
| 128 | yield* Metric.update(Metrics.serverCpuUtilization, cpuMs / readings.elapsedMs); |
| 129 | } |
| 130 | }); |
| 131 | |
| 132 | const encodeStallLine = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)); |
| 133 | |
| 134 | export interface StallLogOptions { |
| 135 | readonly filePath: string; |
| 136 | readonly maxBytes: number; |
| 137 | } |
| 138 | |
| 139 | const layerWith = ( |
| 140 | makeSampler: Effect.Effect<Effect.Effect<EventLoopReadings>, never, Scope.Scope>, |
| 141 | stallLog: StallLogOptions, |
| 142 | ) => |
| 143 | Layer.effect( |
| 144 | EventLoopMonitor, |
| 145 | Effect.gen(function* () { |
| 146 | const sample = yield* makeSampler; |
| 147 | // Clients sample on connect as well as the timer, and the sampler resets shared state. |
| 148 | const sampling = yield* Semaphore.make(1); |
| 149 | const warmedUp = yield* Ref.make(false); |
| 150 | const latestNoticeableStall = yield* Ref.make< |
| 151 | { readonly stall: ServerEventLoopStall; readonly detectedAtMs: number } | undefined |
| 152 | >(undefined); |
| 153 | const stallSink = yield* Effect.try({ |
| 154 | try: () => |
| 155 | new RotatingFileSink({ |
| 156 | filePath: stallLog.filePath, |
| 157 | maxBytes: stallLog.maxBytes, |
| 158 | maxFiles: 1, |
| 159 | throwOnError: true, |
| 160 | }), |
| 161 | catch: (cause) => |
| 162 | new RotatingFileSinkError({ |
| 163 | operation: "initialize", |
| 164 | filePath: stallLog.filePath, |
| 165 | cause, |
| 166 | }), |
| 167 | }).pipe( |
| 168 | Effect.catch(() => |
| 169 | Effect.logWarning("event loop stall log unavailable", { |
| 170 | filePath: stallLog.filePath, |
| 171 | }).pipe(Effect.as(undefined)), |
| 172 | ), |
| 173 | ); |
| 174 | const tick = Effect.gen(function* () { |
| 175 | const readings = yield* sample; |
| 176 | yield* recordGauges(readings); |
| 177 | const delayMaxMs = stallMs(readings); |
| 178 | if (delayMaxMs === undefined) return; |
| 179 | const { utilization, usage, rssBytes, loadAverage, cpuCount } = readings; |
| 180 | const now = yield* DateTime.now; |
| 181 | const detectedAt = DateTime.formatIso(now); |
| 182 | if (delayMaxMs >= NOTICEABLE_STALL_MS) { |
| 183 | yield* Ref.set(latestNoticeableStall, { |
| 184 | detectedAtMs: DateTime.toEpochMillis(now), |
| 185 | stall: { |
| 186 | detectedAt, |
| 187 | durationMs: delayMaxMs, |
| 188 | ...(loadAverage === undefined |
| 189 | ? {} |
| 190 | : { loadAverage: Math.round(loadAverage * 10) / 10 }), |
| 191 | cpuCount, |
| 192 | }, |
| 193 | }); |
| 194 | } |
| 195 | const attributes = { |
| 196 | delayMaxMs, |
| 197 | utilization: Math.round(utilization * 100) / 100, |
| 198 | cpuUserMs: Math.round(usage.userCPUTime / 1000), |
| 199 | cpuSystemMs: Math.round(usage.systemCPUTime / 1000), |
| 200 | majorPageFaults: usage.majorPageFault, |
| 201 | minorPageFaults: usage.minorPageFault, |
| 202 | involuntaryContextSwitches: usage.involuntaryContextSwitches, |
| 203 | rssMb: Math.round(rssBytes / 1024 / 1024), |
| 204 | }; |
| 205 | yield* Effect.logWarning(`event loop stalled for ${delayMaxMs} ms`).pipe( |
| 206 | Effect.withSpan("server.eventLoop.stall", { root: true, level: "Warn", attributes }), |
| 207 | ); |
| 208 | if (stallSink !== undefined) { |
| 209 | yield* encodeStallLine({ time: detectedAt, ...attributes }).pipe( |
| 210 | Effect.flatMap((line) => |
| 211 | Effect.try({ |
| 212 | try: () => stallSink.write(`${line}\n`), |
| 213 | catch: (cause) => |
| 214 | new RotatingFileSinkError({ |
| 215 | operation: "write", |
| 216 | filePath: stallLog.filePath, |
| 217 | cause, |
| 218 | }), |
| 219 | }), |
| 220 | ), |
| 221 | Effect.catch(() => |
| 222 | Effect.logWarning("event loop stall log write failed", { |
| 223 | filePath: stallLog.filePath, |
| 224 | }), |
| 225 | ), |
| 226 | ); |
| 227 | } |
| 228 | }).pipe(sampling.withPermits(1)); |
| 229 | const wait = Effect.sleep(SAMPLE_INTERVAL); |
| 230 | // The first sample includes migrations and bootstrap; skip expected startup stalls. |
| 231 | yield* wait.pipe( |
| 232 | Effect.andThen(sampling.withPermits(1)(sample)), |
| 233 | Effect.andThen(Ref.set(warmedUp, true)), |
| 234 | Effect.andThen(wait.pipe(Effect.andThen(tick), Effect.forever)), |
| 235 | Effect.forkScoped, |
| 236 | ); |
| 237 | |
| 238 | return EventLoopMonitor.of({ |
| 239 | latestNoticeableStall: Effect.gen(function* () { |
| 240 | if (yield* Ref.get(warmedUp)) yield* tick; |
| 241 | const latest = yield* Ref.get(latestNoticeableStall); |
| 242 | if (latest === undefined) return undefined; |
| 243 | const ageMs = DateTime.toEpochMillis(yield* DateTime.now) - latest.detectedAtMs; |
| 244 | return ageMs <= NOTICEABLE_STALL_MAX_AGE_MS ? latest.stall : undefined; |
| 245 | }), |
| 246 | }); |
| 247 | }), |
| 248 | ); |
| 249 | |
| 250 | export const layer = Layer.unwrap( |
| 251 | Effect.gen(function* () { |
| 252 | const { serverTracePath } = yield* ServerConfig.ServerConfig; |
| 253 | const path = yield* Path.Path; |
| 254 | return layerWith(makeNodeSampler, { |
| 255 | filePath: path.join(path.dirname(serverTracePath), "event-loop-stalls.ndjson"), |
| 256 | maxBytes: STALL_LOG_MAX_BYTES, |
| 257 | }); |
| 258 | }), |
| 259 | ); |