@devries-cameron20 / t3code Mirror · read-only / commits / 2f4da5de4e
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 12:00 UTC

feat(server): traces name what grew the heap, for ~0.3µs per span (#27104)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 29, 2026 at 13:28 UTC 2f4da5de4e6ce094da698e969dfa245f6891d8f4
3 files changed +182 -1
apps/server/src/observability/HeapAttribution.test.ts new
+69
@@ -0,0 +1,69 @@
1 +import { assert, it } from "@effect/vitest";
2 +import * as Effect from "effect/Effect";
3 +import * as Option from "effect/Option";
4 +import * as Context from "effect/Context";
5 +import * as TestClock from "effect/testing/TestClock";
6 +import * as Tracer from "effect/Tracer";
7 +
8 +import * as HeapAttribution from "./HeapAttribution.ts";
9 +
10 +const MB = 1024 * 1024;
11 +
12 +it.effect("names the span that grew the heap in a spike, not an older open or ended one", () =>
13 + Effect.gen(function* () {
14 + const ended: Array<Tracer.NativeSpan> = [];
15 + const inner = Tracer.make({
16 + span(options) {
17 + const span = new Tracer.NativeSpan(options);
18 + const end = span.end.bind(span);
19 + span.end = (endTime, exit) => {
20 + end(endTime, exit);
21 + ended.push(span);
22 + };
23 + return span;
24 + },
25 + });
26 + let heap = 300 * MB;
27 + const attribution = yield* HeapAttribution.make(inner, () => heap);
28 + yield* attribution.run;
29 + // The sampler's baseline: heap before the spans allocate.
30 + yield* TestClock.adjust("1 second");
31 + const open = (name: string) =>
32 + attribution.tracer.span({
33 + name,
34 + parent: Option.none(),
35 + annotations: Context.empty(),
36 + links: [],
37 + startTime: 0n,
38 + kind: "internal",
39 + root: false,
40 + sampled: true,
41 + });
42 +
43 + const subscription = open("ws.rpc.orchestration.subscribeShell");
44 + const old = open("ProjectionStore.getSnapshot");
45 + // Slow drift never spikes, but leaves the long-lived span far above its start.
46 + for (let step = 0; step < 5; step++) {
47 + heap += 80 * MB;
48 + yield* TestClock.adjust("11 seconds");
49 + if (step === 1) old.end(1n, undefined as never);
50 + }
51 + const small = open("ProjectionStore.getThreadShell");
52 + const big = open("ProjectionStore.getThreadProjection");
53 + heap += 2 * MB;
54 + small.end(1n, undefined as never);
55 + heap += 198 * MB;
56 + yield* TestClock.adjust("2 seconds");
57 +
58 + const spike = ended.find((span) => span.name === "server.heap.spike");
59 + assert.isDefined(spike);
60 + assert.deepEqual(spike!.attributes.get("suspects"), [
61 + "ProjectionStore.getThreadProjection +200 MB",
62 + ]);
63 + big.end(2n, undefined as never);
64 + subscription.end(3n, undefined as never);
65 + assert.equal(big.attributes.get("memory.heap_growth_bytes"), 200 * MB);
66 + assert.isUndefined(small.attributes.get("memory.heap_growth_bytes"));
67 + assert.equal(old.attributes.get("memory.heap_growth_bytes"), 160 * MB);
68 + }),
69 +);
apps/server/src/observability/HeapAttribution.ts new
+108
@@ -0,0 +1,108 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - only node:v8 exposes a heap reading cheap enough to take per span.
2 +import * as NodeV8 from "node:v8";
3 +
4 +import * as Effect from "effect/Effect";
5 +import type * as Scope from "effect/Scope";
6 +import * as Tracer from "effect/Tracer";
7 +
8 +const MB = 1024 * 1024;
9 +// Smaller growth is noise from concurrent fibers; an attribute on every span would bloat exports.
10 +const SPAN_GROWTH_ATTRIBUTE_BYTES = 8 * MB;
11 +const SPIKE_GROWTH_BYTES = 128 * MB;
12 +const SPIKE_WINDOW_SAMPLES = 10;
13 +const SPIKE_COOLDOWN_SAMPLES = 60;
14 +const RECENT_SPANS = 32;
15 +const REPORTED_SPANS = 8;
16 +
17 +const readHeapUsed = () => NodeV8.getHeapStatistics().used_heap_size;
18 +
19 +interface Growth {
20 + readonly name: string;
21 + readonly growthBytes: number;
22 +}
23 +
24 +/**
25 + * Heap growth is approximate: it includes other fibers' allocations and GC, so it names
26 + * suspects, not culprits. A spike lists what was open or just grew.
27 + */
28 +export const make = (tracer: Tracer.Tracer, heapUsed: () => number = readHeapUsed) =>
29 + Effect.sync(() => {
30 + const open = new Map<
31 + Tracer.Span,
32 + { readonly name: string; readonly startHeap: number; readonly startTick: number }
33 + >();
34 + const recent: Array<Growth & { readonly endTick: number }> = [];
35 + const samples: Array<number> = [];
36 + let cooldown = 0;
37 + let tick = 0;
38 +
39 + const wrapped = Tracer.make({
40 + ...(tracer.context === undefined ? {} : { context: tracer.context }),
41 + span(options) {
42 + const span = tracer.span(options);
43 + const startHeap = heapUsed();
44 + open.set(span, { name: options.name, startHeap, startTick: tick });
45 + const end = span.end.bind(span);
46 + span.end = (endTime, exit) => {
47 + open.delete(span);
48 + const growthBytes = heapUsed() - startHeap;
49 + if (growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES) {
50 + span.attribute("memory.heap_growth_bytes", growthBytes);
51 + recent.push({ name: options.name, growthBytes, endTick: tick });
52 + if (recent.length > RECENT_SPANS) recent.shift();
53 + }
54 + end(endTime, exit);
55 + };
56 + return span;
57 + },
58 + });
59 +
60 + // Untraced: it runs every second and only emits a span when the heap spikes.
61 + const sample = Effect.fnUntraced(function* () {
62 + const heap = heapUsed();
63 + tick++;
64 + samples.push(heap);
65 + if (samples.length > SPIKE_WINDOW_SAMPLES) samples.shift();
66 + if (cooldown > 0) {
67 + cooldown--;
68 + return;
69 + }
70 + const growthBytes = heap - Math.min(...samples);
71 + if (growthBytes < SPIKE_GROWTH_BYTES) return;
72 + cooldown = SPIKE_COOLDOWN_SAMPLES;
73 + const suspects = [
74 + // Spans open since before the window, such as subscriptions, would claim all of its growth.
75 + ...[...open.values()]
76 + .filter(({ startTick }) => tick - startTick <= samples.length)
77 + .map(({ name, startHeap }) => ({ name, growthBytes: heap - startHeap })),
78 + ...recent.filter(({ endTick }) => tick - endTick <= samples.length),
79 + ]
80 + .filter((suspect) => suspect.growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES)
81 + .toSorted((left, right) => right.growthBytes - left.growthBytes)
82 + .slice(0, REPORTED_SPANS)
83 + .map(({ name, growthBytes }) => `${name} +${Math.round(growthBytes / MB)} MB`);
84 + recent.length = 0;
85 + // A span, not a log: this fiber starts before the server logger is provided.
86 + yield* Effect.void.pipe(
87 + Effect.withSpan("server.heap.spike", {
88 + root: true,
89 + level: "Warn",
90 + attributes: {
91 + heapUsedMb: Math.round(heap / MB),
92 + growthMb: Math.round(growthBytes / MB),
93 + suspects,
94 + },
95 + }),
96 + );
97 + });
98 +
99 + const run: Effect.Effect<void, never, Scope.Scope> = sample().pipe(
100 + Effect.andThen(Effect.sleep("1 second")),
101 + Effect.forever,
102 + Effect.withTracer(wrapped),
103 + Effect.forkScoped,
104 + Effect.asVoid,
105 + );
106 +
107 + return { tracer: wrapped, run };
108 + });
apps/server/src/observability/Observability.ts
+5 -1
@@ -16,6 +16,7 @@ import * as ServerConfig from "../config.ts";
16 import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts";
17 import * as ServerLogger from "../serverLogger.ts";
18 import * as BrowserTraceCollector from "./BrowserTraceCollector.ts";
19 +import * as HeapAttribution from "./HeapAttribution.ts";
20
21 export const layer = Layer.unwrap(
22 Effect.gen(function* () {
@@ -71,8 +72,11 @@ export const layer = Layer.unwrap(
72 ...(delegate ? { delegate } : {}),
73 });
74
75 + const heapAttribution = yield* HeapAttribution.make(tracer);
76 + yield* heapAttribution.run;
77 +
78 return Layer.mergeAll(
75 - Layer.succeed(Tracer.Tracer, tracer),
79 + Layer.succeed(Tracer.Tracer, heapAttribution.tracer),
80 BrowserTraceCollector.layer(sink),
81 );
82 }),