The best way to control Claude Code, Codex and more from any computer or phone
feat(server): traces name what grew the heap, for ~0.3µs per span (#27104)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 13:28 UTC
2f4da5de4e6ce094da698e969dfa245f6891d8f4
3 files changed
+182
-1
apps/server/src/observability/HeapAttribution.test.ts
new
+69
@@ -0,0 +1,69 @@
1
+import { assert, it } from "@effect/vitest";
2
+import * as Effect from "effect/Effect";
3
+import * as Option from "effect/Option";
4
+import * as Context from "effect/Context";
5
+import * as TestClock from "effect/testing/TestClock";
6
+import * as Tracer from "effect/Tracer";
7
+
8
+import * as HeapAttribution from "./HeapAttribution.ts";
9
+
10
+const MB = 1024 * 1024;
11
+
12
+it.effect("names the span that grew the heap in a spike, not an older open or ended one", () =>
13
+ Effect.gen(function* () {
14
+ const ended: Array<Tracer.NativeSpan> = [];
15
+ const inner = Tracer.make({
16
+ span(options) {
17
+ const span = new Tracer.NativeSpan(options);
18
+ const end = span.end.bind(span);
19
+ span.end = (endTime, exit) => {
20
+ end(endTime, exit);
21
+ ended.push(span);
22
+ };
23
+ return span;
24
+ },
25
+ });
26
+ let heap = 300 * MB;
27
+ const attribution = yield* HeapAttribution.make(inner, () => heap);
28
+ yield* attribution.run;
29
+ // The sampler's baseline: heap before the spans allocate.
30
+ yield* TestClock.adjust("1 second");
31
+ const open = (name: string) =>
32
+ attribution.tracer.span({
33
+ name,
34
+ parent: Option.none(),
35
+ annotations: Context.empty(),
36
+ links: [],
37
+ startTime: 0n,
38
+ kind: "internal",
39
+ root: false,
40
+ sampled: true,
41
+ });
42
+
43
+ const subscription = open("ws.rpc.orchestration.subscribeShell");
44
+ const old = open("ProjectionStore.getSnapshot");
45
+ // Slow drift never spikes, but leaves the long-lived span far above its start.
46
+ for (let step = 0; step < 5; step++) {
47
+ heap += 80 * MB;
48
+ yield* TestClock.adjust("11 seconds");
49
+ if (step === 1) old.end(1n, undefined as never);
50
+ }
51
+ const small = open("ProjectionStore.getThreadShell");
52
+ const big = open("ProjectionStore.getThreadProjection");
53
+ heap += 2 * MB;
54
+ small.end(1n, undefined as never);
55
+ heap += 198 * MB;
56
+ yield* TestClock.adjust("2 seconds");
57
+
58
+ const spike = ended.find((span) => span.name === "server.heap.spike");
59
+ assert.isDefined(spike);
60
+ assert.deepEqual(spike!.attributes.get("suspects"), [
61
+ "ProjectionStore.getThreadProjection +200 MB",
62
+ ]);
63
+ big.end(2n, undefined as never);
64
+ subscription.end(3n, undefined as never);
65
+ assert.equal(big.attributes.get("memory.heap_growth_bytes"), 200 * MB);
66
+ assert.isUndefined(small.attributes.get("memory.heap_growth_bytes"));
67
+ assert.equal(old.attributes.get("memory.heap_growth_bytes"), 160 * MB);
68
+ }),
69
+);
apps/server/src/observability/HeapAttribution.ts
new
+108
@@ -0,0 +1,108 @@
1
+// @effect-diagnostics nodeBuiltinImport:off - only node:v8 exposes a heap reading cheap enough to take per span.
2
+import * as NodeV8 from "node:v8";
3
+
4
+import * as Effect from "effect/Effect";
5
+import type * as Scope from "effect/Scope";
6
+import * as Tracer from "effect/Tracer";
7
+
8
+const MB = 1024 * 1024;
9
+// Smaller growth is noise from concurrent fibers; an attribute on every span would bloat exports.
10
+const SPAN_GROWTH_ATTRIBUTE_BYTES = 8 * MB;
11
+const SPIKE_GROWTH_BYTES = 128 * MB;
12
+const SPIKE_WINDOW_SAMPLES = 10;
13
+const SPIKE_COOLDOWN_SAMPLES = 60;
14
+const RECENT_SPANS = 32;
15
+const REPORTED_SPANS = 8;
16
+
17
+const readHeapUsed = () => NodeV8.getHeapStatistics().used_heap_size;
18
+
19
+interface Growth {
20
+ readonly name: string;
21
+ readonly growthBytes: number;
22
+}
23
+
24
+/**
25
+ * Heap growth is approximate: it includes other fibers' allocations and GC, so it names
26
+ * suspects, not culprits. A spike lists what was open or just grew.
27
+ */
28
+export const make = (tracer: Tracer.Tracer, heapUsed: () => number = readHeapUsed) =>
29
+ Effect.sync(() => {
30
+ const open = new Map<
31
+ Tracer.Span,
32
+ { readonly name: string; readonly startHeap: number; readonly startTick: number }
33
+ >();
34
+ const recent: Array<Growth & { readonly endTick: number }> = [];
35
+ const samples: Array<number> = [];
36
+ let cooldown = 0;
37
+ let tick = 0;
38
+
39
+ const wrapped = Tracer.make({
40
+ ...(tracer.context === undefined ? {} : { context: tracer.context }),
41
+ span(options) {
42
+ const span = tracer.span(options);
43
+ const startHeap = heapUsed();
44
+ open.set(span, { name: options.name, startHeap, startTick: tick });
45
+ const end = span.end.bind(span);
46
+ span.end = (endTime, exit) => {
47
+ open.delete(span);
48
+ const growthBytes = heapUsed() - startHeap;
49
+ if (growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES) {
50
+ span.attribute("memory.heap_growth_bytes", growthBytes);
51
+ recent.push({ name: options.name, growthBytes, endTick: tick });
52
+ if (recent.length > RECENT_SPANS) recent.shift();
53
+ }
54
+ end(endTime, exit);
55
+ };
56
+ return span;
57
+ },
58
+ });
59
+
60
+ // Untraced: it runs every second and only emits a span when the heap spikes.
61
+ const sample = Effect.fnUntraced(function* () {
62
+ const heap = heapUsed();
63
+ tick++;
64
+ samples.push(heap);
65
+ if (samples.length > SPIKE_WINDOW_SAMPLES) samples.shift();
66
+ if (cooldown > 0) {
67
+ cooldown--;
68
+ return;
69
+ }
70
+ const growthBytes = heap - Math.min(...samples);
71
+ if (growthBytes < SPIKE_GROWTH_BYTES) return;
72
+ cooldown = SPIKE_COOLDOWN_SAMPLES;
73
+ const suspects = [
74
+ // Spans open since before the window, such as subscriptions, would claim all of its growth.
75
+ ...[...open.values()]
76
+ .filter(({ startTick }) => tick - startTick <= samples.length)
77
+ .map(({ name, startHeap }) => ({ name, growthBytes: heap - startHeap })),
78
+ ...recent.filter(({ endTick }) => tick - endTick <= samples.length),
79
+ ]
80
+ .filter((suspect) => suspect.growthBytes >= SPAN_GROWTH_ATTRIBUTE_BYTES)
81
+ .toSorted((left, right) => right.growthBytes - left.growthBytes)
82
+ .slice(0, REPORTED_SPANS)
83
+ .map(({ name, growthBytes }) => `${name} +${Math.round(growthBytes / MB)} MB`);
84
+ recent.length = 0;
85
+ // A span, not a log: this fiber starts before the server logger is provided.
86
+ yield* Effect.void.pipe(
87
+ Effect.withSpan("server.heap.spike", {
88
+ root: true,
89
+ level: "Warn",
90
+ attributes: {
91
+ heapUsedMb: Math.round(heap / MB),
92
+ growthMb: Math.round(growthBytes / MB),
93
+ suspects,
94
+ },
95
+ }),
96
+ );
97
+ });
98
+
99
+ const run: Effect.Effect<void, never, Scope.Scope> = sample().pipe(
100
+ Effect.andThen(Effect.sleep("1 second")),
101
+ Effect.forever,
102
+ Effect.withTracer(wrapped),
103
+ Effect.forkScoped,
104
+ Effect.asVoid,
105
+ );
106
+
107
+ return { tracer: wrapped, run };
108
+ });
apps/server/src/observability/Observability.ts
+5
-1
@@ -16,6 +16,7 @@ import * as ServerConfig from "../config.ts";
16
import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts";
17
import * as ServerLogger from "../serverLogger.ts";
18
import * as BrowserTraceCollector from "./BrowserTraceCollector.ts";
19
+import * as HeapAttribution from "./HeapAttribution.ts";
20
21
export const layer = Layer.unwrap(
22
Effect.gen(function* () {
@@ -71,8 +72,11 @@ export const layer = Layer.unwrap(
72
...(delegate ? { delegate } : {}),
73
});
74
75
+ const heapAttribution = yield* HeapAttribution.make(tracer);
76
+ yield* heapAttribution.run;
77
+
78
return Layer.mergeAll(
75
- Layer.succeed(Tracer.Tracer, tracer),
79
+ Layer.succeed(Tracer.Tracer, heapAttribution.tracer),
80
BrowserTraceCollector.layer(sink),
81
);
82
}),