The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
feat(server): export server and process-group resource metrics (#22122)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 19, 2026 at 21:39 UTC
86d130af6c02a126515488ed6038b68906c35bf7
7 files changed
+271
-3
apps/server/src/observability/EventLoopMonitor.test.ts
+26
@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";
4
import * as FileSystem from "effect/FileSystem";
5
import * as Layer from "effect/Layer";
6
import * as Logger from "effect/Logger";
7
+import * as Metric from "effect/Metric";
8
import * as Tracer from "effect/Tracer";
9
import * as TestClock from "effect/testing/TestClock";
10
@@ -24,6 +25,8 @@ const stalled: EventLoopMonitor.EventLoopReadings = {
25
involuntaryContextSwitches: 57,
26
},
27
rssBytes: 1536 * 1024 * 1024,
28
+ heapUsedBytes: 512 * 1024 * 1024,
29
+ elapsedMs: 30_000,
30
};
31
const quiet: EventLoopMonitor.EventLoopReadings = { ...stalled, delayMaxNs: ms(2_950) };
32
@@ -78,6 +81,29 @@ describe("EventLoopMonitor", () => {
81
}),
82
);
83
84
+ it.effect("exports the server's memory, CPU and event loop from each sample", () =>
85
+ Effect.gen(function* () {
86
+ yield* monitor([quiet, stalled]);
87
+ yield* TestClock.adjust("60 seconds");
88
+
89
+ const gauges = Object.fromEntries(
90
+ (yield* Metric.snapshot).flatMap((snapshot) =>
91
+ snapshot.id.startsWith("t3_server_") && snapshot.type === "Gauge"
92
+ ? [[snapshot.id, snapshot.state.value]]
93
+ : [],
94
+ ),
95
+ );
96
+ assert.deepStrictEqual(gauges, {
97
+ t3_server_resident_bytes: 1536 * 1024 * 1024,
98
+ t3_server_heap_used_bytes: 512 * 1024 * 1024,
99
+ t3_server_event_loop_utilization: 0.176,
100
+ t3_server_event_loop_delay_max: 4_950,
101
+ // 406 ms of CPU over 30 s of wall time.
102
+ t3_server_cpu_utilization: 406 / 30_000,
103
+ });
104
+ }).pipe(Effect.scoped),
105
+ );
106
+
107
it.effect("keeps each stall as one line in a capped stall log", () =>
108
Effect.gen(function* () {
109
const fs = yield* FileSystem.FileSystem;
apps/server/src/observability/EventLoopMonitor.ts
+28
-1
@@ -5,11 +5,13 @@ import { RotatingFileSink } from "@t3tools/shared/logging";
5
import * as DateTime from "effect/DateTime";
6
import * as Effect from "effect/Effect";
7
import * as Layer from "effect/Layer";
8
+import * as Metric from "effect/Metric";
9
import * as Path from "effect/Path";
10
import * as Schema from "effect/Schema";
11
import type * as Scope from "effect/Scope";
12
13
import * as ServerConfig from "../config.ts";
14
+import * as Metrics from "./Metrics.ts";
15
16
// Node's delay histogram wakes a native timer every RESOLUTION_MS and records the
17
// gap between wakeups, so an idle loop reads about RESOLUTION_MS and a stall of S
@@ -34,6 +36,9 @@ export interface EventLoopReadings {
36
| "involuntaryContextSwitches"
37
>;
38
readonly rssBytes: number;
39
+ readonly heapUsedBytes: number;
40
+ /** Wall time the readings cover. */
41
+ readonly elapsedMs: number;
42
}
43
44
// Node skips the first gap after a reset, so a stall right at a sample boundary can be missed.
@@ -48,11 +53,14 @@ const makeNodeSampler = Effect.gen(function* () {
53
);
54
let elu = NodePerfHooks.performance.eventLoopUtilization();
55
let usage = process.resourceUsage();
56
+ let sampledAt = NodePerfHooks.performance.now();
57
58
// @effect-diagnostics-next-line returnEffectInGen:off - the read effect is the result.
59
return Effect.sync(() => {
60
const nextElu = NodePerfHooks.performance.eventLoopUtilization();
61
const nextUsage = process.resourceUsage();
62
+ const nextSampledAt = NodePerfHooks.performance.now();
63
+ const memory = process.memoryUsage();
64
const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu);
65
const readings: EventLoopReadings = {
66
delayMaxNs: histogram.max,
@@ -66,11 +74,14 @@ const makeNodeSampler = Effect.gen(function* () {
74
involuntaryContextSwitches:
75
nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches,
76
},
69
- rssBytes: process.memoryUsage.rss(),
77
+ rssBytes: memory.rss,
78
+ heapUsedBytes: memory.heapUsed,
79
+ elapsedMs: nextSampledAt - sampledAt,
80
};
81
histogram.reset();
82
elu = nextElu;
83
usage = nextUsage;
84
+ sampledAt = nextSampledAt;
85
return readings;
86
});
87
});
@@ -83,6 +94,21 @@ export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => {
94
return delayMs;
95
};
96
97
+// Untraced: it runs inside every 30-second tick, which already has the monitor's own span.
98
+const recordGauges = Effect.fnUntraced(function* (readings: EventLoopReadings) {
99
+ const cpuMs = (readings.usage.userCPUTime + readings.usage.systemCPUTime) / 1000;
100
+ yield* Metric.update(Metrics.serverResidentBytes, readings.rssBytes);
101
+ yield* Metric.update(Metrics.serverHeapUsedBytes, readings.heapUsedBytes);
102
+ yield* Metric.update(Metrics.serverEventLoopUtilization, readings.utilization);
103
+ yield* Metric.update(
104
+ Metrics.serverEventLoopDelayMax,
105
+ Math.max(0, Math.round(readings.delayMaxNs / 1e6) - RESOLUTION_MS),
106
+ );
107
+ if (readings.elapsedMs > 0) {
108
+ yield* Metric.update(Metrics.serverCpuUtilization, cpuMs / readings.elapsedMs);
109
+ }
110
+});
111
+
112
const encodeStallLine = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
113
114
export interface StallLogOptions {
@@ -114,6 +140,7 @@ export const layerWith = (
140
);
141
const tick = Effect.gen(function* () {
142
const readings = yield* sample;
143
+ yield* recordGauges(readings);
144
const delayMaxMs = stallMs(readings);
145
if (delayMaxMs === undefined) return;
146
const { utilization, usage, rssBytes } = readings;
apps/server/src/observability/Metrics.ts
+24
@@ -39,6 +39,30 @@ export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
39
description: "Time for the provider adapter to start a turn, not how long the turn runs.",
40
});
41
42
+// OTLP carries the unit, and Prometheus names the series by it: a unitless gauge gains `_ratio`.
43
+export const serverResidentBytes = Metric.gauge("t3_server_resident_bytes", {
44
+ description: "Resident memory of the server process.",
45
+ attributes: { unit: "By" },
46
+});
47
+
48
+export const serverHeapUsedBytes = Metric.gauge("t3_server_heap_used_bytes", {
49
+ description: "JavaScript heap in use by the server process.",
50
+ attributes: { unit: "By" },
51
+});
52
+
53
+export const serverCpuUtilization = Metric.gauge("t3_server_cpu_utilization", {
54
+ description: "CPU time the server used per second over the last sample; 1 is one full core.",
55
+});
56
+
57
+export const serverEventLoopUtilization = Metric.gauge("t3_server_event_loop_utilization", {
58
+ description: "Share of the last sample the server's event loop spent busy.",
59
+});
60
+
61
+export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_max", {
62
+ description: "Longest event loop delay over the last sample.",
63
+ attributes: { unit: "ms" },
64
+});
65
+
66
export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
67
description: "Total git commands executed by the server runtime.",
68
});
apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts
new
+59
@@ -0,0 +1,59 @@
1
+import { assert, describe, it } from "@effect/vitest";
2
+import type { ResourceTelemetryProcess } from "@t3tools/contracts";
3
+
4
+import { groupProcesses } from "./ProcessGroupMetrics.ts";
5
+
6
+const process = (
7
+ pid: number,
8
+ ppid: number,
9
+ name: string,
10
+ category: ResourceTelemetryProcess["category"] = "server-child",
11
+ command = name,
12
+) =>
13
+ ({
14
+ identity: { pid, startTimeMs: 1 },
15
+ ppid,
16
+ name,
17
+ command,
18
+ category,
19
+ residentBytes: pid * 1_000,
20
+ cpuPercent: pid,
21
+ }) as ResourceTelemetryProcess;
22
+
23
+describe("groupProcesses", () => {
24
+ it("sums each direct child of the server with its whole subtree", () => {
25
+ const totals = groupProcesses([
26
+ process(10, 1, "node", "server"),
27
+ process(11, 10, "t3-resource-mon", "resource-monitor"),
28
+ process(20, 10, "claude"),
29
+ process(21, 20, "bash"),
30
+ process(22, 21, "cargo"),
31
+ process(30, 10, "codex"),
32
+ process(40, 10, "zsh"),
33
+ process(50, 10, "chrome-headless"),
34
+ process(60, 10, "git", "server-child", "git checkout claude/fix"),
35
+ process(70, 99, "orphan"),
36
+ // An older Claude package runs as a script under node.
37
+ process(
38
+ 80,
39
+ 10,
40
+ "node",
41
+ "server-child",
42
+ "node /opt/node_modules/@anthropic-ai/claude-code/cli.js",
43
+ ),
44
+ process(90, 10, "node", "server-child", "node /srv/tools/report.js"),
45
+ ]);
46
+
47
+ assert.deepStrictEqual(Object.fromEntries(totals), {
48
+ server: { residentBytes: 10_000, cpuPercent: 10, count: 1 },
49
+ "resource-monitor": { residentBytes: 11_000, cpuPercent: 11, count: 1 },
50
+ // The agent's shell and the build it ran belong to the agent, not the terminal group.
51
+ claude: { residentBytes: 143_000, cpuPercent: 143, count: 4 },
52
+ codex: { residentBytes: 30_000, cpuPercent: 30, count: 1 },
53
+ terminal: { residentBytes: 40_000, cpuPercent: 40, count: 1 },
54
+ browser: { residentBytes: 50_000, cpuPercent: 50, count: 1 },
55
+ // `git` whose arguments mention claude, a parentless process, and a node script.
56
+ other: { residentBytes: 220_000, cpuPercent: 220, count: 3 },
57
+ });
58
+ });
59
+});
apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts
new
+125
@@ -0,0 +1,125 @@
1
+import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2
+import * as Effect from "effect/Effect";
3
+import * as Layer from "effect/Layer";
4
+import * as Metric from "effect/Metric";
5
+
6
+import * as ServerConfig from "../config.ts";
7
+import { metricAttributes } from "../observability/Metrics.ts";
8
+import * as ResourceTelemetry from "./ResourceTelemetry.ts";
9
+
10
+const SAMPLE_INTERVAL = "1 minute";
11
+
12
+const residentBytes = Metric.gauge("t3_process_group_resident_bytes", {
13
+ description: "Resident memory of the server's processes, summed by group.",
14
+ attributes: { unit: "By" },
15
+});
16
+const cpuPercent = Metric.gauge("t3_process_group_cpu_percent", {
17
+ description: "CPU use of the server's processes, summed by group; 100 is one full core.",
18
+ attributes: { unit: "%" },
19
+});
20
+const processCount = Metric.gauge("t3_process_group_count", {
21
+ description: "Processes in each group.",
22
+ attributes: { unit: "{process}" },
23
+});
24
+
25
+// Each direct child of the server names the group of its whole subtree. A fixed set keeps one
26
+// series per group per server, however many agents and processes come and go.
27
+const PROVIDER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
28
+ [/codex/, "codex"],
29
+ [/claude/, "claude"],
30
+ [/cursor/, "cursor"],
31
+ [/opencode/, "opencode"],
32
+ [/grok/, "grok"],
33
+ [/antigravity|gemini/, "antigravity"],
34
+];
35
+const OTHER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
36
+ [/chrom/, "browser"],
37
+ [/^(?:ba|z|fi|da|c)?sh(?:\.exe)?$|^pwsh|^powershell|^cmd\.exe$/, "terminal"],
38
+];
39
+// A CLI shipped as a script runs under a JavaScript runtime, so its process name is the
40
+// runtime's (older Claude packages run `node .../@anthropic-ai/claude-code/cli.js`). Only those
41
+// are matched by command line; any other name is matched by name alone, so a shell or `git`
42
+// whose arguments mention a provider stays in its own group.
43
+const SCRIPT_RUNTIME = /^(?:node|bun|deno)(?:\.exe)?$/;
44
+
45
+const match = (groups: ReadonlyArray<readonly [RegExp, string]>, text: string) =>
46
+ groups.find(([pattern]) => pattern.test(text))?.[1];
47
+
48
+const childGroup = (name: string, command: string) => {
49
+ const lowerName = name.toLowerCase();
50
+ if (SCRIPT_RUNTIME.test(lowerName))
51
+ return match(PROVIDER_GROUPS, command.toLowerCase()) ?? "other";
52
+ return match(PROVIDER_GROUPS, lowerName) ?? match(OTHER_GROUPS, lowerName) ?? "other";
53
+};
54
+
55
+export interface ProcessGroupTotals {
56
+ readonly residentBytes: number;
57
+ readonly cpuPercent: number;
58
+ readonly count: number;
59
+}
60
+
61
+export function groupProcesses(
62
+ processes: ReadonlyArray<ResourceTelemetryProcess>,
63
+): ReadonlyMap<string, ProcessGroupTotals> {
64
+ const byPid = new Map(processes.map((process) => [process.identity.pid, process]));
65
+ const serverPid = processes.find((process) => process.category === "server")?.identity.pid;
66
+ const groupOf = (process: ResourceTelemetryProcess): string => {
67
+ if (process.category === "server") return "server";
68
+ if (process.category === "resource-monitor") return "resource-monitor";
69
+ if (process.category.startsWith("electron-")) return "electron";
70
+ let current = process;
71
+ // Bounded by the table size, in case a reused PID makes the parent chain loop.
72
+ for (let step = 0; step < processes.length; step++) {
73
+ if (current.ppid === serverPid) return childGroup(current.name, current.command);
74
+ const parent = byPid.get(current.ppid);
75
+ if (parent === undefined) break;
76
+ current = parent;
77
+ }
78
+ return "other";
79
+ };
80
+ const totals = new Map<string, ProcessGroupTotals>();
81
+ for (const process of processes) {
82
+ const group = groupOf(process);
83
+ const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
84
+ totals.set(group, {
85
+ residentBytes: total.residentBytes + process.residentBytes,
86
+ cpuPercent: total.cpuPercent + process.cpuPercent,
87
+ count: total.count + 1,
88
+ });
89
+ }
90
+ return totals;
91
+}
92
+
93
+/**
94
+ * Exports process totals by group while OTLP metrics export is on. Sampling walks the process
95
+ * table, which an idle server otherwise never does, so only a server exporting metrics pays for
96
+ * it, once a minute.
97
+ */
98
+export const layer = Layer.effectDiscard(
99
+ Effect.gen(function* () {
100
+ const config = yield* ServerConfig.ServerConfig;
101
+ if (config.otlpMetricsUrl === undefined) return;
102
+ const telemetry = yield* ResourceTelemetry.ResourceTelemetry;
103
+ // A group that empties reads zero rather than holding its last value.
104
+ const seen = new Set<string>();
105
+ const sample = Effect.fn("ProcessGroupMetrics.sample")(function* () {
106
+ const totals = groupProcesses((yield* telemetry.refresh).processes);
107
+ for (const group of totals.keys()) seen.add(group);
108
+ for (const group of seen) {
109
+ const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
110
+ const attributes = metricAttributes({ group });
111
+ yield* Metric.update(Metric.withAttributes(residentBytes, attributes), total.residentBytes);
112
+ yield* Metric.update(Metric.withAttributes(cpuPercent, attributes), total.cpuPercent);
113
+ yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
114
+ }
115
+ });
116
+ yield* sample().pipe(
117
+ Effect.catch((error) =>
118
+ Effect.logDebug("process group metrics sample failed", { error: error.message }),
119
+ ),
120
+ Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
121
+ Effect.forever,
122
+ Effect.forkScoped,
123
+ );
124
+ }),
125
+);
apps/server/src/server.ts
+2
@@ -163,6 +163,7 @@ import * as DesktopTelemetryReceiver from "./resourceTelemetry/DesktopTelemetryR
163
import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClient.ts";
164
import * as ResourceAttribution from "./resourceTelemetry/ResourceAttribution.ts";
165
import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinary.ts";
166
+import * as ProcessGroupMetrics from "./resourceTelemetry/ProcessGroupMetrics.ts";
167
import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
168
import * as AntigravityUsage from "./provider/Drivers/AntigravityUsage.ts";
169
import * as CursorAccountReader from "@t3tools/provider-cursor/server/CursorAccountReader";
@@ -276,6 +277,7 @@ const layerResourceDiagnostics = Layer.mergeAll(
277
layerResourceTelemetry,
278
ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)),
279
ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)),
280
+ ProcessGroupMetrics.layer.pipe(Layer.provide(layerResourceTelemetry)),
281
);
282
283
const layerRelayClient = Layer.unwrap(
docs/internals/resource-telemetry.md
+7
-2
@@ -11,8 +11,13 @@ protocol. A missing or failed collector leaves the server running.
11
The native child owns sampling and bounded in-memory history. It samples on a timer
12
only while diagnostics has live subscribers. With none, the server sends a sample
13
interval of 0 and the child samples only on request, so an idle server never walks
14
-the process table. History therefore covers only time with a live subscriber, plus
15
-on-demand samples. Consuming host power for background scheduling must not retain
14
+the process table. The exception is OTLP metrics export: a server exporting metrics
15
+requests one sample a minute and exports process totals by group
16
+([`ProcessGroupMetrics`](../../apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts)),
17
+because the user asked for continuous telemetry. Groups are a fixed set named after
18
+the server's direct children, never process IDs or threads, so the series count
19
+stays bounded as agents come and go. History therefore covers only time with a live
20
+subscriber, plus on-demand and export samples. Consuming host power for background scheduling must not retain
21
live diagnostics. There is no telemetry database or recurring shell-probe fallback.
22
23
History has independent bounds for age, snapshot count, process rows, and retained