@devries-cameron20 / t3code Mirror · read-only / commits / 86d130af6c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

feat(server): export server and process-group resource metrics (#22122)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 19, 2026 at 21:39 UTC 86d130af6c02a126515488ed6038b68906c35bf7
7 files changed +271 -3
apps/server/src/observability/EventLoopMonitor.test.ts
+26
@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";
4 import * as FileSystem from "effect/FileSystem";
5 import * as Layer from "effect/Layer";
6 import * as Logger from "effect/Logger";
7 +import * as Metric from "effect/Metric";
8 import * as Tracer from "effect/Tracer";
9 import * as TestClock from "effect/testing/TestClock";
10
@@ -24,6 +25,8 @@ const stalled: EventLoopMonitor.EventLoopReadings = {
25 involuntaryContextSwitches: 57,
26 },
27 rssBytes: 1536 * 1024 * 1024,
28 + heapUsedBytes: 512 * 1024 * 1024,
29 + elapsedMs: 30_000,
30 };
31 const quiet: EventLoopMonitor.EventLoopReadings = { ...stalled, delayMaxNs: ms(2_950) };
32
@@ -78,6 +81,29 @@ describe("EventLoopMonitor", () => {
81 }),
82 );
83
84 + it.effect("exports the server's memory, CPU and event loop from each sample", () =>
85 + Effect.gen(function* () {
86 + yield* monitor([quiet, stalled]);
87 + yield* TestClock.adjust("60 seconds");
88 +
89 + const gauges = Object.fromEntries(
90 + (yield* Metric.snapshot).flatMap((snapshot) =>
91 + snapshot.id.startsWith("t3_server_") && snapshot.type === "Gauge"
92 + ? [[snapshot.id, snapshot.state.value]]
93 + : [],
94 + ),
95 + );
96 + assert.deepStrictEqual(gauges, {
97 + t3_server_resident_bytes: 1536 * 1024 * 1024,
98 + t3_server_heap_used_bytes: 512 * 1024 * 1024,
99 + t3_server_event_loop_utilization: 0.176,
100 + t3_server_event_loop_delay_max: 4_950,
101 + // 406 ms of CPU over 30 s of wall time.
102 + t3_server_cpu_utilization: 406 / 30_000,
103 + });
104 + }).pipe(Effect.scoped),
105 + );
106 +
107 it.effect("keeps each stall as one line in a capped stall log", () =>
108 Effect.gen(function* () {
109 const fs = yield* FileSystem.FileSystem;
apps/server/src/observability/EventLoopMonitor.ts
+28 -1
@@ -5,11 +5,13 @@ import { RotatingFileSink } from "@t3tools/shared/logging";
5 import * as DateTime from "effect/DateTime";
6 import * as Effect from "effect/Effect";
7 import * as Layer from "effect/Layer";
8 +import * as Metric from "effect/Metric";
9 import * as Path from "effect/Path";
10 import * as Schema from "effect/Schema";
11 import type * as Scope from "effect/Scope";
12
13 import * as ServerConfig from "../config.ts";
14 +import * as Metrics from "./Metrics.ts";
15
16 // Node's delay histogram wakes a native timer every RESOLUTION_MS and records the
17 // gap between wakeups, so an idle loop reads about RESOLUTION_MS and a stall of S
@@ -34,6 +36,9 @@ export interface EventLoopReadings {
36 | "involuntaryContextSwitches"
37 >;
38 readonly rssBytes: number;
39 + readonly heapUsedBytes: number;
40 + /** Wall time the readings cover. */
41 + readonly elapsedMs: number;
42 }
43
44 // Node skips the first gap after a reset, so a stall right at a sample boundary can be missed.
@@ -48,11 +53,14 @@ const makeNodeSampler = Effect.gen(function* () {
53 );
54 let elu = NodePerfHooks.performance.eventLoopUtilization();
55 let usage = process.resourceUsage();
56 + let sampledAt = NodePerfHooks.performance.now();
57
58 // @effect-diagnostics-next-line returnEffectInGen:off - the read effect is the result.
59 return Effect.sync(() => {
60 const nextElu = NodePerfHooks.performance.eventLoopUtilization();
61 const nextUsage = process.resourceUsage();
62 + const nextSampledAt = NodePerfHooks.performance.now();
63 + const memory = process.memoryUsage();
64 const loop = NodePerfHooks.performance.eventLoopUtilization(nextElu, elu);
65 const readings: EventLoopReadings = {
66 delayMaxNs: histogram.max,
@@ -66,11 +74,14 @@ const makeNodeSampler = Effect.gen(function* () {
74 involuntaryContextSwitches:
75 nextUsage.involuntaryContextSwitches - usage.involuntaryContextSwitches,
76 },
69 - rssBytes: process.memoryUsage.rss(),
77 + rssBytes: memory.rss,
78 + heapUsedBytes: memory.heapUsed,
79 + elapsedMs: nextSampledAt - sampledAt,
80 };
81 histogram.reset();
82 elu = nextElu;
83 usage = nextUsage;
84 + sampledAt = nextSampledAt;
85 return readings;
86 });
87 });
@@ -83,6 +94,21 @@ export const stallMs = ({ delayMaxNs, activeMs }: EventLoopReadings) => {
94 return delayMs;
95 };
96
97 +// Untraced: it runs inside every 30-second tick, which already has the monitor's own span.
98 +const recordGauges = Effect.fnUntraced(function* (readings: EventLoopReadings) {
99 + const cpuMs = (readings.usage.userCPUTime + readings.usage.systemCPUTime) / 1000;
100 + yield* Metric.update(Metrics.serverResidentBytes, readings.rssBytes);
101 + yield* Metric.update(Metrics.serverHeapUsedBytes, readings.heapUsedBytes);
102 + yield* Metric.update(Metrics.serverEventLoopUtilization, readings.utilization);
103 + yield* Metric.update(
104 + Metrics.serverEventLoopDelayMax,
105 + Math.max(0, Math.round(readings.delayMaxNs / 1e6) - RESOLUTION_MS),
106 + );
107 + if (readings.elapsedMs > 0) {
108 + yield* Metric.update(Metrics.serverCpuUtilization, cpuMs / readings.elapsedMs);
109 + }
110 +});
111 +
112 const encodeStallLine = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
113
114 export interface StallLogOptions {
@@ -114,6 +140,7 @@ export const layerWith = (
140 );
141 const tick = Effect.gen(function* () {
142 const readings = yield* sample;
143 + yield* recordGauges(readings);
144 const delayMaxMs = stallMs(readings);
145 if (delayMaxMs === undefined) return;
146 const { utilization, usage, rssBytes } = readings;
apps/server/src/observability/Metrics.ts
+24
@@ -39,6 +39,30 @@ export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", {
39 description: "Time for the provider adapter to start a turn, not how long the turn runs.",
40 });
41
42 +// OTLP carries the unit, and Prometheus names the series by it: a unitless gauge gains `_ratio`.
43 +export const serverResidentBytes = Metric.gauge("t3_server_resident_bytes", {
44 + description: "Resident memory of the server process.",
45 + attributes: { unit: "By" },
46 +});
47 +
48 +export const serverHeapUsedBytes = Metric.gauge("t3_server_heap_used_bytes", {
49 + description: "JavaScript heap in use by the server process.",
50 + attributes: { unit: "By" },
51 +});
52 +
53 +export const serverCpuUtilization = Metric.gauge("t3_server_cpu_utilization", {
54 + description: "CPU time the server used per second over the last sample; 1 is one full core.",
55 +});
56 +
57 +export const serverEventLoopUtilization = Metric.gauge("t3_server_event_loop_utilization", {
58 + description: "Share of the last sample the server's event loop spent busy.",
59 +});
60 +
61 +export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_max", {
62 + description: "Longest event loop delay over the last sample.",
63 + attributes: { unit: "ms" },
64 +});
65 +
66 export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
67 description: "Total git commands executed by the server runtime.",
68 });
apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts new
+59
@@ -0,0 +1,59 @@
1 +import { assert, describe, it } from "@effect/vitest";
2 +import type { ResourceTelemetryProcess } from "@t3tools/contracts";
3 +
4 +import { groupProcesses } from "./ProcessGroupMetrics.ts";
5 +
6 +const process = (
7 + pid: number,
8 + ppid: number,
9 + name: string,
10 + category: ResourceTelemetryProcess["category"] = "server-child",
11 + command = name,
12 +) =>
13 + ({
14 + identity: { pid, startTimeMs: 1 },
15 + ppid,
16 + name,
17 + command,
18 + category,
19 + residentBytes: pid * 1_000,
20 + cpuPercent: pid,
21 + }) as ResourceTelemetryProcess;
22 +
23 +describe("groupProcesses", () => {
24 + it("sums each direct child of the server with its whole subtree", () => {
25 + const totals = groupProcesses([
26 + process(10, 1, "node", "server"),
27 + process(11, 10, "t3-resource-mon", "resource-monitor"),
28 + process(20, 10, "claude"),
29 + process(21, 20, "bash"),
30 + process(22, 21, "cargo"),
31 + process(30, 10, "codex"),
32 + process(40, 10, "zsh"),
33 + process(50, 10, "chrome-headless"),
34 + process(60, 10, "git", "server-child", "git checkout claude/fix"),
35 + process(70, 99, "orphan"),
36 + // An older Claude package runs as a script under node.
37 + process(
38 + 80,
39 + 10,
40 + "node",
41 + "server-child",
42 + "node /opt/node_modules/@anthropic-ai/claude-code/cli.js",
43 + ),
44 + process(90, 10, "node", "server-child", "node /srv/tools/report.js"),
45 + ]);
46 +
47 + assert.deepStrictEqual(Object.fromEntries(totals), {
48 + server: { residentBytes: 10_000, cpuPercent: 10, count: 1 },
49 + "resource-monitor": { residentBytes: 11_000, cpuPercent: 11, count: 1 },
50 + // The agent's shell and the build it ran belong to the agent, not the terminal group.
51 + claude: { residentBytes: 143_000, cpuPercent: 143, count: 4 },
52 + codex: { residentBytes: 30_000, cpuPercent: 30, count: 1 },
53 + terminal: { residentBytes: 40_000, cpuPercent: 40, count: 1 },
54 + browser: { residentBytes: 50_000, cpuPercent: 50, count: 1 },
55 + // `git` whose arguments mention claude, a parentless process, and a node script.
56 + other: { residentBytes: 220_000, cpuPercent: 220, count: 3 },
57 + });
58 + });
59 +});
apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts new
+125
@@ -0,0 +1,125 @@
1 +import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2 +import * as Effect from "effect/Effect";
3 +import * as Layer from "effect/Layer";
4 +import * as Metric from "effect/Metric";
5 +
6 +import * as ServerConfig from "../config.ts";
7 +import { metricAttributes } from "../observability/Metrics.ts";
8 +import * as ResourceTelemetry from "./ResourceTelemetry.ts";
9 +
10 +const SAMPLE_INTERVAL = "1 minute";
11 +
12 +const residentBytes = Metric.gauge("t3_process_group_resident_bytes", {
13 + description: "Resident memory of the server's processes, summed by group.",
14 + attributes: { unit: "By" },
15 +});
16 +const cpuPercent = Metric.gauge("t3_process_group_cpu_percent", {
17 + description: "CPU use of the server's processes, summed by group; 100 is one full core.",
18 + attributes: { unit: "%" },
19 +});
20 +const processCount = Metric.gauge("t3_process_group_count", {
21 + description: "Processes in each group.",
22 + attributes: { unit: "{process}" },
23 +});
24 +
25 +// Each direct child of the server names the group of its whole subtree. A fixed set keeps one
26 +// series per group per server, however many agents and processes come and go.
27 +const PROVIDER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
28 + [/codex/, "codex"],
29 + [/claude/, "claude"],
30 + [/cursor/, "cursor"],
31 + [/opencode/, "opencode"],
32 + [/grok/, "grok"],
33 + [/antigravity|gemini/, "antigravity"],
34 +];
35 +const OTHER_GROUPS: ReadonlyArray<readonly [RegExp, string]> = [
36 + [/chrom/, "browser"],
37 + [/^(?:ba|z|fi|da|c)?sh(?:\.exe)?$|^pwsh|^powershell|^cmd\.exe$/, "terminal"],
38 +];
39 +// A CLI shipped as a script runs under a JavaScript runtime, so its process name is the
40 +// runtime's (older Claude packages run `node .../@anthropic-ai/claude-code/cli.js`). Only those
41 +// are matched by command line; any other name is matched by name alone, so a shell or `git`
42 +// whose arguments mention a provider stays in its own group.
43 +const SCRIPT_RUNTIME = /^(?:node|bun|deno)(?:\.exe)?$/;
44 +
45 +const match = (groups: ReadonlyArray<readonly [RegExp, string]>, text: string) =>
46 + groups.find(([pattern]) => pattern.test(text))?.[1];
47 +
48 +const childGroup = (name: string, command: string) => {
49 + const lowerName = name.toLowerCase();
50 + if (SCRIPT_RUNTIME.test(lowerName))
51 + return match(PROVIDER_GROUPS, command.toLowerCase()) ?? "other";
52 + return match(PROVIDER_GROUPS, lowerName) ?? match(OTHER_GROUPS, lowerName) ?? "other";
53 +};
54 +
55 +export interface ProcessGroupTotals {
56 + readonly residentBytes: number;
57 + readonly cpuPercent: number;
58 + readonly count: number;
59 +}
60 +
61 +export function groupProcesses(
62 + processes: ReadonlyArray<ResourceTelemetryProcess>,
63 +): ReadonlyMap<string, ProcessGroupTotals> {
64 + const byPid = new Map(processes.map((process) => [process.identity.pid, process]));
65 + const serverPid = processes.find((process) => process.category === "server")?.identity.pid;
66 + const groupOf = (process: ResourceTelemetryProcess): string => {
67 + if (process.category === "server") return "server";
68 + if (process.category === "resource-monitor") return "resource-monitor";
69 + if (process.category.startsWith("electron-")) return "electron";
70 + let current = process;
71 + // Bounded by the table size, in case a reused PID makes the parent chain loop.
72 + for (let step = 0; step < processes.length; step++) {
73 + if (current.ppid === serverPid) return childGroup(current.name, current.command);
74 + const parent = byPid.get(current.ppid);
75 + if (parent === undefined) break;
76 + current = parent;
77 + }
78 + return "other";
79 + };
80 + const totals = new Map<string, ProcessGroupTotals>();
81 + for (const process of processes) {
82 + const group = groupOf(process);
83 + const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
84 + totals.set(group, {
85 + residentBytes: total.residentBytes + process.residentBytes,
86 + cpuPercent: total.cpuPercent + process.cpuPercent,
87 + count: total.count + 1,
88 + });
89 + }
90 + return totals;
91 +}
92 +
93 +/**
94 + * Exports process totals by group while OTLP metrics export is on. Sampling walks the process
95 + * table, which an idle server otherwise never does, so only a server exporting metrics pays for
96 + * it, once a minute.
97 + */
98 +export const layer = Layer.effectDiscard(
99 + Effect.gen(function* () {
100 + const config = yield* ServerConfig.ServerConfig;
101 + if (config.otlpMetricsUrl === undefined) return;
102 + const telemetry = yield* ResourceTelemetry.ResourceTelemetry;
103 + // A group that empties reads zero rather than holding its last value.
104 + const seen = new Set<string>();
105 + const sample = Effect.fn("ProcessGroupMetrics.sample")(function* () {
106 + const totals = groupProcesses((yield* telemetry.refresh).processes);
107 + for (const group of totals.keys()) seen.add(group);
108 + for (const group of seen) {
109 + const total = totals.get(group) ?? { residentBytes: 0, cpuPercent: 0, count: 0 };
110 + const attributes = metricAttributes({ group });
111 + yield* Metric.update(Metric.withAttributes(residentBytes, attributes), total.residentBytes);
112 + yield* Metric.update(Metric.withAttributes(cpuPercent, attributes), total.cpuPercent);
113 + yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
114 + }
115 + });
116 + yield* sample().pipe(
117 + Effect.catch((error) =>
118 + Effect.logDebug("process group metrics sample failed", { error: error.message }),
119 + ),
120 + Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
121 + Effect.forever,
122 + Effect.forkScoped,
123 + );
124 + }),
125 +);
apps/server/src/server.ts
+2
@@ -163,6 +163,7 @@ import * as DesktopTelemetryReceiver from "./resourceTelemetry/DesktopTelemetryR
163 import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClient.ts";
164 import * as ResourceAttribution from "./resourceTelemetry/ResourceAttribution.ts";
165 import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinary.ts";
166 +import * as ProcessGroupMetrics from "./resourceTelemetry/ProcessGroupMetrics.ts";
167 import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
168 import * as AntigravityUsage from "./provider/Drivers/AntigravityUsage.ts";
169 import * as CursorAccountReader from "@t3tools/provider-cursor/server/CursorAccountReader";
@@ -276,6 +277,7 @@ const layerResourceDiagnostics = Layer.mergeAll(
277 layerResourceTelemetry,
278 ProcessDiagnostics.layer.pipe(Layer.provide(layerResourceTelemetry)),
279 ProcessResourceMonitor.layer.pipe(Layer.provide(layerResourceTelemetry)),
280 + ProcessGroupMetrics.layer.pipe(Layer.provide(layerResourceTelemetry)),
281 );
282
283 const layerRelayClient = Layer.unwrap(
docs/internals/resource-telemetry.md
+7 -2
@@ -11,8 +11,13 @@ protocol. A missing or failed collector leaves the server running.
11 The native child owns sampling and bounded in-memory history. It samples on a timer
12 only while diagnostics has live subscribers. With none, the server sends a sample
13 interval of 0 and the child samples only on request, so an idle server never walks
14 -the process table. History therefore covers only time with a live subscriber, plus
15 -on-demand samples. Consuming host power for background scheduling must not retain
14 +the process table. The exception is OTLP metrics export: a server exporting metrics
15 +requests one sample a minute and exports process totals by group
16 +([`ProcessGroupMetrics`](../../apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts)),
17 +because the user asked for continuous telemetry. Groups are a fixed set named after
18 +the server's direct children, never process IDs or threads, so the series count
19 +stays bounded as agents come and go. History therefore covers only time with a live
20 +subscriber, plus on-demand and export samples. Consuming host power for background scheduling must not retain
21 live diagnostics. There is no telemetry database or recurring shell-probe fallback.
22
23 History has independent bounds for age, snapshot count, process rows, and retained