@devries-cameron20 / t3code Mirror · read-only / commits / daa182b816
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 17:00 UTC

fix(server): surface why process group metrics are not exported (#23969)

Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>

Yordis Prieto committed Oct 25, 2026 at 14:27 UTC daa182b8160b4acb4c3d518f28c4c2528e63d3a7
2 files changed +86 -5
apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts
+63 -2
@@ -1,7 +1,17 @@
1 import { assert, describe, it } from "@effect/vitest";
2 -import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2 +import type { ResourceTelemetryProcess, ResourceTelemetrySnapshot } from "@t3tools/contracts";
3 +import * as Effect from "effect/Effect";
4 +import * as Layer from "effect/Layer";
5 +import * as Logger from "effect/Logger";
6 +import * as References from "effect/References";
7 +import * as Ref from "effect/Ref";
8 +import * as TestClock from "effect/testing/TestClock";
9
4 -import { groupProcesses } from "./ProcessGroupMetrics.ts";
10 +import * as ServerConfig from "../config.ts";
11 +import * as ProcessGroupMetrics from "./ProcessGroupMetrics.ts";
12 +import * as ResourceTelemetry from "./ResourceTelemetry.ts";
13 +
14 +const { groupProcesses } = ProcessGroupMetrics;
15
16 const process = (
17 pid: number,
@@ -57,3 +67,54 @@ describe("groupProcesses", () => {
67 });
68 });
69 });
70 +
71 +describe("ProcessGroupMetrics.layer", () => {
72 + it.effect("logs the first failed sample and the recovery, not every repeat", () => {
73 + const logs: Array<string> = [];
74 + const logger = Logger.make(({ logLevel, message }) => {
75 + logs.push(`${logLevel.toUpperCase()} ${String([message].flat()[0])}`);
76 + });
77 + return Effect.gen(function* () {
78 + const outcomes = yield* Ref.make<ReadonlyArray<"fail" | "ok">>(["fail", "fail", "ok"]);
79 + const refresh = Ref.modify(outcomes, ([next = "ok", ...rest]) => [next, rest] as const).pipe(
80 + Effect.flatMap((next) =>
81 + next === "fail"
82 + ? Effect.fail(
83 + new ResourceTelemetry.ResourceTelemetryRefreshFailed({
84 + operation: "refresh",
85 + cause: new Error("Resource monitor binary was not found for darwin/arm64."),
86 + }),
87 + )
88 + : Effect.succeed({ processes: [] } as unknown as ResourceTelemetrySnapshot),
89 + ),
90 + );
91 + const telemetry = { refresh } as unknown as ResourceTelemetry.ResourceTelemetry["Service"];
92 +
93 + yield* Layer.build(
94 + ProcessGroupMetrics.layer.pipe(
95 + Layer.provide(Layer.succeed(ResourceTelemetry.ResourceTelemetry, telemetry)),
96 + Layer.provide(
97 + ServerConfig.layer({
98 + otlpMetricsUrl: "http://localhost:4318/v1/metrics",
99 + } as ServerConfig.ServerConfig["Service"]),
100 + ),
101 + ),
102 + );
103 + yield* TestClock.adjust("3 minutes");
104 +
105 + assert.deepStrictEqual(logs, [
106 + "WARN process group metrics are not exported: sample failed",
107 + "DEBUG process group metrics sample failed",
108 + "INFO process group metrics sample recovered",
109 + ]);
110 + }).pipe(
111 + Effect.scoped,
112 + Effect.provide(
113 + Layer.mergeAll(
114 + Logger.layer([logger], { mergeWithExisting: false }),
115 + Layer.succeed(References.MinimumLogLevel, "Debug"),
116 + ),
117 + ),
118 + );
119 + });
120 +});
apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts
+23 -3
@@ -1,7 +1,9 @@
1 import type { ResourceTelemetryProcess } from "@t3tools/contracts";
2 +import * as Cause from "effect/Cause";
3 import * as Effect from "effect/Effect";
4 import * as Layer from "effect/Layer";
5 import * as Metric from "effect/Metric";
6 +import * as Ref from "effect/Ref";
7
8 import * as ServerConfig from "../config.ts";
9 import { metricAttributes } from "../observability/Metrics.ts";
@@ -113,10 +115,28 @@ export const layer = Layer.effectDiscard(
115 yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
116 }
117 });
116 - yield* sample().pipe(
117 - Effect.catch((error) =>
118 - Effect.logDebug("process group metrics sample failed", { error: error.message }),
118 + // A failing collector would otherwise leave these series silently absent, so the first
119 + // failure and the recovery are logged; the repeats stay at debug.
120 + const failing = yield* Ref.make(false);
121 + const recordSuccess = Ref.getAndSet(failing, false).pipe(
122 + Effect.flatMap((wasFailing) =>
123 + wasFailing ? Effect.logInfo("process group metrics sample recovered") : Effect.void,
124 ),
125 + );
126 + const recordFailure = (error: ResourceTelemetry.ResourceTelemetryRefreshFailed) => {
127 + const cause = Cause.fail(error);
128 + return Ref.getAndSet(failing, true).pipe(
129 + Effect.flatMap((wasFailing) =>
130 + wasFailing
131 + ? Effect.logDebug("process group metrics sample failed", cause)
132 + : Effect.logWarning("process group metrics are not exported: sample failed", cause),
133 + ),
134 + Effect.annotateLogs({ operation: error.operation }),
135 + );
136 + };
137 + yield* sample().pipe(
138 + Effect.andThen(recordSuccess),
139 + Effect.catch(recordFailure),
140 Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
141 Effect.forever,
142 Effect.forkScoped,