@devries-cameron20 / t3code Mirror · read-only / commits / 79aa7a7b5c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 08:30 UTC

fix(server): failed OTLP exports no longer go unrecorded (#38091)

Co-authored-by: Martin Bergo <14320143+mInrOz@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Martin Bergo committed Nov 17, 2026 at 13:01 UTC 79aa7a7b5c56d9c9e65df4d3d2a0259b1ac37612
6 files changed +158 -2
apps/server/src/observability/Metrics.ts
+11
@@ -35,6 +35,17 @@ export const orchestrationEffectQueueWait = Metric.timer("t3_orchestration_effec
35 "Time from an orchestration effect's temporal availability until claim, including same-thread blocking.",
36 });
37
38 +// A failed batch (transport errors and 408/429/500/502/503/504 after 3 retries) is dropped with
39 +// every push for the next 60 s, logged only at Debug. Incremental exports it as a monotonic counter.
40 +export const otlpExportAttemptFailuresTotal = Metric.counter(
41 + "t3_otlp_export_attempt_failures_total",
42 + {
43 + description:
44 + "Failed OTLP export attempts by signal and reason. Retries count too, so a dropped batch counts up to four.",
45 + incremental: true,
46 + },
47 +);
48 +
49 export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
50 description: "Total provider session lifecycle operations.",
51 });
apps/server/src/observability/Observability.ts
+6 -2
@@ -17,6 +17,7 @@ import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.t
17 import * as ServerLogger from "../serverLogger.ts";
18 import * as BrowserTraceCollector from "./BrowserTraceCollector.ts";
19 import * as HeapAttribution from "./HeapAttribution.ts";
20 +import * as OtlpExportFailures from "./OtlpExportFailures.ts";
21
22 export const layer = Layer.unwrap(
23 Effect.gen(function* () {
@@ -61,7 +62,7 @@ export const layer = Layer.unwrap(
62 exportInterval: `${traces.exportIntervalMs} millis`,
63 headers: traces.headers,
64 resource,
64 - });
65 + }).pipe(Effect.provide(OtlpExportFailures.layer("traces")));
66
67 const tracer = yield* makeLocalFileTracer({
68 filePath: config.serverTracePath,
@@ -90,7 +91,10 @@ export const layer = Layer.unwrap(
91 exportInterval: `${metrics.exportIntervalMs} millis`,
92 headers: metrics.headers,
93 resource,
93 - }).pipe(Layer.provide(SharedObservability.layerOtlpSerialization(metrics.protocol)));
94 + }).pipe(
95 + Layer.provide(OtlpExportFailures.layer("metrics")),
96 + Layer.provide(SharedObservability.layerOtlpSerialization(metrics.protocol)),
97 + );
98
99 const layerOtelWarnings = Layer.effectDiscard(
100 Effect.forEach(config.otelEnvironment.warnings, (warning) => Effect.logWarning(warning)),
apps/server/src/observability/OtlpExportFailures.test.ts new
+78
@@ -0,0 +1,78 @@
1 +import { assert, it } from "@effect/vitest";
2 +import * as Effect from "effect/Effect";
3 +import * as Layer from "effect/Layer";
4 +import * as Metric from "effect/Metric";
5 +import {
6 + HttpBody,
7 + HttpClient,
8 + HttpClientError,
9 + HttpClientRequest,
10 + HttpClientResponse,
11 +} from "effect/http";
12 +import { OtlpSerialization } from "effect/observability";
13 +import { describe } from "vite-plus/test";
14 +
15 +import { otlpExportAttemptFailuresTotal } from "./Metrics.ts";
16 +import * as OtlpExportFailures from "./OtlpExportFailures.ts";
17 +
18 +const failures = (reason: string) =>
19 + Metric.value(
20 + Metric.withAttributes(otlpExportAttemptFailuresTotal, { signal: "logs", reason }),
21 + ).pipe(Effect.map((state) => state.count));
22 +
23 +describe("OtlpExportFailures", () => {
24 + it.effect("counts each failed attempt by reason and leaves successes alone", () => {
25 + const outcomes: Array<number | "transport"> = [503, "transport", 200];
26 + const client = HttpClient.make((request) => {
27 + const outcome = outcomes.shift() ?? 200;
28 + return outcome === "transport"
29 + ? Effect.fail(
30 + new HttpClientError.HttpClientError({
31 + reason: new HttpClientError.TransportError({ request }),
32 + }),
33 + )
34 + : Effect.succeed(
35 + HttpClientResponse.fromWeb(request, new Response(null, { status: outcome })),
36 + );
37 + });
38 + const serialization = OtlpSerialization.OtlpSerialization.of({
39 + traces: () => HttpBody.empty,
40 + metrics: () => HttpBody.empty,
41 + logs: () => {
42 + throw new Error("unencodable attribute");
43 + },
44 + });
45 + const layer = OtlpExportFailures.layer("logs").pipe(
46 + Layer.provide(Layer.succeed(HttpClient.HttpClient, client)),
47 + Layer.provide(Layer.succeed(OtlpSerialization.OtlpSerialization, serialization)),
48 + );
49 +
50 + return Effect.gen(function* () {
51 + const counted = yield* HttpClient.HttpClient;
52 + const encode = yield* OtlpSerialization.OtlpSerialization;
53 + const request = HttpClientRequest.post("http://localhost:4318/v1/logs");
54 + yield* counted.execute(request);
55 + yield* Effect.flip(counted.execute(request));
56 + yield* counted.execute(request);
57 + assert.throws(() => encode.logs({ resourceLogs: [] }), "unencodable attribute");
58 +
59 + const transport = yield* Metric.value(
60 + Metric.withAttributes(otlpExportAttemptFailuresTotal, {
61 + signal: "logs",
62 + reason: "transport",
63 + }),
64 + );
65 + // OTLP marks the sum monotonic only for an incremental counter.
66 + assert.isTrue(transport.incremental);
67 + assert.deepStrictEqual(
68 + [
69 + yield* failures("status_503"),
70 + yield* failures("transport"),
71 + yield* failures("encode"),
72 + yield* failures("status_200"),
73 + ],
74 + [1, 1, 1, 0],
75 + );
76 + }).pipe(Effect.provide(layer), Effect.provideService(Metric.MetricRegistry, new Map()));
77 + });
78 +});
apps/server/src/observability/OtlpExportFailures.ts new
+54
@@ -0,0 +1,54 @@
1 +import type { OtlpSignal } from "@t3tools/contracts";
2 +import * as Context from "effect/Context";
3 +import * as Effect from "effect/Effect";
4 +import * as Layer from "effect/Layer";
5 +import * as Metric from "effect/Metric";
6 +import { HttpClient } from "effect/http";
7 +import type { HttpBody } from "effect/http";
8 +import { OtlpSerialization } from "effect/observability";
9 +
10 +import { otlpExportAttemptFailuresTotal } from "./Metrics.ts";
11 +
12 +/**
13 + * Re-provides the HttpClient and OtlpSerialization one OTLP exporter uses, counting its failed
14 + * attempts on the metrics channel. The exporter applies `filterStatusOk` and its retries outside
15 + * this client, so a non-2xx arrives here as a response and each retry is counted.
16 + */
17 +export const layer = (signal: OtlpSignal) =>
18 + Layer.effectContext(
19 + Effect.gen(function* () {
20 + const context = yield* Effect.context<never>();
21 + const client = yield* HttpClient.HttpClient;
22 + const serialization = yield* OtlpSerialization.OtlpSerialization;
23 + const failures = (reason: string) =>
24 + Metric.withAttributes(otlpExportAttemptFailuresTotal, { signal, reason });
25 + // Encoding runs synchronously inside the exporter, before any request exists.
26 + const counted =
27 + <A>(encode: (data: A) => HttpBody.HttpBody) =>
28 + (data: A) => {
29 + try {
30 + return encode(data);
31 + } catch (error) {
32 + failures("encode").updateUnsafe(1, context);
33 + throw error;
34 + }
35 + };
36 + return Context.make(
37 + HttpClient.HttpClient,
38 + client.pipe(
39 + HttpClient.tap((response) =>
40 + response.status >= 200 && response.status < 300
41 + ? Effect.void
42 + : Metric.update(failures(`status_${response.status}`), 1),
43 + ),
44 + HttpClient.tapError(() => Metric.update(failures("transport"), 1)),
45 + ),
46 + ).pipe(
47 + Context.add(OtlpSerialization.OtlpSerialization, {
48 + traces: counted(serialization.traces),
49 + metrics: counted(serialization.metrics),
50 + logs: counted(serialization.logs),
51 + }),
52 + );
53 + }),
54 + );
apps/server/src/serverLogger.ts
+2
@@ -10,6 +10,7 @@ import * as OtlpExporter from "effect/observability/OtlpExporter";
10 import * as OtlpLogger from "effect/observability/OtlpLogger";
11
12 import * as ServerConfig from "./config.ts";
13 +import * as OtlpExportFailures from "./observability/OtlpExportFailures.ts";
14
15 // HTTP errors logged with --log-ws-events carry the request URL and its connection ticket.
16 const redactingCredentials = <Output>(logger: Logger.Logger<unknown, Output>) =>
@@ -73,6 +74,7 @@ export const layer = Effect.gen(function* () {
74 { mergeWithExisting: false },
75 ).pipe(
76 Layer.provide(OtlpExporter.layerFlusher),
77 + Layer.provide(OtlpExportFailures.layer("logs")),
78 Layer.provide(SharedObservability.layerOtlpSerialization(logs.protocol)),
79 );
80
docs/operations/observability.md
+7
@@ -475,6 +475,13 @@ Usually one of these is true:
475
476 If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.
477
478 +If metrics arrive but logs or traces have gaps, read `t3_otlp_export_attempt_failures_total` by
479 +`signal` and `reason` (`status_<code>`, `transport`, `encode`). The exporter drops a batch once it
480 +fails (transport errors and 408, 429, 500, 502, 503 and 504 after 3 retries; other statuses and
481 +encoding at once) and then drops that signal for 60 s, logging it only at Debug, so at the default
482 +log level this counter is the only record. Every retry counts, so one dropped batch reads as up to
483 +four.
484 +
485 ## How To Think About Adding Tracing To Future Code
486
487 ### Prefer Boundaries Over Tiny Helpers