The best way to control Claude Code, Codex and more from any computer or phone
fix(server): failed OTLP exports no longer go unrecorded (#38091)
Co-authored-by: Martin Bergo <14320143+mInrOz@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Martin Bergo committed
Nov 17, 2026 at 13:01 UTC
79aa7a7b5c56d9c9e65df4d3d2a0259b1ac37612
6 files changed
+158
-2
apps/server/src/observability/Metrics.ts
+11
@@ -35,6 +35,17 @@ export const orchestrationEffectQueueWait = Metric.timer("t3_orchestration_effec
35
"Time from an orchestration effect's temporal availability until claim, including same-thread blocking.",
36
});
37
38
+// A failed batch (transport errors and 408/429/500/502/503/504 after 3 retries) is dropped with
39
+// every push for the next 60 s, logged only at Debug. Incremental exports it as a monotonic counter.
40
+export const otlpExportAttemptFailuresTotal = Metric.counter(
41
+ "t3_otlp_export_attempt_failures_total",
42
+ {
43
+ description:
44
+ "Failed OTLP export attempts by signal and reason. Retries count too, so a dropped batch counts up to four.",
45
+ incremental: true,
46
+ },
47
+);
48
+
49
export const providerSessionsTotal = Metric.counter("t3_provider_sessions_total", {
50
description: "Total provider session lifecycle operations.",
51
});
apps/server/src/observability/Observability.ts
+6
-2
@@ -17,6 +17,7 @@ import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.t
17
import * as ServerLogger from "../serverLogger.ts";
18
import * as BrowserTraceCollector from "./BrowserTraceCollector.ts";
19
import * as HeapAttribution from "./HeapAttribution.ts";
20
+import * as OtlpExportFailures from "./OtlpExportFailures.ts";
21
22
export const layer = Layer.unwrap(
23
Effect.gen(function* () {
@@ -61,7 +62,7 @@ export const layer = Layer.unwrap(
62
exportInterval: `${traces.exportIntervalMs} millis`,
63
headers: traces.headers,
64
resource,
64
- });
65
+ }).pipe(Effect.provide(OtlpExportFailures.layer("traces")));
66
67
const tracer = yield* makeLocalFileTracer({
68
filePath: config.serverTracePath,
@@ -90,7 +91,10 @@ export const layer = Layer.unwrap(
91
exportInterval: `${metrics.exportIntervalMs} millis`,
92
headers: metrics.headers,
93
resource,
93
- }).pipe(Layer.provide(SharedObservability.layerOtlpSerialization(metrics.protocol)));
94
+ }).pipe(
95
+ Layer.provide(OtlpExportFailures.layer("metrics")),
96
+ Layer.provide(SharedObservability.layerOtlpSerialization(metrics.protocol)),
97
+ );
98
99
const layerOtelWarnings = Layer.effectDiscard(
100
Effect.forEach(config.otelEnvironment.warnings, (warning) => Effect.logWarning(warning)),
apps/server/src/observability/OtlpExportFailures.test.ts
new
+78
@@ -0,0 +1,78 @@
1
+import { assert, it } from "@effect/vitest";
2
+import * as Effect from "effect/Effect";
3
+import * as Layer from "effect/Layer";
4
+import * as Metric from "effect/Metric";
5
+import {
6
+ HttpBody,
7
+ HttpClient,
8
+ HttpClientError,
9
+ HttpClientRequest,
10
+ HttpClientResponse,
11
+} from "effect/http";
12
+import { OtlpSerialization } from "effect/observability";
13
+import { describe } from "vite-plus/test";
14
+
15
+import { otlpExportAttemptFailuresTotal } from "./Metrics.ts";
16
+import * as OtlpExportFailures from "./OtlpExportFailures.ts";
17
+
18
+const failures = (reason: string) =>
19
+ Metric.value(
20
+ Metric.withAttributes(otlpExportAttemptFailuresTotal, { signal: "logs", reason }),
21
+ ).pipe(Effect.map((state) => state.count));
22
+
23
+describe("OtlpExportFailures", () => {
24
+ it.effect("counts each failed attempt by reason and leaves successes alone", () => {
25
+ const outcomes: Array<number | "transport"> = [503, "transport", 200];
26
+ const client = HttpClient.make((request) => {
27
+ const outcome = outcomes.shift() ?? 200;
28
+ return outcome === "transport"
29
+ ? Effect.fail(
30
+ new HttpClientError.HttpClientError({
31
+ reason: new HttpClientError.TransportError({ request }),
32
+ }),
33
+ )
34
+ : Effect.succeed(
35
+ HttpClientResponse.fromWeb(request, new Response(null, { status: outcome })),
36
+ );
37
+ });
38
+ const serialization = OtlpSerialization.OtlpSerialization.of({
39
+ traces: () => HttpBody.empty,
40
+ metrics: () => HttpBody.empty,
41
+ logs: () => {
42
+ throw new Error("unencodable attribute");
43
+ },
44
+ });
45
+ const layer = OtlpExportFailures.layer("logs").pipe(
46
+ Layer.provide(Layer.succeed(HttpClient.HttpClient, client)),
47
+ Layer.provide(Layer.succeed(OtlpSerialization.OtlpSerialization, serialization)),
48
+ );
49
+
50
+ return Effect.gen(function* () {
51
+ const counted = yield* HttpClient.HttpClient;
52
+ const encode = yield* OtlpSerialization.OtlpSerialization;
53
+ const request = HttpClientRequest.post("http://localhost:4318/v1/logs");
54
+ yield* counted.execute(request);
55
+ yield* Effect.flip(counted.execute(request));
56
+ yield* counted.execute(request);
57
+ assert.throws(() => encode.logs({ resourceLogs: [] }), "unencodable attribute");
58
+
59
+ const transport = yield* Metric.value(
60
+ Metric.withAttributes(otlpExportAttemptFailuresTotal, {
61
+ signal: "logs",
62
+ reason: "transport",
63
+ }),
64
+ );
65
+ // OTLP marks the sum monotonic only for an incremental counter.
66
+ assert.isTrue(transport.incremental);
67
+ assert.deepStrictEqual(
68
+ [
69
+ yield* failures("status_503"),
70
+ yield* failures("transport"),
71
+ yield* failures("encode"),
72
+ yield* failures("status_200"),
73
+ ],
74
+ [1, 1, 1, 0],
75
+ );
76
+ }).pipe(Effect.provide(layer), Effect.provideService(Metric.MetricRegistry, new Map()));
77
+ });
78
+});
apps/server/src/observability/OtlpExportFailures.ts
new
+54
@@ -0,0 +1,54 @@
1
+import type { OtlpSignal } from "@t3tools/contracts";
2
+import * as Context from "effect/Context";
3
+import * as Effect from "effect/Effect";
4
+import * as Layer from "effect/Layer";
5
+import * as Metric from "effect/Metric";
6
+import { HttpClient } from "effect/http";
7
+import type { HttpBody } from "effect/http";
8
+import { OtlpSerialization } from "effect/observability";
9
+
10
+import { otlpExportAttemptFailuresTotal } from "./Metrics.ts";
11
+
12
+/**
13
+ * Re-provides the HttpClient and OtlpSerialization one OTLP exporter uses, counting its failed
14
+ * attempts on the metrics channel. The exporter applies `filterStatusOk` and its retries outside
15
+ * this client, so a non-2xx arrives here as a response and each retry is counted.
16
+ */
17
+export const layer = (signal: OtlpSignal) =>
18
+ Layer.effectContext(
19
+ Effect.gen(function* () {
20
+ const context = yield* Effect.context<never>();
21
+ const client = yield* HttpClient.HttpClient;
22
+ const serialization = yield* OtlpSerialization.OtlpSerialization;
23
+ const failures = (reason: string) =>
24
+ Metric.withAttributes(otlpExportAttemptFailuresTotal, { signal, reason });
25
+ // Encoding runs synchronously inside the exporter, before any request exists.
26
+ const counted =
27
+ <A>(encode: (data: A) => HttpBody.HttpBody) =>
28
+ (data: A) => {
29
+ try {
30
+ return encode(data);
31
+ } catch (error) {
32
+ failures("encode").updateUnsafe(1, context);
33
+ throw error;
34
+ }
35
+ };
36
+ return Context.make(
37
+ HttpClient.HttpClient,
38
+ client.pipe(
39
+ HttpClient.tap((response) =>
40
+ response.status >= 200 && response.status < 300
41
+ ? Effect.void
42
+ : Metric.update(failures(`status_${response.status}`), 1),
43
+ ),
44
+ HttpClient.tapError(() => Metric.update(failures("transport"), 1)),
45
+ ),
46
+ ).pipe(
47
+ Context.add(OtlpSerialization.OtlpSerialization, {
48
+ traces: counted(serialization.traces),
49
+ metrics: counted(serialization.metrics),
50
+ logs: counted(serialization.logs),
51
+ }),
52
+ );
53
+ }),
54
+ );
apps/server/src/serverLogger.ts
+2
@@ -10,6 +10,7 @@ import * as OtlpExporter from "effect/observability/OtlpExporter";
10
import * as OtlpLogger from "effect/observability/OtlpLogger";
11
12
import * as ServerConfig from "./config.ts";
13
+import * as OtlpExportFailures from "./observability/OtlpExportFailures.ts";
14
15
// HTTP errors logged with --log-ws-events carry the request URL and its connection ticket.
16
const redactingCredentials = <Output>(logger: Logger.Logger<unknown, Output>) =>
@@ -73,6 +74,7 @@ export const layer = Effect.gen(function* () {
74
{ mergeWithExisting: false },
75
).pipe(
76
Layer.provide(OtlpExporter.layerFlusher),
77
+ Layer.provide(OtlpExportFailures.layer("logs")),
78
Layer.provide(SharedObservability.layerOtlpSerialization(logs.protocol)),
79
);
80
docs/operations/observability.md
+7
@@ -475,6 +475,13 @@ Usually one of these is true:
475
476
If the local NDJSON file is updating, local tracing is working. The problem is almost always OTLP export configuration or process startup.
477
478
+If metrics arrive but logs or traces have gaps, read `t3_otlp_export_attempt_failures_total` by
479
+`signal` and `reason` (`status_<code>`, `transport`, `encode`). The exporter drops a batch once it
480
+fails (transport errors and 408, 429, 500, 502, 503 and 504 after 3 retries; other statuses and
481
+encoding at once) and then drops that signal for 60 s, logging it only at Debug, so at the default
482
+log level this counter is the only record. Every retry counts, so one dropped batch reads as up to
483
+four.
484
+
485
## How To Think About Adding Tracing To Future Code
486
487
### Prefer Boundaries Over Tiny Helpers