@devries-cameron20 / t3code-1 / commits / 576a40fa53
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): telemetry keeps a failed batch for 30 minutes, not 1.5 (#25474)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 26, 2026 at 16:05 UTC 576a40fa530d0883068377aec83332ffbfb42629
2 files changed +86 -10
apps/server/src/telemetry/AnalyticsService.test.ts
+55 -4
@@ -1,6 +1,7 @@
1 import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
2 import * as NodeServices from "@effect/platform-node/NodeServices";
3 import { assert, it } from "@effect/vitest";
4 +import * as Clock from "effect/Clock";
5 import * as ConfigProvider from "effect/ConfigProvider";
6 import * as Effect from "effect/Effect";
7 import * as Layer from "effect/Layer";
@@ -80,6 +81,24 @@ const respondWithStatus =
81 (request) =>
82 Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status })));
83
84 +/**
85 + * Answers 503 for `minutes` after the first send, then 204. `answers` holds each
86 + * answer with its time in ms after the first send (the test clock runs on across tests).
87 + */
88 +const failingFor = (minutes: number) => {
89 + const answers: Array<{ readonly at: number; readonly status: number }> = [];
90 + let firstSend: number | undefined;
91 + const respond: Respond = (request) =>
92 + Effect.flatMap(Clock.currentTimeMillis, (now) => {
93 + firstSend ??= now;
94 + const at = now - firstSend;
95 + const status = at < minutes * 60_000 ? 503 : 204;
96 + answers.push({ at, status });
97 + return respondWithStatus(status)(request);
98 + });
99 + return { respond, answers };
100 +};
101 +
102 const recordTwentyEvents = Effect.gen(function* () {
103 const analytics = yield* AnalyticsService.AnalyticsService;
104 for (let index = 0; index < 20; index += 1) {
@@ -88,11 +107,12 @@ const recordTwentyEvents = Effect.gen(function* () {
107 });
108
109 /**
91 - * Runs `record` (20 events by default), then lets ten minutes of background
110 + * Runs `record` (20 events by default), then lets `minutes` of background
111 * flushes run against a client that answers every send with `respond`.
112 * Returns the sent batches.
113 */
95 -const sendForTenMinutes = (
114 +const sendFor = (
115 + minutes: number,
116 respond: Respond,
117 record: Effect.Effect<
118 void,
@@ -137,7 +157,7 @@ const sendForTenMinutes = (
157
158 yield* Effect.gen(function* () {
159 yield* record;
140 - for (let second = 0; second < 600; second += 1) {
160 + for (let second = 0; second < minutes * 60; second += 1) {
161 yield* TestClock.adjust("1 second");
162 }
163 }).pipe(Effect.provide(runtimeLayer));
@@ -147,7 +167,8 @@ const sendForTenMinutes = (
167 it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
168 it.effect("client owner reports cannot use up the server's owner cap", () =>
169 Effect.gen(function* () {
150 - const batches = yield* sendForTenMinutes(
170 + const batches = yield* sendFor(
171 + 10,
172 respondWithStatus(204),
173 Effect.gen(function* () {
174 const analytics = yield* AnalyticsService.AnalyticsService;
@@ -170,6 +191,36 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
191 }),
192 );
193
194 + it.effect("keeps a failed batch through a 20-minute outage and sends it once after", () =>
195 + Effect.gen(function* () {
196 + const ingest = failingFor(20);
197 + const batches = yield* sendFor(40, ingest.respond);
198 + const statuses = ingest.answers.map(({ status }) => status);
199 + assert.equal(statuses.at(-1), 204);
200 + assert.equal(statuses.filter((status) => status === 204).length, 1);
201 + const sentUuids = new Set(batches.map((batch) => batch.map(({ uuid }) => uuid).join()));
202 + assert.equal(sentUuids.size, 1);
203 + assert.equal(batches[0]?.length, 20);
204 + }),
205 + );
206 +
207 + it.effect("drops a batch that keeps failing for 30 minutes", () =>
208 + Effect.gen(function* () {
209 + const ingest = failingFor(Infinity);
210 + yield* sendFor(60, ingest.respond);
211 + const lastSend = ingest.answers.at(-1)?.at ?? 0;
212 + assert.isAtLeast(lastSend, 30 * 60_000);
213 + assert.isBelow(lastSend, 36 * 60_000);
214 + }),
215 + );
216 +
217 + it.effect("drops a batch the endpoint rejects without sending it again", () =>
218 + Effect.gen(function* () {
219 + const batches = yield* sendFor(10, respondWithStatus(400));
220 + assert.equal(batches.length, 1);
221 + }),
222 + );
223 +
224 it.effect("flush drains all buffered events across multiple batches", () =>
225 Effect.gen(function* () {
226 const capturedRequests: Array<RecordedBatchRequest> = [];
apps/server/src/telemetry/AnalyticsService.ts
+31 -6
@@ -32,6 +32,7 @@ interface BufferedAnalyticsEvent {
32 interface DeliveryState {
33 readonly failedBatch: ReadonlyArray<BufferedAnalyticsEvent>;
34 readonly batchAttempts: number;
35 + readonly firstFailedAt: number;
36 readonly failures: number;
37 readonly retryAt: number;
38 }
@@ -40,7 +41,10 @@ interface DeliveryState {
41 const FLUSH_INTERVAL_MS = 10_000;
42 // A hung send would hold the flush lock, and with it the shutdown flush.
43 const SEND_TIMEOUT = "10 seconds";
43 -const MAX_BATCH_ATTEMPTS = 5;
44 +// A failed batch is tried again until it has failed for this long. Ingest answers 503 when it
45 +// cannot store a batch, and a broken Cloudflare location can take about 5 minutes to fail over.
46 +// The waiting batch holds newer events back; `maxBufferedEvents` bounds them meanwhile.
47 +const MAX_BATCH_RETRY_MS = 30 * 60_000;
48 const RETRY_BASE_DELAY_MS = 2_000;
49 // Clients can report any number of owner ids; past this many per observer, new ones
50 // are not sent. Each observer has its own cap, so client reports never crowd out the
@@ -133,6 +137,7 @@ export const make = Effect.gen(function* () {
137 const deliveryRef = yield* Ref.make<DeliveryState>({
138 failedBatch: [],
139 batchAttempts: 0,
140 + firstFailedAt: 0,
141 failures: 0,
142 retryAt: 0,
143 });
@@ -232,27 +237,47 @@ export const make = Effect.gen(function* () {
237
238 const sent = yield* Effect.result(sendBatch(batch));
239 if (Result.isSuccess(sent)) {
235 - yield* Ref.set(deliveryRef, { failedBatch: [], batchAttempts: 0, failures: 0, retryAt: 0 });
240 + yield* Ref.set(deliveryRef, {
241 + failedBatch: [],
242 + batchAttempts: 0,
243 + firstFailedAt: 0,
244 + failures: 0,
245 + retryAt: 0,
246 + });
247 continue;
248 }
249
250 + const now = yield* Clock.currentTimeMillis;
251 const failures = delivery.failures + 1;
252 const batchAttempts = delivery.batchAttempts + 1;
241 - const retryAt = (yield* Clock.currentTimeMillis) + retryDelayMs(failures, yield* Random.next);
253 + const firstFailedAt = delivery.batchAttempts === 0 ? now : delivery.firstFailedAt;
254 + const retryAt = now + retryDelayMs(failures, yield* Random.next);
255 const error = sent.failure;
256 const rejected =
257 error._tag === "HttpClientError" &&
258 error.reason._tag === "StatusCodeError" &&
259 REJECTED_BATCH_STATUSES.has(error.reason.response.status);
247 - if (!rejected && batchAttempts < MAX_BATCH_ATTEMPTS) {
248 - yield* Ref.set(deliveryRef, { failedBatch: batch, batchAttempts, failures, retryAt });
260 + if (!rejected && now - firstFailedAt < MAX_BATCH_RETRY_MS) {
261 + yield* Ref.set(deliveryRef, {
262 + failedBatch: batch,
263 + batchAttempts,
264 + firstFailedAt,
265 + failures,
266 + retryAt,
267 + });
268 yield* Effect.logDebug("Failed to send telemetry batch; will retry", {
269 attempt: batchAttempts,
270 cause: error,
271 });
272 return;
273 }
255 - yield* Ref.set(deliveryRef, { failedBatch: [], batchAttempts: 0, failures, retryAt });
274 + yield* Ref.set(deliveryRef, {
275 + failedBatch: [],
276 + batchAttempts: 0,
277 + firstFailedAt: 0,
278 + failures,
279 + retryAt,
280 + });
281 yield* Effect.logWarning(
282 rejected
283 ? "Dropped telemetry batch the endpoint rejected"