The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 15:45 UTC
fix(server): telemetry keeps a failed batch for 30 minutes, not 1.5 (#25474)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Oct 26, 2026 at 16:05 UTC
576a40fa530d0883068377aec83332ffbfb42629
2 files changed
+86
-10
apps/server/src/telemetry/AnalyticsService.test.ts
+55
-4
@@ -1,6 +1,7 @@
1
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
2
import * as NodeServices from "@effect/platform-node/NodeServices";
3
import { assert, it } from "@effect/vitest";
4
+import * as Clock from "effect/Clock";
5
import * as ConfigProvider from "effect/ConfigProvider";
6
import * as Effect from "effect/Effect";
7
import * as Layer from "effect/Layer";
@@ -80,6 +81,24 @@ const respondWithStatus =
81
(request) =>
82
Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status })));
83
84
+/**
85
+ * Answers 503 for `minutes` after the first send, then 204. `answers` holds each
86
+ * answer with its time in ms after the first send (the test clock runs on across tests).
87
+ */
88
+const failingFor = (minutes: number) => {
89
+ const answers: Array<{ readonly at: number; readonly status: number }> = [];
90
+ let firstSend: number | undefined;
91
+ const respond: Respond = (request) =>
92
+ Effect.flatMap(Clock.currentTimeMillis, (now) => {
93
+ firstSend ??= now;
94
+ const at = now - firstSend;
95
+ const status = at < minutes * 60_000 ? 503 : 204;
96
+ answers.push({ at, status });
97
+ return respondWithStatus(status)(request);
98
+ });
99
+ return { respond, answers };
100
+};
101
+
102
const recordTwentyEvents = Effect.gen(function* () {
103
const analytics = yield* AnalyticsService.AnalyticsService;
104
for (let index = 0; index < 20; index += 1) {
@@ -88,11 +107,12 @@ const recordTwentyEvents = Effect.gen(function* () {
107
});
108
109
/**
91
- * Runs `record` (20 events by default), then lets ten minutes of background
110
+ * Runs `record` (20 events by default), then lets `minutes` of background
111
* flushes run against a client that answers every send with `respond`.
112
* Returns the sent batches.
113
*/
95
-const sendForTenMinutes = (
114
+const sendFor = (
115
+ minutes: number,
116
respond: Respond,
117
record: Effect.Effect<
118
void,
@@ -137,7 +157,7 @@ const sendForTenMinutes = (
157
158
yield* Effect.gen(function* () {
159
yield* record;
140
- for (let second = 0; second < 600; second += 1) {
160
+ for (let second = 0; second < minutes * 60; second += 1) {
161
yield* TestClock.adjust("1 second");
162
}
163
}).pipe(Effect.provide(runtimeLayer));
@@ -147,7 +167,8 @@ const sendForTenMinutes = (
167
it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
168
it.effect("client owner reports cannot use up the server's owner cap", () =>
169
Effect.gen(function* () {
150
- const batches = yield* sendForTenMinutes(
170
+ const batches = yield* sendFor(
171
+ 10,
172
respondWithStatus(204),
173
Effect.gen(function* () {
174
const analytics = yield* AnalyticsService.AnalyticsService;
@@ -170,6 +191,36 @@ it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
191
}),
192
);
193
194
+ it.effect("keeps a failed batch through a 20-minute outage and sends it once after", () =>
195
+ Effect.gen(function* () {
196
+ const ingest = failingFor(20);
197
+ const batches = yield* sendFor(40, ingest.respond);
198
+ const statuses = ingest.answers.map(({ status }) => status);
199
+ assert.equal(statuses.at(-1), 204);
200
+ assert.equal(statuses.filter((status) => status === 204).length, 1);
201
+ const sentUuids = new Set(batches.map((batch) => batch.map(({ uuid }) => uuid).join()));
202
+ assert.equal(sentUuids.size, 1);
203
+ assert.equal(batches[0]?.length, 20);
204
+ }),
205
+ );
206
+
207
+ it.effect("drops a batch that keeps failing for 30 minutes", () =>
208
+ Effect.gen(function* () {
209
+ const ingest = failingFor(Infinity);
210
+ yield* sendFor(60, ingest.respond);
211
+ const lastSend = ingest.answers.at(-1)?.at ?? 0;
212
+ assert.isAtLeast(lastSend, 30 * 60_000);
213
+ assert.isBelow(lastSend, 36 * 60_000);
214
+ }),
215
+ );
216
+
217
+ it.effect("drops a batch the endpoint rejects without sending it again", () =>
218
+ Effect.gen(function* () {
219
+ const batches = yield* sendFor(10, respondWithStatus(400));
220
+ assert.equal(batches.length, 1);
221
+ }),
222
+ );
223
+
224
it.effect("flush drains all buffered events across multiple batches", () =>
225
Effect.gen(function* () {
226
const capturedRequests: Array<RecordedBatchRequest> = [];
apps/server/src/telemetry/AnalyticsService.ts
+31
-6
@@ -32,6 +32,7 @@ interface BufferedAnalyticsEvent {
32
interface DeliveryState {
33
readonly failedBatch: ReadonlyArray<BufferedAnalyticsEvent>;
34
readonly batchAttempts: number;
35
+ readonly firstFailedAt: number;
36
readonly failures: number;
37
readonly retryAt: number;
38
}
@@ -40,7 +41,10 @@ interface DeliveryState {
41
const FLUSH_INTERVAL_MS = 10_000;
42
// A hung send would hold the flush lock, and with it the shutdown flush.
43
const SEND_TIMEOUT = "10 seconds";
43
-const MAX_BATCH_ATTEMPTS = 5;
44
+// A failed batch is tried again until it has failed for this long. Ingest answers 503 when it
45
+// cannot store a batch, and a broken Cloudflare location can take about 5 minutes to fail over.
46
+// The waiting batch holds newer events back; `maxBufferedEvents` bounds them meanwhile.
47
+const MAX_BATCH_RETRY_MS = 30 * 60_000;
48
const RETRY_BASE_DELAY_MS = 2_000;
49
// Clients can report any number of owner ids; past this many per observer, new ones
50
// are not sent. Each observer has its own cap, so client reports never crowd out the
@@ -133,6 +137,7 @@ export const make = Effect.gen(function* () {
137
const deliveryRef = yield* Ref.make<DeliveryState>({
138
failedBatch: [],
139
batchAttempts: 0,
140
+ firstFailedAt: 0,
141
failures: 0,
142
retryAt: 0,
143
});
@@ -232,27 +237,47 @@ export const make = Effect.gen(function* () {
237
238
const sent = yield* Effect.result(sendBatch(batch));
239
if (Result.isSuccess(sent)) {
235
- yield* Ref.set(deliveryRef, { failedBatch: [], batchAttempts: 0, failures: 0, retryAt: 0 });
240
+ yield* Ref.set(deliveryRef, {
241
+ failedBatch: [],
242
+ batchAttempts: 0,
243
+ firstFailedAt: 0,
244
+ failures: 0,
245
+ retryAt: 0,
246
+ });
247
continue;
248
}
249
250
+ const now = yield* Clock.currentTimeMillis;
251
const failures = delivery.failures + 1;
252
const batchAttempts = delivery.batchAttempts + 1;
241
- const retryAt = (yield* Clock.currentTimeMillis) + retryDelayMs(failures, yield* Random.next);
253
+ const firstFailedAt = delivery.batchAttempts === 0 ? now : delivery.firstFailedAt;
254
+ const retryAt = now + retryDelayMs(failures, yield* Random.next);
255
const error = sent.failure;
256
const rejected =
257
error._tag === "HttpClientError" &&
258
error.reason._tag === "StatusCodeError" &&
259
REJECTED_BATCH_STATUSES.has(error.reason.response.status);
247
- if (!rejected && batchAttempts < MAX_BATCH_ATTEMPTS) {
248
- yield* Ref.set(deliveryRef, { failedBatch: batch, batchAttempts, failures, retryAt });
260
+ if (!rejected && now - firstFailedAt < MAX_BATCH_RETRY_MS) {
261
+ yield* Ref.set(deliveryRef, {
262
+ failedBatch: batch,
263
+ batchAttempts,
264
+ firstFailedAt,
265
+ failures,
266
+ retryAt,
267
+ });
268
yield* Effect.logDebug("Failed to send telemetry batch; will retry", {
269
attempt: batchAttempts,
270
cause: error,
271
});
272
return;
273
}
255
- yield* Ref.set(deliveryRef, { failedBatch: [], batchAttempts: 0, failures, retryAt });
274
+ yield* Ref.set(deliveryRef, {
275
+ failedBatch: [],
276
+ batchAttempts: 0,
277
+ firstFailedAt: 0,
278
+ failures,
279
+ retryAt,
280
+ });
281
yield* Effect.logWarning(
282
rejected
283
? "Dropped telemetry batch the endpoint rejected"