The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
fix(server): T3 Connect falls back to HTTP/2 after repeated QUIC stalls (#20382)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Лев Федорец committed
Oct 20, 2026 at 08:05 UTC
96a76b140de57cf319cf1688347c08b69cbe80ab
2 files changed
+260
apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+210
@@ -261,6 +261,216 @@ describe("CloudManagedEndpointRuntime", () => {
261
}),
262
);
263
264
+ const tunnelConfig = {
265
+ providerKind: "cloudflare_tunnel" as const,
266
+ connectorToken: "token",
267
+ tunnelId: "tunnel-1",
268
+ };
269
+ const quicDialWarning =
270
+ '2026-10-16T20:29:43Z ERR Failed to dial a quic connection error="failed to dial to edge with quic: timeout: no recent network activity" connIndex=1\n';
271
+ const quicDatagramWarning =
272
+ '2026-10-16T20:29:41Z ERR Connection terminated error="datagram manager error: timeout: no recent network activity" connIndex=1\n';
273
+
274
+ const makeTransportSpawner = Effect.fnUntraced(function* (
275
+ options: {
276
+ transport?: string;
277
+ failSpawn?: number;
278
+ isRunning?: Effect.Effect<boolean>;
279
+ } = {},
280
+ ) {
281
+ yield* Effect.acquireRelease(
282
+ Effect.sync(() => {
283
+ const previous = process.env.TUNNEL_TRANSPORT_PROTOCOL;
284
+ if (options.transport === undefined) delete process.env.TUNNEL_TRANSPORT_PROTOCOL;
285
+ else process.env.TUNNEL_TRANSPORT_PROTOCOL = options.transport;
286
+ return previous;
287
+ }),
288
+ (previous) =>
289
+ Effect.sync(() => {
290
+ if (previous === undefined) delete process.env.TUNNEL_TRANSPORT_PROTOCOL;
291
+ else process.env.TUNNEL_TRANSPORT_PROTOCOL = previous;
292
+ }),
293
+ );
294
+ const spawned: Array<ChildProcess.StandardCommand> = [];
295
+ const killed: Array<number> = [];
296
+ const outputs: Array<Queue.Queue<Uint8Array>> = [];
297
+ const exits: Array<Deferred.Deferred<ChildProcessSpawner.ExitCode>> = [];
298
+ const started = yield* Queue.unbounded<number>();
299
+ const spawner = ChildProcessSpawner.make((command) =>
300
+ Effect.gen(function* () {
301
+ if (!ChildProcess.isStandardCommand(command)) {
302
+ return yield* Effect.die("Expected standard command.");
303
+ }
304
+ const index = spawned.length;
305
+ spawned.push(command);
306
+ if (index === options.failSpawn) {
307
+ return yield* PlatformError.systemError({
308
+ _tag: "NotFound",
309
+ module: "ChildProcess",
310
+ method: "spawn",
311
+ description: "cloudflared temporarily unavailable",
312
+ });
313
+ }
314
+ const output = yield* Queue.unbounded<Uint8Array>();
315
+ const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
316
+ outputs[index] = output;
317
+ exits[index] = exit;
318
+ const handle = ChildProcessSpawner.makeHandle({
319
+ ...makeHandle({
320
+ pid: 800 + index,
321
+ exitCode: Deferred.await(exit),
322
+ output: Stream.fromQueue(output),
323
+ onKill: () => {
324
+ killed.push(800 + index);
325
+ Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143)));
326
+ },
327
+ }),
328
+ isRunning: options.isRunning ?? Effect.succeed(true),
329
+ });
330
+ yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
331
+ yield* Queue.offer(started, index);
332
+ return handle;
333
+ }),
334
+ );
335
+ const emit = (index: number, lines: string) =>
336
+ Queue.offer(outputs[index]!, new TextEncoder().encode(lines));
337
+ const observe = (
338
+ runtime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"],
339
+ index: number,
340
+ lines = "",
341
+ ) =>
342
+ emit(
343
+ index,
344
+ lines + "2026-10-16T20:30:00Z INF Registered tunnel connection connIndex=0\n",
345
+ ).pipe(Effect.andThen(Stream.runHead(runtime.tunnelConnected)));
346
+ return { spawner, spawned, killed, exits, started, emit, observe };
347
+ });
348
+
349
+ it.effect("falls back after 30 recent QUIC stalls across exits, until the runtime restarts", () =>
350
+ Effect.gen(function* () {
351
+ const fixture = yield* makeTransportSpawner();
352
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
353
+ yield* runtime.applyConfig(tunnelConfig);
354
+ expect(yield* Queue.take(fixture.started)).toBe(0);
355
+ expect(fixture.spawned[0]!.args).not.toContain("--protocol");
356
+
357
+ yield* fixture.observe(runtime, 0, quicDialWarning.repeat(20));
358
+ yield* TestClock.adjust(Duration.minutes(4));
359
+ yield* fixture.observe(runtime, 0, quicDatagramWarning.repeat(9));
360
+ yield* TestClock.adjust(Duration.minutes(1));
361
+ // The first 20 have expired, but the newer nine must survive this restart.
362
+ yield* Deferred.succeed(fixture.exits[0]!, ChildProcessSpawner.ExitCode(1));
363
+ expect(yield* Queue.take(fixture.started)).toBe(1);
364
+ yield* fixture.observe(runtime, 1, quicDialWarning.repeat(20));
365
+ yield* fixture.observe(
366
+ runtime,
367
+ 1,
368
+ '2026-10-16T20:30:00Z ERR Register tunnel error from server side error="context deadline exceeded" connIndex=0\n'.repeat(
369
+ 30,
370
+ ),
371
+ );
372
+ expect(fixture.spawned).toHaveLength(2);
373
+ expect(fixture.spawned[1]!.args).not.toContain("--protocol");
374
+
375
+ yield* fixture.emit(1, quicDatagramWarning);
376
+ expect(yield* Queue.take(fixture.started)).toBe(2);
377
+ yield* fixture.observe(runtime, 2);
378
+ expect(fixture.killed).toEqual([800, 801]);
379
+ expect(fixture.spawned[2]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
380
+
381
+ yield* TestClock.adjust(Duration.seconds(30));
382
+ yield* Deferred.succeed(fixture.exits[2]!, ChildProcessSpawner.ExitCode(1));
383
+ expect(yield* Queue.take(fixture.started)).toBe(3);
384
+ yield* fixture.observe(runtime, 3);
385
+ expect(fixture.spawned[3]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
386
+ yield* runtime.applyConfig(null);
387
+ expect(fixture.killed).toEqual([800, 801, 802, 803]);
388
+
389
+ const freshRuntime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
390
+ yield* freshRuntime.applyConfig(tunnelConfig);
391
+ expect(yield* Queue.take(fixture.started)).toBe(4);
392
+ expect(fixture.spawned[4]!.args).not.toContain("--protocol");
393
+ }).pipe(Effect.provide(TestClock.layer())),
394
+ );
395
+
396
+ it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
397
+ Effect.gen(function* () {
398
+ const fixture = yield* makeTransportSpawner({ transport: "quic" });
399
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
400
+ yield* runtime.applyConfig(tunnelConfig);
401
+ yield* fixture.observe(runtime, 0, quicDialWarning.repeat(30));
402
+ yield* fixture.observe(runtime, 0, quicDatagramWarning.repeat(30));
403
+ expect(fixture.spawned).toHaveLength(1);
404
+ expect(fixture.killed).toEqual([]);
405
+ expect(fixture.spawned[0]!.args).not.toContain("--protocol");
406
+ expect(fixture.spawned[0]!.options.env?.TUNNEL_TRANSPORT_PROTOCOL).toBe("quic");
407
+ }),
408
+ );
409
+
410
+ it.effect("requests recovery after a failed HTTP/2 spawn and retries with HTTP/2", () =>
411
+ Effect.gen(function* () {
412
+ const fixture = yield* makeTransportSpawner({ failSpawn: 1 });
413
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
414
+ yield* runtime.applyConfig(tunnelConfig);
415
+ expect(yield* Queue.take(fixture.started)).toBe(0);
416
+ yield* fixture.emit(0, quicDialWarning.repeat(30));
417
+
418
+ const requested = Option.getOrThrow(yield* Stream.runHead(runtime.recoveryRequests));
419
+ expect(requested).toEqual(tunnelConfig);
420
+ expect(fixture.killed).toEqual([800]);
421
+ expect(fixture.spawned).toHaveLength(2);
422
+ expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
423
+ expect(yield* runtime.applyConfig(requested)).toMatchObject({ status: "running", pid: 802 });
424
+ yield* fixture.observe(runtime, 2);
425
+ expect(fixture.spawned[2]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
426
+ }),
427
+ );
428
+
429
+ it.effect("discards a pending fallback when its connector is replaced or removed", () =>
430
+ Effect.gen(function* () {
431
+ for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) {
432
+ yield* Effect.gen(function* () {
433
+ const checking = yield* Deferred.make<void>();
434
+ const releaseCheck = yield* Deferred.make<void>();
435
+ const fixture = yield* makeTransportSpawner({
436
+ isRunning: Deferred.succeed(checking, undefined).pipe(
437
+ Effect.andThen(Deferred.await(releaseCheck)),
438
+ Effect.as(true),
439
+ ),
440
+ });
441
+ const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
442
+ yield* runtime.applyConfig(tunnelConfig);
443
+ yield* fixture.observe(runtime, 0);
444
+
445
+ // Hold reconciliation in a process probe so config changes queue before fallback.
446
+ const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);
447
+ yield* Deferred.await(checking);
448
+ const change = yield* runtime
449
+ .applyConfig(nextConfig)
450
+ .pipe(Effect.forkChild({ startImmediately: true }));
451
+ yield* fixture.observe(runtime, 0, quicDialWarning.repeat(30));
452
+ yield* Deferred.succeed(releaseCheck, undefined);
453
+ yield* Fiber.join(probe);
454
+ yield* Fiber.join(change);
455
+ yield* TestClock.adjust(Duration.zero);
456
+
457
+ expect(fixture.killed).toEqual([800]);
458
+ expect(fixture.spawned).toHaveLength(nextConfig ? 2 : 1);
459
+ if (nextConfig) {
460
+ expect(fixture.spawned[1]!.options.env?.TUNNEL_TOKEN).toBe("replacement");
461
+ expect(fixture.spawned[1]!.args).not.toContain("--protocol");
462
+ }
463
+ yield* runtime.applyConfig(null);
464
+ // Buffered output and the old exit cannot revive a removed configuration.
465
+ yield* fixture.emit(0, quicDialWarning.repeat(30));
466
+ yield* TestClock.adjust(Duration.minutes(5));
467
+ expect(fixture.spawned).toHaveLength(nextConfig ? 2 : 1);
468
+ expect(fixture.killed).toEqual(nextConfig ? [800, 801] : [800]);
469
+ }).pipe(Effect.scoped);
470
+ }
471
+ }).pipe(Effect.provide(TestClock.layer())),
472
+ );
473
+
474
it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
475
Effect.gen(function* () {
476
const output = yield* Queue.unbounded<Uint8Array>();
apps/server/src/cloud/ManagedEndpointRuntime.ts
+50
@@ -86,6 +86,10 @@ const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5);
86
const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6;
87
const CONNECTOR_NOT_READY_CHECKS_MAX = 60;
88
const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u;
89
+// Registration resets cloudflared's own fallback, even when QUIC stalls again
90
+// immediately. Keep that evidence across registrations and connector restarts.
91
+const QUIC_STALLS_BEFORE_HTTP2 = 30;
92
+const QUIC_STALL_WINDOW_MS = 5 * 60_000;
93
// A failed background install of the pinned relay client is retried no sooner
94
// than this, so a crash-looping connector or an offline host does not redownload
95
// on every reconcile.
@@ -178,6 +182,10 @@ export const make = Effect.gen(function* () {
182
const prunedRef = yield* Ref.make(false);
183
const retryLoopRunningRef = yield* Ref.make(false);
184
const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
185
+ // An explicit cloudflared transport in the environment is the user's choice.
186
+ const transportChosenByUser = process.env.TUNNEL_TRANSPORT_PROTOCOL !== undefined;
187
+ const http2Ref = yield* Ref.make(false);
188
+ const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
189
let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
190
191
const stopActive = Effect.gen(function* () {
@@ -429,6 +437,38 @@ export const make = Effect.gen(function* () {
437
}
438
});
439
440
+ const switchToHttp2 = Effect.fnUntraced(function* (connector: ActiveConnector) {
441
+ const active = yield* Ref.get(activeRef);
442
+ const desiredConfig = yield* Ref.get(desiredConfigRef);
443
+ if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return;
444
+ yield* Ref.set(http2Ref, true);
445
+ yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", {
446
+ pid: Number(connector.child.pid),
447
+ tunnelId: connector.config.tunnelId,
448
+ tunnelName: connector.config.tunnelName,
449
+ });
450
+ // Shutdown must not interrupt cleanup after active ownership is cleared.
451
+ yield* Effect.uninterruptible(stopActive);
452
+ const status = yield* reconcileConfig(desiredConfig);
453
+ // A failed spawn has no supervisor to request recovery.
454
+ if (status.status !== "running") {
455
+ yield* Queue.offer(recoveryRequests, desiredConfig);
456
+ }
457
+ }, reconcileSemaphore.withPermits(1));
458
+
459
+ const noteQuicStall = Effect.fnUntraced(function* (connector: ActiveConnector) {
460
+ if (transportChosenByUser || (yield* Ref.get(http2Ref))) return;
461
+ const now = yield* Clock.currentTimeMillis;
462
+ if ((yield* Ref.get(activeRef)) !== connector) return;
463
+ const stalls = yield* Ref.updateAndGet(quicStallsRef, (previous) => [
464
+ ...previous.filter((at) => now - at < QUIC_STALL_WINDOW_MS),
465
+ now,
466
+ ]);
467
+ if (stalls.length < QUIC_STALLS_BEFORE_HTTP2) return;
468
+ // Stopping the connector interrupts the output observer that called this.
469
+ yield* Effect.forkIn(switchToHttp2(connector), runtimeScope);
470
+ });
471
+
472
const observeConnectorOutput = (connector: ActiveConnector) => {
473
let rejectedRegistrations = 0;
474
@@ -465,6 +505,13 @@ export const make = Effect.gen(function* () {
505
Effect.asVoid,
506
);
507
case "warning":
508
+ if (/\btimeout: no recent network activity\b/u.test(line)) {
509
+ return noteQuicStall(connector).pipe(
510
+ Effect.andThen(
511
+ Effect.logWarning("Relay client reported a transport warning", attributes),
512
+ ),
513
+ );
514
+ }
515
if (isRejectedRelayClientTunnelOutput(line)) {
516
rejectedRegistrations += 1;
517
if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
@@ -544,6 +591,7 @@ export const make = Effect.gen(function* () {
591
} satisfies CloudManagedEndpointRuntimeStatus;
592
}
593
594
+ const http2 = yield* Ref.get(http2Ref);
595
const connectorScope = yield* Scope.make("sequential");
596
const child = yield* spawner
597
.spawn(
@@ -558,6 +606,7 @@ export const make = Effect.gen(function* () {
606
"default",
607
"--metrics",
608
"127.0.0.1:0",
609
+ ...(http2 ? ["--protocol", "http2"] : []),
610
"run",
611
],
612
{
@@ -579,6 +628,7 @@ export const make = Effect.gen(function* () {
628
pid: Number(child.pid),
629
source: executable.source,
630
version: executable.version,
631
+ protocol: http2 ? "http2" : "auto",
632
tunnelId: config.tunnelId,
633
tunnelName: config.tunnelName,
634
}),