@devries-cameron20 / t3code Mirror · read-only / commits / 96a76b140d
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

fix(server): T3 Connect falls back to HTTP/2 after repeated QUIC stalls (#20382)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Лев Федорец committed Oct 20, 2026 at 08:05 UTC 96a76b140de57cf319cf1688347c08b69cbe80ab
2 files changed +260
apps/server/src/cloud/ManagedEndpointRuntime.test.ts
+210
@@ -261,6 +261,216 @@ describe("CloudManagedEndpointRuntime", () => {
261 }),
262 );
263
264 + const tunnelConfig = {
265 + providerKind: "cloudflare_tunnel" as const,
266 + connectorToken: "token",
267 + tunnelId: "tunnel-1",
268 + };
269 + const quicDialWarning =
270 + '2026-10-16T20:29:43Z ERR Failed to dial a quic connection error="failed to dial to edge with quic: timeout: no recent network activity" connIndex=1\n';
271 + const quicDatagramWarning =
272 + '2026-10-16T20:29:41Z ERR Connection terminated error="datagram manager error: timeout: no recent network activity" connIndex=1\n';
273 +
274 + const makeTransportSpawner = Effect.fnUntraced(function* (
275 + options: {
276 + transport?: string;
277 + failSpawn?: number;
278 + isRunning?: Effect.Effect<boolean>;
279 + } = {},
280 + ) {
281 + yield* Effect.acquireRelease(
282 + Effect.sync(() => {
283 + const previous = process.env.TUNNEL_TRANSPORT_PROTOCOL;
284 + if (options.transport === undefined) delete process.env.TUNNEL_TRANSPORT_PROTOCOL;
285 + else process.env.TUNNEL_TRANSPORT_PROTOCOL = options.transport;
286 + return previous;
287 + }),
288 + (previous) =>
289 + Effect.sync(() => {
290 + if (previous === undefined) delete process.env.TUNNEL_TRANSPORT_PROTOCOL;
291 + else process.env.TUNNEL_TRANSPORT_PROTOCOL = previous;
292 + }),
293 + );
294 + const spawned: Array<ChildProcess.StandardCommand> = [];
295 + const killed: Array<number> = [];
296 + const outputs: Array<Queue.Queue<Uint8Array>> = [];
297 + const exits: Array<Deferred.Deferred<ChildProcessSpawner.ExitCode>> = [];
298 + const started = yield* Queue.unbounded<number>();
299 + const spawner = ChildProcessSpawner.make((command) =>
300 + Effect.gen(function* () {
301 + if (!ChildProcess.isStandardCommand(command)) {
302 + return yield* Effect.die("Expected standard command.");
303 + }
304 + const index = spawned.length;
305 + spawned.push(command);
306 + if (index === options.failSpawn) {
307 + return yield* PlatformError.systemError({
308 + _tag: "NotFound",
309 + module: "ChildProcess",
310 + method: "spawn",
311 + description: "cloudflared temporarily unavailable",
312 + });
313 + }
314 + const output = yield* Queue.unbounded<Uint8Array>();
315 + const exit = yield* Deferred.make<ChildProcessSpawner.ExitCode>();
316 + outputs[index] = output;
317 + exits[index] = exit;
318 + const handle = ChildProcessSpawner.makeHandle({
319 + ...makeHandle({
320 + pid: 800 + index,
321 + exitCode: Deferred.await(exit),
322 + output: Stream.fromQueue(output),
323 + onKill: () => {
324 + killed.push(800 + index);
325 + Deferred.doneUnsafe(exit, Exit.succeed(ChildProcessSpawner.ExitCode(143)));
326 + },
327 + }),
328 + isRunning: options.isRunning ?? Effect.succeed(true),
329 + });
330 + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
331 + yield* Queue.offer(started, index);
332 + return handle;
333 + }),
334 + );
335 + const emit = (index: number, lines: string) =>
336 + Queue.offer(outputs[index]!, new TextEncoder().encode(lines));
337 + const observe = (
338 + runtime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"],
339 + index: number,
340 + lines = "",
341 + ) =>
342 + emit(
343 + index,
344 + lines + "2026-10-16T20:30:00Z INF Registered tunnel connection connIndex=0\n",
345 + ).pipe(Effect.andThen(Stream.runHead(runtime.tunnelConnected)));
346 + return { spawner, spawned, killed, exits, started, emit, observe };
347 + });
348 +
349 + it.effect("falls back after 30 recent QUIC stalls across exits, until the runtime restarts", () =>
350 + Effect.gen(function* () {
351 + const fixture = yield* makeTransportSpawner();
352 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
353 + yield* runtime.applyConfig(tunnelConfig);
354 + expect(yield* Queue.take(fixture.started)).toBe(0);
355 + expect(fixture.spawned[0]!.args).not.toContain("--protocol");
356 +
357 + yield* fixture.observe(runtime, 0, quicDialWarning.repeat(20));
358 + yield* TestClock.adjust(Duration.minutes(4));
359 + yield* fixture.observe(runtime, 0, quicDatagramWarning.repeat(9));
360 + yield* TestClock.adjust(Duration.minutes(1));
361 + // The first 20 have expired, but the newer nine must survive this restart.
362 + yield* Deferred.succeed(fixture.exits[0]!, ChildProcessSpawner.ExitCode(1));
363 + expect(yield* Queue.take(fixture.started)).toBe(1);
364 + yield* fixture.observe(runtime, 1, quicDialWarning.repeat(20));
365 + yield* fixture.observe(
366 + runtime,
367 + 1,
368 + '2026-10-16T20:30:00Z ERR Register tunnel error from server side error="context deadline exceeded" connIndex=0\n'.repeat(
369 + 30,
370 + ),
371 + );
372 + expect(fixture.spawned).toHaveLength(2);
373 + expect(fixture.spawned[1]!.args).not.toContain("--protocol");
374 +
375 + yield* fixture.emit(1, quicDatagramWarning);
376 + expect(yield* Queue.take(fixture.started)).toBe(2);
377 + yield* fixture.observe(runtime, 2);
378 + expect(fixture.killed).toEqual([800, 801]);
379 + expect(fixture.spawned[2]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
380 +
381 + yield* TestClock.adjust(Duration.seconds(30));
382 + yield* Deferred.succeed(fixture.exits[2]!, ChildProcessSpawner.ExitCode(1));
383 + expect(yield* Queue.take(fixture.started)).toBe(3);
384 + yield* fixture.observe(runtime, 3);
385 + expect(fixture.spawned[3]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
386 + yield* runtime.applyConfig(null);
387 + expect(fixture.killed).toEqual([800, 801, 802, 803]);
388 +
389 + const freshRuntime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
390 + yield* freshRuntime.applyConfig(tunnelConfig);
391 + expect(yield* Queue.take(fixture.started)).toBe(4);
392 + expect(fixture.spawned[4]!.args).not.toContain("--protocol");
393 + }).pipe(Effect.provide(TestClock.layer())),
394 + );
395 +
396 + it.effect("preserves an explicit transport choice despite repeated QUIC stalls", () =>
397 + Effect.gen(function* () {
398 + const fixture = yield* makeTransportSpawner({ transport: "quic" });
399 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
400 + yield* runtime.applyConfig(tunnelConfig);
401 + yield* fixture.observe(runtime, 0, quicDialWarning.repeat(30));
402 + yield* fixture.observe(runtime, 0, quicDatagramWarning.repeat(30));
403 + expect(fixture.spawned).toHaveLength(1);
404 + expect(fixture.killed).toEqual([]);
405 + expect(fixture.spawned[0]!.args).not.toContain("--protocol");
406 + expect(fixture.spawned[0]!.options.env?.TUNNEL_TRANSPORT_PROTOCOL).toBe("quic");
407 + }),
408 + );
409 +
410 + it.effect("requests recovery after a failed HTTP/2 spawn and retries with HTTP/2", () =>
411 + Effect.gen(function* () {
412 + const fixture = yield* makeTransportSpawner({ failSpawn: 1 });
413 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
414 + yield* runtime.applyConfig(tunnelConfig);
415 + expect(yield* Queue.take(fixture.started)).toBe(0);
416 + yield* fixture.emit(0, quicDialWarning.repeat(30));
417 +
418 + const requested = Option.getOrThrow(yield* Stream.runHead(runtime.recoveryRequests));
419 + expect(requested).toEqual(tunnelConfig);
420 + expect(fixture.killed).toEqual([800]);
421 + expect(fixture.spawned).toHaveLength(2);
422 + expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
423 + expect(yield* runtime.applyConfig(requested)).toMatchObject({ status: "running", pid: 802 });
424 + yield* fixture.observe(runtime, 2);
425 + expect(fixture.spawned[2]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]);
426 + }),
427 + );
428 +
429 + it.effect("discards a pending fallback when its connector is replaced or removed", () =>
430 + Effect.gen(function* () {
431 + for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) {
432 + yield* Effect.gen(function* () {
433 + const checking = yield* Deferred.make<void>();
434 + const releaseCheck = yield* Deferred.make<void>();
435 + const fixture = yield* makeTransportSpawner({
436 + isRunning: Deferred.succeed(checking, undefined).pipe(
437 + Effect.andThen(Deferred.await(releaseCheck)),
438 + Effect.as(true),
439 + ),
440 + });
441 + const runtime = yield* buildCloudManagedEndpointRuntime(fixture.spawner);
442 + yield* runtime.applyConfig(tunnelConfig);
443 + yield* fixture.observe(runtime, 0);
444 +
445 + // Hold reconciliation in a process probe so config changes queue before fallback.
446 + const probe = yield* runtime.applyConfig(tunnelConfig).pipe(Effect.forkChild);
447 + yield* Deferred.await(checking);
448 + const change = yield* runtime
449 + .applyConfig(nextConfig)
450 + .pipe(Effect.forkChild({ startImmediately: true }));
451 + yield* fixture.observe(runtime, 0, quicDialWarning.repeat(30));
452 + yield* Deferred.succeed(releaseCheck, undefined);
453 + yield* Fiber.join(probe);
454 + yield* Fiber.join(change);
455 + yield* TestClock.adjust(Duration.zero);
456 +
457 + expect(fixture.killed).toEqual([800]);
458 + expect(fixture.spawned).toHaveLength(nextConfig ? 2 : 1);
459 + if (nextConfig) {
460 + expect(fixture.spawned[1]!.options.env?.TUNNEL_TOKEN).toBe("replacement");
461 + expect(fixture.spawned[1]!.args).not.toContain("--protocol");
462 + }
463 + yield* runtime.applyConfig(null);
464 + // Buffered output and the old exit cannot revive a removed configuration.
465 + yield* fixture.emit(0, quicDialWarning.repeat(30));
466 + yield* TestClock.adjust(Duration.minutes(5));
467 + expect(fixture.spawned).toHaveLength(nextConfig ? 2 : 1);
468 + expect(fixture.killed).toEqual(nextConfig ? [800, 801] : [800]);
469 + }).pipe(Effect.scoped);
470 + }
471 + }).pipe(Effect.provide(TestClock.layer())),
472 + );
473 +
474 it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
475 Effect.gen(function* () {
476 const output = yield* Queue.unbounded<Uint8Array>();
apps/server/src/cloud/ManagedEndpointRuntime.ts
+50
@@ -86,6 +86,10 @@ const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5);
86 const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6;
87 const CONNECTOR_NOT_READY_CHECKS_MAX = 60;
88 const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u;
89 +// Registration resets cloudflared's own fallback, even when QUIC stalls again
90 +// immediately. Keep that evidence across registrations and connector restarts.
91 +const QUIC_STALLS_BEFORE_HTTP2 = 30;
92 +const QUIC_STALL_WINDOW_MS = 5 * 60_000;
93 // A failed background install of the pinned relay client is retried no sooner
94 // than this, so a crash-looping connector or an offline host does not redownload
95 // on every reconcile.
@@ -178,6 +182,10 @@ export const make = Effect.gen(function* () {
182 const prunedRef = yield* Ref.make(false);
183 const retryLoopRunningRef = yield* Ref.make(false);
184 const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
185 + // An explicit cloudflared transport in the environment is the user's choice.
186 + const transportChosenByUser = process.env.TUNNEL_TRANSPORT_PROTOCOL !== undefined;
187 + const http2Ref = yield* Ref.make(false);
188 + const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
189 let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
190
191 const stopActive = Effect.gen(function* () {
@@ -429,6 +437,38 @@ export const make = Effect.gen(function* () {
437 }
438 });
439
440 + const switchToHttp2 = Effect.fnUntraced(function* (connector: ActiveConnector) {
441 + const active = yield* Ref.get(activeRef);
442 + const desiredConfig = yield* Ref.get(desiredConfigRef);
443 + if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return;
444 + yield* Ref.set(http2Ref, true);
445 + yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", {
446 + pid: Number(connector.child.pid),
447 + tunnelId: connector.config.tunnelId,
448 + tunnelName: connector.config.tunnelName,
449 + });
450 + // Shutdown must not interrupt cleanup after active ownership is cleared.
451 + yield* Effect.uninterruptible(stopActive);
452 + const status = yield* reconcileConfig(desiredConfig);
453 + // A failed spawn has no supervisor to request recovery.
454 + if (status.status !== "running") {
455 + yield* Queue.offer(recoveryRequests, desiredConfig);
456 + }
457 + }, reconcileSemaphore.withPermits(1));
458 +
459 + const noteQuicStall = Effect.fnUntraced(function* (connector: ActiveConnector) {
460 + if (transportChosenByUser || (yield* Ref.get(http2Ref))) return;
461 + const now = yield* Clock.currentTimeMillis;
462 + if ((yield* Ref.get(activeRef)) !== connector) return;
463 + const stalls = yield* Ref.updateAndGet(quicStallsRef, (previous) => [
464 + ...previous.filter((at) => now - at < QUIC_STALL_WINDOW_MS),
465 + now,
466 + ]);
467 + if (stalls.length < QUIC_STALLS_BEFORE_HTTP2) return;
468 + // Stopping the connector interrupts the output observer that called this.
469 + yield* Effect.forkIn(switchToHttp2(connector), runtimeScope);
470 + });
471 +
472 const observeConnectorOutput = (connector: ActiveConnector) => {
473 let rejectedRegistrations = 0;
474
@@ -465,6 +505,13 @@ export const make = Effect.gen(function* () {
505 Effect.asVoid,
506 );
507 case "warning":
508 + if (/\btimeout: no recent network activity\b/u.test(line)) {
509 + return noteQuicStall(connector).pipe(
510 + Effect.andThen(
511 + Effect.logWarning("Relay client reported a transport warning", attributes),
512 + ),
513 + );
514 + }
515 if (isRejectedRelayClientTunnelOutput(line)) {
516 rejectedRegistrations += 1;
517 if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
@@ -544,6 +591,7 @@ export const make = Effect.gen(function* () {
591 } satisfies CloudManagedEndpointRuntimeStatus;
592 }
593
594 + const http2 = yield* Ref.get(http2Ref);
595 const connectorScope = yield* Scope.make("sequential");
596 const child = yield* spawner
597 .spawn(
@@ -558,6 +606,7 @@ export const make = Effect.gen(function* () {
606 "default",
607 "--metrics",
608 "127.0.0.1:0",
609 + ...(http2 ? ["--protocol", "http2"] : []),
610 "run",
611 ],
612 {
@@ -579,6 +628,7 @@ export const make = Effect.gen(function* () {
628 pid: Number(child.pid),
629 source: executable.source,
630 version: executable.version,
631 + protocol: http2 ? "http2" : "auto",
632 tunnelId: config.tunnelId,
633 tunnelName: config.tunnelName,
634 }),