@devries-cameron20 / t3code Mirror · read-only / commits / 00f617db4e
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 17:15 UTC

fix(client-runtime): recover subscriptions that stop on a healthy connection (#24400)

Julius Marminge committed Nov 13, 2026 at 08:43 UTC 00f617db4e97dbfc4d713f69d9f90394e7d3b12a
3 files changed +289 -9
packages/client-runtime/src/rpc/client.test.ts
+137 -1
@@ -7,15 +7,17 @@ import {
7 } from "@t3tools/contracts";
8 import { describe, expect, it } from "@effect/vitest";
9 import * as Cause from "effect/Cause";
10 +import * as Clock from "effect/Clock";
11 import * as Deferred from "effect/Deferred";
12 import * as Effect from "effect/Effect";
13 import * as Fiber from "effect/Fiber";
14 import * as Option from "effect/Option";
15 import * as Queue from "effect/Queue";
16 import * as Ref from "effect/Ref";
17 +import * as Schema from "effect/Schema";
18 import * as Stream from "effect/Stream";
17 -import * as TestClock from "effect/testing/TestClock";
19 import * as SubscriptionRef from "effect/SubscriptionRef";
20 +import * as TestClock from "effect/testing/TestClock";
21 import { RpcClientError } from "effect/rpc";
22
23 import {
@@ -45,6 +47,8 @@ const INSTALL_DOWNLOADING: RelayClientInstallProgressEvent = {
47 stage: "downloading",
48 };
49
50 +const decodeNumber = Schema.decodeUnknownEffect(Schema.Number);
51 +
52 function session(client: WsRpcProtocolClient): RpcSession.RpcSession {
53 return {
54 client,
@@ -265,6 +269,8 @@ describe("environment RPC", () => {
269 while (subscriptions.length < 2) {
270 yield* Queue.take(subscriptionChanges);
271 }
272 + // The pending same-session retry must not resubscribe the replaced session.
273 + yield* TestClock.adjust("1 minute");
274 yield* Fiber.interrupt(subscriptionFiber);
275
276 expect(subscriptions).toEqual(["first", "second"]);
@@ -272,6 +278,136 @@ describe("environment RPC", () => {
278 }),
279 );
280
281 + // The session stays healthy (`closed` never resolves), as when one stream chunk fails to decode,
282 + // so the supervisor will not replace it and only resubscribing restores delivery.
283 + const resumesOnTheSameSession = Effect.fn("TestEnvironmentRpc.resumesOnTheSameSession")(
284 + function* (streamFailure: Stream.Stream<never, unknown>) {
285 + const resumedValue = { source: "resumed" } as unknown as ServerLifecycleStreamEvent;
286 + const firstSubscribed = yield* Deferred.make<void>();
287 + let subscriptions = 0;
288 + const client = {
289 + [WS_METHODS.subscribeServerLifecycle]: () => {
290 + subscriptions += 1;
291 + return subscriptions === 1
292 + ? Stream.fromEffect(Deferred.succeed(firstSubscribed, undefined)).pipe(
293 + Stream.drain,
294 + Stream.concat(streamFailure),
295 + )
296 + : Stream.make(resumedValue).pipe(Stream.concat(Stream.never));
297 + },
298 + } as unknown as WsRpcProtocolClient;
299 + const { activeSession, retryCount, supervisor } = yield* makeHarness();
300 +
301 + yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
302 + const resultFiber = yield* subscribe(WS_METHODS.subscribeServerLifecycle, {}).pipe(
303 + Stream.runHead,
304 + Effect.timeoutOption("2 seconds"),
305 + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
306 + Effect.forkChild,
307 + );
308 + yield* Deferred.await(firstSubscribed);
309 + yield* TestClock.adjust("2 seconds");
310 +
311 + expect(
312 + yield* Fiber.join(resultFiber),
313 + "durable subscription stopped delivering after its stream ended on a healthy session",
314 + ).toEqual(Option.some(Option.some(resumedValue)));
315 + expect(subscriptions).toBe(2);
316 + expect(yield* Ref.get(retryCount)).toBe(0);
317 + },
318 + );
319 +
320 + it.effect("resubscribes on the same session after a transport failure", () =>
321 + resumesOnTheSameSession(
322 + Stream.fail(
323 + new RpcClientError.RpcClientError({
324 + reason: new RpcClientError.RpcClientDefect({
325 + message: "stream failed",
326 + cause: new Error("stream failed"),
327 + }),
328 + }),
329 + ),
330 + ),
331 + );
332 +
333 + it.effect("resubscribes on the same session after a stream chunk fails to decode", () =>
334 + // Mirrors RpcClient, which decodes each chunk with `Effect.orDie`.
335 + resumesOnTheSameSession(
336 + Stream.fromEffect(decodeNumber("not a number").pipe(Effect.orDie)).pipe(Stream.drain),
337 + ),
338 + );
339 +
340 + it.effect(
341 + "backs off a same-session stream that keeps failing to decode after its first value",
342 + () =>
343 + Effect.gen(function* () {
344 + // Like the shell stream: a valid value (metadata) arrives, then a chunk fails to decode.
345 + const value = { source: "metadata" } as unknown as ServerLifecycleStreamEvent;
346 + const decodeDefect = decodeNumber("not a number").pipe(Effect.orDie);
347 + const subscriptions = yield* Queue.unbounded<{
348 + readonly session: "first" | "second";
349 + readonly at: number;
350 + }>();
351 + let secondSubscriptions = 0;
352 + const failingClient = (name: "first" | "second") =>
353 + ({
354 + [WS_METHODS.subscribeServerLifecycle]: () =>
355 + Stream.unwrap(
356 + Effect.gen(function* () {
357 + yield* Queue.offer(subscriptions, {
358 + session: name,
359 + at: yield* Clock.currentTimeMillis,
360 + });
361 + if (name === "second" && ++secondSubscriptions > 2) return Stream.never;
362 + return Stream.make(value).pipe(
363 + Stream.concat(Stream.fromEffect(decodeDefect).pipe(Stream.drain)),
364 + );
365 + }),
366 + ),
367 + }) as unknown as WsRpcProtocolClient;
368 + const { activeSession, retryCount, supervisor } = yield* makeHarness();
369 +
370 + yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("first"))));
371 + const subscriptionFiber = yield* subscribe(
372 + WS_METHODS.subscribeServerLifecycle,
373 + {},
374 + {
375 + onExpectedFailure: () => Effect.void,
376 + retryExpectedFailureAfter: "250 millis",
377 + },
378 + ).pipe(
379 + Stream.runDrain,
380 + Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
381 + Effect.forkChild,
382 + );
383 +
384 + const expectedFirstSessionDelays = [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
385 + const firstSessionDelays: number[] = [];
386 + let previous = yield* Queue.take(subscriptions);
387 + for (const delay of expectedFirstSessionDelays) {
388 + yield* TestClock.adjust(delay);
389 + const next = yield* Queue.take(subscriptions);
390 + firstSessionDelays.push(next.at - previous.at);
391 + previous = next;
392 + }
393 +
394 + // A replacement session starts a fresh backoff.
395 + yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("second"))));
396 + const replaced = yield* Queue.take(subscriptions);
397 + yield* TestClock.adjust(250);
398 + const retried = yield* Queue.take(subscriptions);
399 + yield* Fiber.interrupt(subscriptionFiber);
400 +
401 + expect(
402 + firstSessionDelays,
403 + "same-session decode retries reset to the base delay after every first value",
404 + ).toEqual(expectedFirstSessionDelays);
405 + expect([replaced.session, retried.session]).toEqual(["second", "second"]);
406 + expect(retried.at - replaced.at).toBe(250);
407 + expect(yield* Ref.get(retryCount)).toBe(0);
408 + }),
409 + );
410 +
411 it.effect("surfaces domain subscription failures without reconnecting", () =>
412 Effect.gen(function* () {
413 const domainError = new Error("terminal subscription rejected");
packages/client-runtime/src/rpc/client.ts
+48 -8
@@ -114,6 +114,7 @@ export const isAuthorizationFailure = (cause: Cause.Cause<unknown>) =>
114 );
115
116 const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000;
117 +const STREAM_FAILURE_RETRY_DELAY = Duration.seconds(1);
118
119 export const subscriptionRetryDelay = (base: Duration.Input, retries: number) =>
120 Duration.millis(
@@ -318,6 +319,8 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
319 EnvironmentRpcStreamFailure<TTag>
320 >;
321 let expectedFailureRetries = 0;
322 + // Not reset by a first value: a stream can emit before every chunk failure.
323 + let streamFailureRetries = 0;
324 const subscribeToSession = (): Stream.Stream<A, EnvironmentRpcStreamFailure<TTag>> =>
325 Stream.suspend(() =>
326 Stream.unwrap(
@@ -354,6 +357,11 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
357 : Effect.void,
358 ),
359 Stream.catchCause((cause) => {
360 + const retryAfter = (base: Duration.Input, retries: number) =>
361 + Stream.fromEffect(Effect.sleep(subscriptionRetryDelay(base, retries))).pipe(
362 + Stream.drain,
363 + Stream.concat(subscribeToSession()),
364 + );
365 const hasOnlyExpectedFailures =
366 cause.reasons.length > 0 &&
367 cause.reasons.every((reason) => reason._tag === "Fail");
@@ -362,7 +370,22 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
370 cause.reasons.every(
371 (reason) => reason._tag === "Fail" && isRpcClientError(reason.error),
372 );
365 - if (isTransportFailure) {
373 + // The RPC client dies with a SchemaError when one stream chunk fails to
374 + // decode, ending only that request while the socket stays open. A caller
375 + // with `onDefect` owns its defect recovery.
376 + const isChunkDecodeDefect =
377 + options?.onDefect === undefined &&
378 + cause.reasons.length > 0 &&
379 + cause.reasons.every(
380 + (reason) => reason._tag === "Die" && Schema.isSchemaError(reason.defect),
381 + );
382 + // Both end one stream while the session can stay healthy, so nothing else
383 + // would replace it. A dead session is replaced by the supervisor, which
384 + // interrupts this retry through switchMap. A finite search must settle
385 + // rather than restart, so it keeps waiting for the next session.
386 + const resubscribesOnSameSession =
387 + tag !== ORCHESTRATION_V2_WS_METHODS.searchThreadStream;
388 + if (isTransportFailure && !resubscribesOnSameSession) {
389 return Stream.fromEffect(
390 Effect.logWarning(
391 "Durable RPC subscription lost its transport; waiting for the next session.",
@@ -374,6 +397,29 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
397 ),
398 ).pipe(Stream.drain);
399 }
400 + if (
401 + resubscribesOnSameSession &&
402 + (isTransportFailure || isChunkDecodeDefect)
403 + ) {
404 + return Stream.fromEffect(
405 + Effect.logWarning(
406 + "Durable RPC subscription stream ended; resubscribing.",
407 + {
408 + cause: Cause.pretty(cause),
409 + method: tag,
410 + environmentId: supervisor.target.environmentId,
411 + },
412 + ),
413 + ).pipe(
414 + Stream.drain,
415 + Stream.concat(
416 + retryAfter(
417 + options?.retryExpectedFailureAfter ?? STREAM_FAILURE_RETRY_DELAY,
418 + streamFailureRetries++,
419 + ),
420 + ),
421 + );
422 + }
423 if (hasOnlyExpectedFailures && options?.onExpectedFailure !== undefined) {
424 const handled = Stream.fromEffect(options.onExpectedFailure(cause)).pipe(
425 Stream.drain,
@@ -384,16 +430,10 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
430 ) {
431 return handled;
432 }
387 - const retryDelay = subscriptionRetryDelay(
388 - options.retryExpectedFailureAfter,
389 - expectedFailureRetries,
390 - );
391 - expectedFailureRetries += 1;
433 return handled.pipe(
434 Stream.concat(
394 - Stream.fromEffect(Effect.sleep(retryDelay)).pipe(Stream.drain),
435 + retryAfter(options.retryExpectedFailureAfter, expectedFailureRetries++),
436 ),
396 - Stream.concat(subscribeToSession()),
437 );
438 }
439 // A reported defect parks until the next session or resubscribe. Failing would
packages/client-runtime/src/state/runtime.test.ts
+104
@@ -18,6 +18,7 @@ import {
18 WS_METHODS,
19 } from "@t3tools/contracts";
20 import * as Cause from "effect/Cause";
21 +import * as Clock from "effect/Clock";
22 import * as DateTime from "effect/DateTime";
23 import * as Deferred from "effect/Deferred";
24 import * as Effect from "effect/Effect";
@@ -29,6 +30,7 @@ import * as Queue from "effect/Queue";
30 import * as Schema from "effect/Schema";
31 import * as Stream from "effect/Stream";
32 import * as SubscriptionRef from "effect/SubscriptionRef";
33 +import * as TestClock from "effect/testing/TestClock";
34 import type { HttpClient } from "effect/http";
35 import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
36
@@ -43,6 +45,7 @@ import {
45 import * as EnvironmentRegistry from "../connection/registry.ts";
46 import * as EnvironmentSupervisor from "../connection/supervisor.ts";
47 import { EnvironmentRpcUnavailableError } from "../rpc/client.ts";
48 +import type { WsRpcProtocolClient } from "../rpc/protocol.ts";
49 import type * as RpcSession from "../rpc/session.ts";
50 import { createOrchestrationEnvironmentAtoms } from "./orchestration.ts";
51 import { createReviewEnvironmentAtoms } from "./review.ts";
@@ -54,6 +57,7 @@ import { createTerminalEnvironmentAtoms } from "./terminal.ts";
57 import {
58 createAtomCommandScheduler,
59 createEnvironmentQueryAtomFamily,
60 + createEnvironmentRpcSubscriptionAtomFamily,
61 selectEnvironmentQueryData,
62 createEnvironmentSubscriptionAtomFamily,
63 createRuntimeCommand,
@@ -70,6 +74,8 @@ const QUERY_ENVIRONMENT = new PrimaryConnectionTarget({
74 wsBaseUrl: "wss://query.example.test",
75 });
76
77 +const decodeNumber = Schema.decodeUnknownEffect(Schema.Number);
78 +
79 const QUERY_RPC_SESSION = {} as RpcSession.RpcSession;
80
81 class TestQueryError extends Schema.TaggedError<TestQueryError>()("TestQueryError", {
@@ -393,6 +399,104 @@ describe("finite environment subscription lifecycle", () => {
399 expect(observed.some((result) => AsyncResult.isFailure(result))).toBe(false);
400 }),
401 );
402 + it.effect(
403 + "surfaces a decode defect from a finite RPC subscription instead of resubscribing",
404 + () =>
405 + Effect.scoped(
406 + Effect.gen(function* () {
407 + const firstSubscribed = yield* Deferred.make<void>();
408 + const resubscribed = yield* Deferred.make<void>();
409 + const surfaced = yield* Deferred.make<Cause.Cause<unknown>>();
410 + let subscriptions = 0;
411 + // Mirrors RpcClient, which decodes each chunk with `Effect.orDie`.
412 + const decodeDefect = decodeNumber("not a number").pipe(Effect.orDie);
413 + const client = {
414 + [ORCHESTRATION_V2_WS_METHODS.searchThreadStream]: () => {
415 + subscriptions += 1;
416 + Deferred.doneUnsafe(subscriptions === 1 ? firstSubscribed : resubscribed, Exit.void);
417 + return Stream.fromEffect(decodeDefect).pipe(Stream.drain);
418 + },
419 + } as unknown as WsRpcProtocolClient;
420 + const rpcSession = {
421 + client,
422 + initialConfig: Effect.never,
423 + subscribeServerConfig: () => Stream.never,
424 + ready: Effect.void,
425 + probe: Effect.void,
426 + closed: Effect.never,
427 + } as unknown as RpcSession.RpcSession;
428 + const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
429 + target: QUERY_ENVIRONMENT,
430 + state: yield* SubscriptionRef.make(queryConnectionState()),
431 + session: yield* SubscriptionRef.make(Option.some(rpcSession)),
432 + prepared: yield* SubscriptionRef.make<Option.Option<PreparedConnection>>(Option.none()),
433 + connect: Effect.void,
434 + disconnect: Effect.void,
435 + retryNow: Effect.void,
436 + } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]);
437 + const environmentRegistry = EnvironmentRegistry.EnvironmentRegistry.of({
438 + followStream: (_environmentId: EnvironmentId, stream: Stream.Stream<unknown>) =>
439 + Stream.provideService(
440 + stream,
441 + EnvironmentSupervisor.EnvironmentSupervisor,
442 + supervisor,
443 + ),
444 + } as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]);
445 + // Atoms run their own fibers; share the test clock so a pending retry is driven by it.
446 + const runtime = Atom.runtime(
447 + Layer.mergeAll(
448 + Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, environmentRegistry),
449 + Layer.succeed(Clock.Clock, yield* Clock.Clock),
450 + ),
451 + );
452 + // Same options as the progressive thread find in orchestration.ts.
453 + const family = createEnvironmentRpcSubscriptionAtomFamily(runtime, {
454 + label: "test.finite-rpc-subscription",
455 + tag: ORCHESTRATION_V2_WS_METHODS.searchThreadStream,
456 + completeWhen: (result) => result.complete !== false,
457 + idleTtlMs: 0,
458 + });
459 + const atom = family({
460 + environmentId: QUERY_ENVIRONMENT.environmentId,
461 + input: {} as never,
462 + });
463 + const registry = AtomRegistry.make();
464 + const unsubscribe = registry.subscribe(
465 + atom,
466 + (result) => {
467 + if (AsyncResult.isFailure(result))
468 + Deferred.doneUnsafe(surfaced, Exit.succeed(result.cause));
469 + },
470 + { immediate: true },
471 + );
472 + yield* Effect.addFinalizer(() =>
473 + Effect.sync(() => {
474 + unsubscribe();
475 + registry.dispose();
476 + }),
477 + );
478 + yield* Deferred.await(firstSubscribed);
479 + // Lets any same-session retry run, so its resubscription can win the race below.
480 + yield* TestClock.adjust("1 minute");
481 + const outcome = yield* Effect.raceFirst(
482 + Deferred.await(surfaced).pipe(
483 + Effect.map((cause) => ({ _tag: "surfaced", cause }) as const),
484 + ),
485 + Deferred.await(resubscribed).pipe(Effect.as({ _tag: "resubscribed" } as const)),
486 + );
487 +
488 + expect(
489 + outcome._tag,
490 + "finite subscription resubscribed after a decode defect instead of failing its consumer",
491 + ).toBe("surfaced");
492 + if (outcome._tag === "surfaced") {
493 + expect(Cause.hasDies(outcome.cause)).toBe(true);
494 + expect(Schema.isSchemaError(Cause.squash(outcome.cause))).toBe(true);
495 + }
496 + expect(subscriptions).toBe(1);
497 + }),
498 + ),
499 + );
500 it.effect.each([
501 { completedTtlMs: undefined, subscriptions: 2 },
502 { completedTtlMs: 60_000, subscriptions: 1 },