The best way to control Claude Code, Codex and more from any computer or phone
fix(client-runtime): recover subscriptions that stop on a healthy connection (#24400)
Julius Marminge committed
Nov 13, 2026 at 08:43 UTC
00f617db4e97dbfc4d713f69d9f90394e7d3b12a
3 files changed
+289
-9
packages/client-runtime/src/rpc/client.test.ts
+137
-1
@@ -7,15 +7,17 @@ import {
7
} from "@t3tools/contracts";
8
import { describe, expect, it } from "@effect/vitest";
9
import * as Cause from "effect/Cause";
10
+import * as Clock from "effect/Clock";
11
import * as Deferred from "effect/Deferred";
12
import * as Effect from "effect/Effect";
13
import * as Fiber from "effect/Fiber";
14
import * as Option from "effect/Option";
15
import * as Queue from "effect/Queue";
16
import * as Ref from "effect/Ref";
17
+import * as Schema from "effect/Schema";
18
import * as Stream from "effect/Stream";
17
-import * as TestClock from "effect/testing/TestClock";
19
import * as SubscriptionRef from "effect/SubscriptionRef";
20
+import * as TestClock from "effect/testing/TestClock";
21
import { RpcClientError } from "effect/rpc";
22
23
import {
@@ -45,6 +47,8 @@ const INSTALL_DOWNLOADING: RelayClientInstallProgressEvent = {
47
stage: "downloading",
48
};
49
50
+const decodeNumber = Schema.decodeUnknownEffect(Schema.Number);
51
+
52
function session(client: WsRpcProtocolClient): RpcSession.RpcSession {
53
return {
54
client,
@@ -265,6 +269,8 @@ describe("environment RPC", () => {
269
while (subscriptions.length < 2) {
270
yield* Queue.take(subscriptionChanges);
271
}
272
+ // The pending same-session retry must not resubscribe the replaced session.
273
+ yield* TestClock.adjust("1 minute");
274
yield* Fiber.interrupt(subscriptionFiber);
275
276
expect(subscriptions).toEqual(["first", "second"]);
@@ -272,6 +278,136 @@ describe("environment RPC", () => {
278
}),
279
);
280
281
+ // The session stays healthy (`closed` never resolves), as when one stream chunk fails to decode,
282
+ // so the supervisor will not replace it and only resubscribing restores delivery.
283
+ const resumesOnTheSameSession = Effect.fn("TestEnvironmentRpc.resumesOnTheSameSession")(
284
+ function* (streamFailure: Stream.Stream<never, unknown>) {
285
+ const resumedValue = { source: "resumed" } as unknown as ServerLifecycleStreamEvent;
286
+ const firstSubscribed = yield* Deferred.make<void>();
287
+ let subscriptions = 0;
288
+ const client = {
289
+ [WS_METHODS.subscribeServerLifecycle]: () => {
290
+ subscriptions += 1;
291
+ return subscriptions === 1
292
+ ? Stream.fromEffect(Deferred.succeed(firstSubscribed, undefined)).pipe(
293
+ Stream.drain,
294
+ Stream.concat(streamFailure),
295
+ )
296
+ : Stream.make(resumedValue).pipe(Stream.concat(Stream.never));
297
+ },
298
+ } as unknown as WsRpcProtocolClient;
299
+ const { activeSession, retryCount, supervisor } = yield* makeHarness();
300
+
301
+ yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
302
+ const resultFiber = yield* subscribe(WS_METHODS.subscribeServerLifecycle, {}).pipe(
303
+ Stream.runHead,
304
+ Effect.timeoutOption("2 seconds"),
305
+ Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
306
+ Effect.forkChild,
307
+ );
308
+ yield* Deferred.await(firstSubscribed);
309
+ yield* TestClock.adjust("2 seconds");
310
+
311
+ expect(
312
+ yield* Fiber.join(resultFiber),
313
+ "durable subscription stopped delivering after its stream ended on a healthy session",
314
+ ).toEqual(Option.some(Option.some(resumedValue)));
315
+ expect(subscriptions).toBe(2);
316
+ expect(yield* Ref.get(retryCount)).toBe(0);
317
+ },
318
+ );
319
+
320
+ it.effect("resubscribes on the same session after a transport failure", () =>
321
+ resumesOnTheSameSession(
322
+ Stream.fail(
323
+ new RpcClientError.RpcClientError({
324
+ reason: new RpcClientError.RpcClientDefect({
325
+ message: "stream failed",
326
+ cause: new Error("stream failed"),
327
+ }),
328
+ }),
329
+ ),
330
+ ),
331
+ );
332
+
333
+ it.effect("resubscribes on the same session after a stream chunk fails to decode", () =>
334
+ // Mirrors RpcClient, which decodes each chunk with `Effect.orDie`.
335
+ resumesOnTheSameSession(
336
+ Stream.fromEffect(decodeNumber("not a number").pipe(Effect.orDie)).pipe(Stream.drain),
337
+ ),
338
+ );
339
+
340
+ it.effect(
341
+ "backs off a same-session stream that keeps failing to decode after its first value",
342
+ () =>
343
+ Effect.gen(function* () {
344
+ // Like the shell stream: a valid value (metadata) arrives, then a chunk fails to decode.
345
+ const value = { source: "metadata" } as unknown as ServerLifecycleStreamEvent;
346
+ const decodeDefect = decodeNumber("not a number").pipe(Effect.orDie);
347
+ const subscriptions = yield* Queue.unbounded<{
348
+ readonly session: "first" | "second";
349
+ readonly at: number;
350
+ }>();
351
+ let secondSubscriptions = 0;
352
+ const failingClient = (name: "first" | "second") =>
353
+ ({
354
+ [WS_METHODS.subscribeServerLifecycle]: () =>
355
+ Stream.unwrap(
356
+ Effect.gen(function* () {
357
+ yield* Queue.offer(subscriptions, {
358
+ session: name,
359
+ at: yield* Clock.currentTimeMillis,
360
+ });
361
+ if (name === "second" && ++secondSubscriptions > 2) return Stream.never;
362
+ return Stream.make(value).pipe(
363
+ Stream.concat(Stream.fromEffect(decodeDefect).pipe(Stream.drain)),
364
+ );
365
+ }),
366
+ ),
367
+ }) as unknown as WsRpcProtocolClient;
368
+ const { activeSession, retryCount, supervisor } = yield* makeHarness();
369
+
370
+ yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("first"))));
371
+ const subscriptionFiber = yield* subscribe(
372
+ WS_METHODS.subscribeServerLifecycle,
373
+ {},
374
+ {
375
+ onExpectedFailure: () => Effect.void,
376
+ retryExpectedFailureAfter: "250 millis",
377
+ },
378
+ ).pipe(
379
+ Stream.runDrain,
380
+ Effect.provideService(EnvironmentSupervisor.EnvironmentSupervisor, supervisor),
381
+ Effect.forkChild,
382
+ );
383
+
384
+ const expectedFirstSessionDelays = [250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000];
385
+ const firstSessionDelays: number[] = [];
386
+ let previous = yield* Queue.take(subscriptions);
387
+ for (const delay of expectedFirstSessionDelays) {
388
+ yield* TestClock.adjust(delay);
389
+ const next = yield* Queue.take(subscriptions);
390
+ firstSessionDelays.push(next.at - previous.at);
391
+ previous = next;
392
+ }
393
+
394
+ // A replacement session starts a fresh backoff.
395
+ yield* SubscriptionRef.set(activeSession, Option.some(session(failingClient("second"))));
396
+ const replaced = yield* Queue.take(subscriptions);
397
+ yield* TestClock.adjust(250);
398
+ const retried = yield* Queue.take(subscriptions);
399
+ yield* Fiber.interrupt(subscriptionFiber);
400
+
401
+ expect(
402
+ firstSessionDelays,
403
+ "same-session decode retries reset to the base delay after every first value",
404
+ ).toEqual(expectedFirstSessionDelays);
405
+ expect([replaced.session, retried.session]).toEqual(["second", "second"]);
406
+ expect(retried.at - replaced.at).toBe(250);
407
+ expect(yield* Ref.get(retryCount)).toBe(0);
408
+ }),
409
+ );
410
+
411
it.effect("surfaces domain subscription failures without reconnecting", () =>
412
Effect.gen(function* () {
413
const domainError = new Error("terminal subscription rejected");
packages/client-runtime/src/rpc/client.ts
+48
-8
@@ -114,6 +114,7 @@ export const isAuthorizationFailure = (cause: Cause.Cause<unknown>) =>
114
);
115
116
const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000;
117
+const STREAM_FAILURE_RETRY_DELAY = Duration.seconds(1);
118
119
export const subscriptionRetryDelay = (base: Duration.Input, retries: number) =>
120
Duration.millis(
@@ -318,6 +319,8 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
319
EnvironmentRpcStreamFailure<TTag>
320
>;
321
let expectedFailureRetries = 0;
322
+ // Not reset by a first value: a stream can emit before every chunk failure.
323
+ let streamFailureRetries = 0;
324
const subscribeToSession = (): Stream.Stream<A, EnvironmentRpcStreamFailure<TTag>> =>
325
Stream.suspend(() =>
326
Stream.unwrap(
@@ -354,6 +357,11 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
357
: Effect.void,
358
),
359
Stream.catchCause((cause) => {
360
+ const retryAfter = (base: Duration.Input, retries: number) =>
361
+ Stream.fromEffect(Effect.sleep(subscriptionRetryDelay(base, retries))).pipe(
362
+ Stream.drain,
363
+ Stream.concat(subscribeToSession()),
364
+ );
365
const hasOnlyExpectedFailures =
366
cause.reasons.length > 0 &&
367
cause.reasons.every((reason) => reason._tag === "Fail");
@@ -362,7 +370,22 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
370
cause.reasons.every(
371
(reason) => reason._tag === "Fail" && isRpcClientError(reason.error),
372
);
365
- if (isTransportFailure) {
373
+ // The RPC client dies with a SchemaError when one stream chunk fails to
374
+ // decode, ending only that request while the socket stays open. A caller
375
+ // with `onDefect` owns its defect recovery.
376
+ const isChunkDecodeDefect =
377
+ options?.onDefect === undefined &&
378
+ cause.reasons.length > 0 &&
379
+ cause.reasons.every(
380
+ (reason) => reason._tag === "Die" && Schema.isSchemaError(reason.defect),
381
+ );
382
+ // Both end one stream while the session can stay healthy, so nothing else
383
+ // would replace it. A dead session is replaced by the supervisor, which
384
+ // interrupts this retry through switchMap. A finite search must settle
385
+ // rather than restart, so it keeps waiting for the next session.
386
+ const resubscribesOnSameSession =
387
+ tag !== ORCHESTRATION_V2_WS_METHODS.searchThreadStream;
388
+ if (isTransportFailure && !resubscribesOnSameSession) {
389
return Stream.fromEffect(
390
Effect.logWarning(
391
"Durable RPC subscription lost its transport; waiting for the next session.",
@@ -374,6 +397,29 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
397
),
398
).pipe(Stream.drain);
399
}
400
+ if (
401
+ resubscribesOnSameSession &&
402
+ (isTransportFailure || isChunkDecodeDefect)
403
+ ) {
404
+ return Stream.fromEffect(
405
+ Effect.logWarning(
406
+ "Durable RPC subscription stream ended; resubscribing.",
407
+ {
408
+ cause: Cause.pretty(cause),
409
+ method: tag,
410
+ environmentId: supervisor.target.environmentId,
411
+ },
412
+ ),
413
+ ).pipe(
414
+ Stream.drain,
415
+ Stream.concat(
416
+ retryAfter(
417
+ options?.retryExpectedFailureAfter ?? STREAM_FAILURE_RETRY_DELAY,
418
+ streamFailureRetries++,
419
+ ),
420
+ ),
421
+ );
422
+ }
423
if (hasOnlyExpectedFailures && options?.onExpectedFailure !== undefined) {
424
const handled = Stream.fromEffect(options.onExpectedFailure(cause)).pipe(
425
Stream.drain,
@@ -384,16 +430,10 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
430
) {
431
return handled;
432
}
387
- const retryDelay = subscriptionRetryDelay(
388
- options.retryExpectedFailureAfter,
389
- expectedFailureRetries,
390
- );
391
- expectedFailureRetries += 1;
433
return handled.pipe(
434
Stream.concat(
394
- Stream.fromEffect(Effect.sleep(retryDelay)).pipe(Stream.drain),
435
+ retryAfter(options.retryExpectedFailureAfter, expectedFailureRetries++),
436
),
396
- Stream.concat(subscribeToSession()),
437
);
438
}
439
// A reported defect parks until the next session or resubscribe. Failing would
packages/client-runtime/src/state/runtime.test.ts
+104
@@ -18,6 +18,7 @@ import {
18
WS_METHODS,
19
} from "@t3tools/contracts";
20
import * as Cause from "effect/Cause";
21
+import * as Clock from "effect/Clock";
22
import * as DateTime from "effect/DateTime";
23
import * as Deferred from "effect/Deferred";
24
import * as Effect from "effect/Effect";
@@ -29,6 +30,7 @@ import * as Queue from "effect/Queue";
30
import * as Schema from "effect/Schema";
31
import * as Stream from "effect/Stream";
32
import * as SubscriptionRef from "effect/SubscriptionRef";
33
+import * as TestClock from "effect/testing/TestClock";
34
import type { HttpClient } from "effect/http";
35
import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
36
@@ -43,6 +45,7 @@ import {
45
import * as EnvironmentRegistry from "../connection/registry.ts";
46
import * as EnvironmentSupervisor from "../connection/supervisor.ts";
47
import { EnvironmentRpcUnavailableError } from "../rpc/client.ts";
48
+import type { WsRpcProtocolClient } from "../rpc/protocol.ts";
49
import type * as RpcSession from "../rpc/session.ts";
50
import { createOrchestrationEnvironmentAtoms } from "./orchestration.ts";
51
import { createReviewEnvironmentAtoms } from "./review.ts";
@@ -54,6 +57,7 @@ import { createTerminalEnvironmentAtoms } from "./terminal.ts";
57
import {
58
createAtomCommandScheduler,
59
createEnvironmentQueryAtomFamily,
60
+ createEnvironmentRpcSubscriptionAtomFamily,
61
selectEnvironmentQueryData,
62
createEnvironmentSubscriptionAtomFamily,
63
createRuntimeCommand,
@@ -70,6 +74,8 @@ const QUERY_ENVIRONMENT = new PrimaryConnectionTarget({
74
wsBaseUrl: "wss://query.example.test",
75
});
76
77
+const decodeNumber = Schema.decodeUnknownEffect(Schema.Number);
78
+
79
const QUERY_RPC_SESSION = {} as RpcSession.RpcSession;
80
81
class TestQueryError extends Schema.TaggedError<TestQueryError>()("TestQueryError", {
@@ -393,6 +399,104 @@ describe("finite environment subscription lifecycle", () => {
399
expect(observed.some((result) => AsyncResult.isFailure(result))).toBe(false);
400
}),
401
);
402
+ it.effect(
403
+ "surfaces a decode defect from a finite RPC subscription instead of resubscribing",
404
+ () =>
405
+ Effect.scoped(
406
+ Effect.gen(function* () {
407
+ const firstSubscribed = yield* Deferred.make<void>();
408
+ const resubscribed = yield* Deferred.make<void>();
409
+ const surfaced = yield* Deferred.make<Cause.Cause<unknown>>();
410
+ let subscriptions = 0;
411
+ // Mirrors RpcClient, which decodes each chunk with `Effect.orDie`.
412
+ const decodeDefect = decodeNumber("not a number").pipe(Effect.orDie);
413
+ const client = {
414
+ [ORCHESTRATION_V2_WS_METHODS.searchThreadStream]: () => {
415
+ subscriptions += 1;
416
+ Deferred.doneUnsafe(subscriptions === 1 ? firstSubscribed : resubscribed, Exit.void);
417
+ return Stream.fromEffect(decodeDefect).pipe(Stream.drain);
418
+ },
419
+ } as unknown as WsRpcProtocolClient;
420
+ const rpcSession = {
421
+ client,
422
+ initialConfig: Effect.never,
423
+ subscribeServerConfig: () => Stream.never,
424
+ ready: Effect.void,
425
+ probe: Effect.void,
426
+ closed: Effect.never,
427
+ } as unknown as RpcSession.RpcSession;
428
+ const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
429
+ target: QUERY_ENVIRONMENT,
430
+ state: yield* SubscriptionRef.make(queryConnectionState()),
431
+ session: yield* SubscriptionRef.make(Option.some(rpcSession)),
432
+ prepared: yield* SubscriptionRef.make<Option.Option<PreparedConnection>>(Option.none()),
433
+ connect: Effect.void,
434
+ disconnect: Effect.void,
435
+ retryNow: Effect.void,
436
+ } satisfies EnvironmentSupervisor.EnvironmentSupervisor["Service"]);
437
+ const environmentRegistry = EnvironmentRegistry.EnvironmentRegistry.of({
438
+ followStream: (_environmentId: EnvironmentId, stream: Stream.Stream<unknown>) =>
439
+ Stream.provideService(
440
+ stream,
441
+ EnvironmentSupervisor.EnvironmentSupervisor,
442
+ supervisor,
443
+ ),
444
+ } as unknown as EnvironmentRegistry.EnvironmentRegistry["Service"]);
445
+ // Atoms run their own fibers; share the test clock so a pending retry is driven by it.
446
+ const runtime = Atom.runtime(
447
+ Layer.mergeAll(
448
+ Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, environmentRegistry),
449
+ Layer.succeed(Clock.Clock, yield* Clock.Clock),
450
+ ),
451
+ );
452
+ // Same options as the progressive thread find in orchestration.ts.
453
+ const family = createEnvironmentRpcSubscriptionAtomFamily(runtime, {
454
+ label: "test.finite-rpc-subscription",
455
+ tag: ORCHESTRATION_V2_WS_METHODS.searchThreadStream,
456
+ completeWhen: (result) => result.complete !== false,
457
+ idleTtlMs: 0,
458
+ });
459
+ const atom = family({
460
+ environmentId: QUERY_ENVIRONMENT.environmentId,
461
+ input: {} as never,
462
+ });
463
+ const registry = AtomRegistry.make();
464
+ const unsubscribe = registry.subscribe(
465
+ atom,
466
+ (result) => {
467
+ if (AsyncResult.isFailure(result))
468
+ Deferred.doneUnsafe(surfaced, Exit.succeed(result.cause));
469
+ },
470
+ { immediate: true },
471
+ );
472
+ yield* Effect.addFinalizer(() =>
473
+ Effect.sync(() => {
474
+ unsubscribe();
475
+ registry.dispose();
476
+ }),
477
+ );
478
+ yield* Deferred.await(firstSubscribed);
479
+ // Lets any same-session retry run, so its resubscription can win the race below.
480
+ yield* TestClock.adjust("1 minute");
481
+ const outcome = yield* Effect.raceFirst(
482
+ Deferred.await(surfaced).pipe(
483
+ Effect.map((cause) => ({ _tag: "surfaced", cause }) as const),
484
+ ),
485
+ Deferred.await(resubscribed).pipe(Effect.as({ _tag: "resubscribed" } as const)),
486
+ );
487
+
488
+ expect(
489
+ outcome._tag,
490
+ "finite subscription resubscribed after a decode defect instead of failing its consumer",
491
+ ).toBe("surfaced");
492
+ if (outcome._tag === "surfaced") {
493
+ expect(Cause.hasDies(outcome.cause)).toBe(true);
494
+ expect(Schema.isSchemaError(Cause.squash(outcome.cause))).toBe(true);
495
+ }
496
+ expect(subscriptions).toBe(1);
497
+ }),
498
+ ),
499
+ );
500
it.effect.each([
501
{ completedTtlMs: undefined, subscriptions: 2 },
502
{ completedTtlMs: 60_000, subscriptions: 1 },