@devries-cameron20 / t3code Mirror · read-only / commits / de21fbcff3
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 21:00 UTC

fix(client): a thread whose subscription dies on a defect resubscribes instead of staying blank (#25919)

Julius Marminge committed Nov 11, 2026 at 15:55 UTC de21fbcff3f0bd50058030d973c31c4255141c9e
3 files changed +141 -27
packages/client-runtime/src/rpc/client.ts
+8 -6
@@ -114,6 +114,11 @@ export const isAuthorizationFailure = (cause: Cause.Cause<unknown>) =>
114
115 const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000;
116
117 +export const subscriptionRetryDelay = (base: Duration.Input, retries: number) =>
118 + Duration.millis(
119 + Math.min(Duration.toMillis(base) * 2 ** retries, MAX_EXPECTED_FAILURE_RETRY_DELAY_MS),
120 + );
121 +
122 export type EnvironmentRpcInput<TTag extends EnvironmentRpcTag> = Parameters<RpcMethod<TTag>>[0];
123
124 export type EnvironmentRpcSuccess<TTag extends EnvironmentUnaryRpcTag> =
@@ -378,12 +383,9 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
383 ) {
384 return handled;
385 }
381 - const retryDelay = Duration.millis(
382 - Math.min(
383 - Duration.toMillis(options.retryExpectedFailureAfter) *
384 - 2 ** expectedFailureRetries,
385 - MAX_EXPECTED_FAILURE_RETRY_DELAY_MS,
386 - ),
386 + const retryDelay = subscriptionRetryDelay(
387 + options.retryExpectedFailureAfter,
388 + expectedFailureRetries,
389 );
390 expectedFailureRetries += 1;
391 return handled.pipe(
packages/client-runtime/src/state/threads-sync.test.ts
+66 -5
@@ -118,6 +118,8 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
118 readonly latestLocalTurnOrdinal?: number | null;
119 };
120 readonly httpSnapshot?: ThreadSnapshotLoadResult;
121 + /** Answers per load call; the last one repeats. Overrides `httpSnapshot`. */
122 + readonly httpSnapshots?: readonly ThreadSnapshotLoadResult[];
123 /** The test sends the completion marker itself instead of the fake server sending it first. */
124 readonly completionMarker?: boolean;
125 readonly resumeCache?: NonNullable<Parameters<typeof makeEnvironmentThreadState>[1]>;
@@ -193,11 +195,12 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
195 );
196 const snapshotLoader = ThreadSnapshotLoader.ThreadSnapshotLoader.of({
197 load: (_prepared, threadId) =>
196 - Ref.update(loaderCalls, (count) => count + 1).pipe(
197 - Effect.as(
198 + Ref.updateAndGet(loaderCalls, (count) => count + 1).pipe(
199 + Effect.map((count) =>
200 threadId === THREAD_ID
199 - ? (options?.httpSnapshot ??
200 - ({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult))
201 + ? (options?.httpSnapshots?.[Math.min(count, options.httpSnapshots.length) - 1] ??
202 + options?.httpSnapshot ??
203 + ({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult))
204 : ({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult),
205 ),
206 ),
@@ -798,6 +801,27 @@ describe("EnvironmentThreads", () => {
801 }),
802 );
803
804 + it.effect("keeps asking a mounted thread over HTTP after a miss until it exists", () =>
805 + Effect.gen(function* () {
806 + const harness = yield* makeHarness({
807 + httpSnapshots: [
808 + { _tag: "missing" },
809 + { _tag: "missing" },
810 + {
811 + _tag: "present",
812 + snapshot: { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, projection: BASE_PROJECTION },
813 + },
814 + ],
815 + });
816 + yield* awaitThreadState(harness.observed, (value) => value.status === "deleted");
817 + yield* TestClock.adjust("250 millis");
818 + yield* TestClock.adjust("500 millis");
819 + const state = yield* awaitThreadState(harness.observed, (value) => value.status === "live");
820 + expect(Option.getOrThrow(state.data)).toEqual(BASE_PROJECTION);
821 + expect(yield* Ref.get(harness.loaderCalls)).toBe(3);
822 + }),
823 + );
824 +
825 it.effect.each([
826 // The HTTP retry decides, so a missing thread never flashes an error.
827 {
@@ -869,7 +893,7 @@ describe("EnvironmentThreads", () => {
893 let configReads = 0;
894 const harness = yield* makeHarness({
895 httpSnapshot: { _tag: "missing" },
872 - // The retry's config read dies too, so a stale flag would resubscribe forever.
896 + // The retry's config read dies too, so a stale flag would resubscribe without backing off.
897 initialConfig: Effect.suspend(() =>
898 ++configReads === 1
899 ? Effect.succeed({ threadSnapshotPagination: true })
@@ -2382,6 +2406,43 @@ describe("EnvironmentThreads", () => {
2406 }),
2407 );
2408
2409 + it.effect(
2410 + "resubscribes after a snapshot decode defect ends the thread subscription (#23041)",
2411 + () =>
2412 + Effect.gen(function* () {
2413 + const harness = yield* makeHarness({ httpSnapshot: { _tag: "unavailable" } });
2414 + yield* SubscriptionRef.changes(harness.subscriptionCount).pipe(
2415 + Stream.filter((count) => count >= 1),
2416 + Stream.runHead,
2417 + );
2418 + yield* Queue.offer(harness.inputs, TEST_DEFECT);
2419 + const failed = yield* awaitThreadState(harness.observed, (value) =>
2420 + Option.isSome(value.error),
2421 + );
2422 + expect(failed.status).toBe("empty");
2423 + expect(Option.getOrThrow(failed.error)).toBe("Could not synchronize the thread.");
2424 +
2425 + yield* TestClock.adjust("60 seconds");
2426 + expect(yield* SubscriptionRef.get(harness.subscriptionCount)).toBe(2);
2427 +
2428 + yield* Queue.offer(
2429 + harness.inputs,
2430 + snapshot({
2431 + ...BASE_PROJECTION,
2432 + thread: { ...BASE_PROJECTION.thread, title: "Recovered thread" },
2433 + }),
2434 + );
2435 + const recovered = yield* awaitThreadState(
2436 + harness.observed,
2437 + (value) =>
2438 + value.status === "live" &&
2439 + Option.isSome(value.data) &&
2440 + value.data.value.thread.title === "Recovered thread",
2441 + );
2442 + expect(Option.isNone(recovered.error)).toBe(true);
2443 + }),
2444 + );
2445 +
2446 it.effect("keeps replayed updates synchronizing until the completion marker arrives", () =>
2447 Effect.gen(function* () {
2448 const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true });
packages/client-runtime/src/state/threads.ts
+67 -16
@@ -32,7 +32,12 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";
32 import * as Persistence from "../platform/persistence.ts";
33 import { runCachePersistence } from "./cachePersistence.ts";
34 import * as ManagedRelay from "../relay/managedRelay.ts";
35 -import { isAuthorizationFailure, request, subscribeDynamic } from "../rpc/client.ts";
35 +import {
36 + isAuthorizationFailure,
37 + request,
38 + subscribeDynamic,
39 + subscriptionRetryDelay,
40 +} from "../rpc/client.ts";
41 import { parseThreadKey, threadKey } from "./entities.ts";
42 import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts";
43 import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts";
@@ -55,6 +60,8 @@ import {
60 type EnvironmentThreadStatus,
61 } from "./threadState.ts";
62
63 +const THREAD_SUBSCRIPTION_RETRY_DELAY = "250 millis";
64 +
65 function statusWithoutLiveData(
66 data: Option.Option<OrchestrationV2ThreadProjection>,
67 ): EnvironmentThreadStatus {
@@ -967,8 +974,8 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
974 (yield* Ref.get(socketOnlyOpen)) && Option.isNone((yield* SubscriptionRef.get(state)).data)
975 );
976 });
970 - // A defect parks the subscription, so a socket-only open resubscribes once.
971 - const httpRetries = yield* Queue.unbounded<void>();
977 + // A stream defect parks the subscription until something resubscribes it.
978 + const resubscribes = yield* Queue.unbounded<void>();
979 const markSynchronizing = Effect.gen(function* () {
980 if (yield* Ref.get(resumingLive)) return;
981 yield* SubscriptionRef.update(state, (current) =>
@@ -979,15 +986,27 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
986 });
987
988 yield* markSynchronizing;
982 - yield* Effect.forkScoped(
989 + // A 404 can arrive before the thread exists, so a miss asks HTTP again while mounted.
990 + let httpMisses = 0;
991 + let defectRetries = 0;
992 + const backOffAfterDefect = Effect.suspend(() => {
993 + const retryDelay = subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, defectRetries);
994 + defectRetries += 1;
995 + return setStreamError("Could not synchronize the thread.").pipe(
996 + Effect.andThen(Effect.sleep(retryDelay)),
997 + );
998 + });
999 + const followThread = (): Effect.Effect<
1000 + void,
1001 + never,
1002 + EnvironmentSupervisor.EnvironmentSupervisor
1003 + > =>
1004 subscribeDynamic(
1005 ORCHESTRATION_V2_WS_METHODS.subscribeThread,
1006 Effect.fn("EnvironmentThreadState.makeSubscribeInput")(function* (session) {
1007 let current = yield* SubscriptionRef.get(state);
987 - // A prior definitive miss (or delete event) already cleared this thread.
988 - // Park the subscription attempt without opening the socket so we do not
989 - // retry forever against a known-missing id.
990 - if (current.status === "deleted") {
1008 + // A delete event already cleared this thread. Park without opening the socket.
1009 + if (current.status === "deleted" && httpMisses === 0) {
1010 return yield* Effect.never;
1011 }
1012
@@ -1024,6 +1043,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
1043 yield* snapshotLoader.load(prepared, threadId);
1044 switch (httpResult._tag) {
1045 case "present": {
1046 + httpMisses = 0;
1047 if (canLoadHistory && httpResult.history !== undefined) {
1048 yield* Ref.set(acceptsBoundedSocketSnapshots, true);
1049 }
@@ -1052,9 +1072,13 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
1072 break;
1073 }
1074 case "missing": {
1055 - yield* setDeleted();
1075 + if (current.status !== "deleted") yield* setDeleted();
1076 // A 404 also means "not created yet". Only a delete event may outlive this mount.
1077 if (resumeCache?.owner === owner) resumeCache.snapshot = undefined;
1078 + yield* Effect.sleep(
1079 + subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, httpMisses++),
1080 + );
1081 + yield* Queue.offer(resubscribes, undefined);
1082 return yield* Effect.never;
1083 }
1084 case "unavailable": {
@@ -1085,22 +1109,49 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
1109 {
1110 // A socket-only open that fails retries over HTTP, which tells a
1111 // missing thread apart from a failure, before showing an error.
1088 - onDefect: Effect.fn("EnvironmentThreadState.onDefect")(function* () {
1112 + onDefect: Effect.fn("EnvironmentThreadState.onDefect")(function* (cause) {
1113 if (yield* failedSocketOnlyOpen) {
1114 yield* Ref.set(socketOnlyOpen, false);
1091 - return yield* Queue.offer(httpRetries, undefined);
1115 + return yield* Queue.offer(resubscribes, undefined);
1116 + }
1117 + if (!Cause.hasDies(cause)) {
1118 + return yield* setStreamError("Could not synchronize the thread.");
1119 }
1093 - yield* setStreamError("Could not synchronize the thread.");
1120 + yield* backOffAfterDefect;
1121 + yield* Queue.offer(resubscribes, undefined);
1122 }),
1123 onExpectedFailure: Effect.fn("EnvironmentThreadState.onExpectedFailure")(function* (cause) {
1124 const retryOverHttp = !isAuthorizationFailure(cause) && (yield* failedSocketOnlyOpen);
1125 if (!retryOverHttp) yield* setStreamError(formatThreadError(cause));
1126 }),
1099 - retryExpectedFailureAfter: "250 millis",
1100 - resubscribe: Stream.merge(foregroundResubscriptions, Stream.fromQueue(httpRetries)),
1127 + retryExpectedFailureAfter: THREAD_SUBSCRIPTION_RETRY_DELAY,
1128 + resubscribe: Stream.merge(foregroundResubscriptions, Stream.fromQueue(resubscribes)),
1129 },
1102 - ).pipe(Stream.runForEachArray(applyItems)),
1103 - );
1130 + ).pipe(
1131 + Stream.runForEachArray((items) =>
1132 + applyItems(items).pipe(
1133 + Effect.andThen(
1134 + Effect.sync(() => {
1135 + defectRetries = 0;
1136 + }),
1137 + ),
1138 + ),
1139 + ),
1140 + // A defect while applying items ends the stream; resubscribe for a fresh snapshot.
1141 + Effect.catchCause((cause) =>
1142 + Cause.hasInterrupts(cause)
1143 + ? Effect.interrupt
1144 + : Effect.logWarning("Thread subscription failed; resubscribing.", {
1145 + cause: Cause.pretty(cause),
1146 + environmentId,
1147 + threadId,
1148 + }).pipe(
1149 + Effect.andThen(backOffAfterDefect),
1150 + Effect.andThen(Effect.suspend(followThread)),
1151 + ),
1152 + ),
1153 + );
1154 + yield* Effect.forkScoped(followThread());
1155
1156 return state;
1157 });