The best way to control Claude Code, Codex and more from any computer or phone
fix(client-runtime): an unexpected failure no longer stops the reconnect loop for good (#39420)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Viktor Leukocyte committed
Nov 19, 2026 at 23:54 UTC
38bac7091a71eb6e01b63b70f6b1bae7441ed5bc
3 files changed
+531
-106
packages/client-runtime/src/connection/supervisor.test.ts
+323
-16
@@ -6,14 +6,19 @@ import {
6
} from "@t3tools/contracts";
7
import { describe, expect, it } from "@effect/vitest";
8
import { RelayClientTracer } from "@t3tools/shared/relayTracing";
9
+import * as Cause from "effect/Cause";
10
import * as Deferred from "effect/Deferred";
11
import * as Effect from "effect/Effect";
12
+import * as Exit from "effect/Exit";
13
import * as Fiber from "effect/Fiber";
14
import * as Layer from "effect/Layer";
15
+import * as Logger from "effect/Logger";
16
import * as Option from "effect/Option";
17
import * as Queue from "effect/Queue";
18
import * as Random from "effect/Random";
19
import * as Ref from "effect/Ref";
20
+import * as References from "effect/References";
21
+import * as Scope from "effect/Scope";
22
import * as Stream from "effect/Stream";
23
import * as SubscriptionRef from "effect/SubscriptionRef";
24
import * as Clock from "effect/Clock";
@@ -167,6 +172,18 @@ function awaitState(
172
);
173
}
174
175
+function recordLogs(logs: Array<unknown>, annotations?: Array<unknown>) {
176
+ return Logger.layer(
177
+ [
178
+ Logger.make(({ message, fiber }) => {
179
+ logs.push(message);
180
+ annotations?.push(fiber.getRef(References.CurrentLogAnnotations));
181
+ }),
182
+ ],
183
+ { mergeWithExisting: false },
184
+ );
185
+}
186
+
187
const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: {
188
readonly networkStatus?: NetworkStatus;
189
readonly prepare?: (
@@ -174,6 +191,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
191
target: ConnectionTarget,
192
) => Effect.Effect<PreparedConnection, ConnectionAttemptError>;
193
readonly ready?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
194
+ readonly release?: (attempt: number) => Effect.Effect<void>;
195
readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
196
readonly configEvents?: Stream.Stream<ServerConfigStreamEvent> | undefined;
197
readonly initialConfig?: (attempt: number) => Effect.Effect<ServerConfig, ConnectionAttemptError>;
@@ -245,7 +263,10 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
263
inboundFrames: options?.inboundFrames ?? Effect.succeed(0),
264
closed: Deferred.await(closed),
265
} satisfies RpcSession.RpcSession),
248
- () => Ref.update(releaseCount, (count) => count + 1),
266
+ () =>
267
+ Ref.update(releaseCount, (count) => count + 1).pipe(
268
+ Effect.andThen(options?.release?.(attempt) ?? Effect.void),
269
+ ),
270
);
271
272
yield* reportProgress({ stage: "synchronizing", prepared });
@@ -308,12 +329,12 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
329
reason,
330
})),
331
closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* (
311
- error = transient("Session closed."),
332
+ exit: Exit.Exit<never, ConnectionTransientError> = Exit.fail(transient("Session closed.")),
333
) {
334
const sessions = yield* Ref.get(closedSessions);
335
const latest = sessions.at(-1);
336
if (latest) {
316
- yield* Deferred.fail(latest, error);
337
+ yield* Deferred.done(latest, exit);
338
}
339
}),
340
};
@@ -581,9 +602,11 @@ describe("EnvironmentSupervisor", () => {
602
? Effect.die(new Error("Native transport defect."))
603
: Effect.succeed(PREPARED_CONNECTION),
604
});
605
+ const logs: unknown[] = [];
606
+ const annotations: unknown[] = [];
607
const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
608
initiallyDesired: true,
586
- }).pipe(Effect.provide(harness.dependencies));
609
+ }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))));
610
611
const failed = yield* awaitState(
612
supervisor.state,
@@ -596,6 +619,10 @@ describe("EnvironmentSupervisor", () => {
619
message: "Test environment connection failed unexpectedly.",
620
},
621
});
622
+ expect(logs).toEqual([["Connection attempt failed with an unexpected defect."]]);
623
+ expect(annotations).toMatchObject([
624
+ { "environment.id": RELAY_TARGET.environmentId, errorName: "Error" },
625
+ ]);
626
627
yield* TestClock.adjust("3 seconds");
628
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
@@ -603,42 +630,322 @@ describe("EnvironmentSupervisor", () => {
630
}).pipe(Effect.provide(TestClock.layer())),
631
);
632
606
- it.effect("explicit retry interrupts the current backoff", () =>
633
+ it.effect.each([
634
+ {
635
+ label: "the attempt's network-status read dies",
636
+ firstPrepare: Effect.never,
637
+ statusDefect: true,
638
+ log: "Connection attempt failed with an unexpected defect.",
639
+ },
640
+ {
641
+ label: "an attempt interrupts itself",
642
+ firstPrepare: Effect.interrupt,
643
+ statusDefect: false,
644
+ log: "Connection attempt failed with an unexpected interruption.",
645
+ },
646
+ ])("retries after $label", ({ firstPrepare, statusDefect, log }) =>
647
+ Effect.gen(function* () {
648
+ const harness = yield* makeHarness({
649
+ prepare: (attempt) => (attempt === 1 ? firstPrepare : Effect.succeed(PREPARED_CONNECTION)),
650
+ });
651
+ const statusReads = yield* Ref.make(0);
652
+ const logs: unknown[] = [];
653
+ const annotations: Array<Record<string, unknown>> = [];
654
+ const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
655
+ initiallyDesired: true,
656
+ }).pipe(
657
+ Effect.provideService(
658
+ Connectivity.Connectivity,
659
+ Connectivity.Connectivity.of({
660
+ // The supervisor reads once at startup; the second read belongs to the first attempt.
661
+ status: Ref.updateAndGet(statusReads, (count) => count + 1).pipe(
662
+ Effect.flatMap((count) =>
663
+ statusDefect && count === 2
664
+ ? Effect.die(new Error("Platform connectivity defect."))
665
+ : Effect.succeed("online" as const),
666
+ ),
667
+ ),
668
+ changes: Stream.never,
669
+ }),
670
+ ),
671
+ Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))),
672
+ );
673
+
674
+ const failed = yield* awaitState(
675
+ supervisor.state,
676
+ (state) => state.phase === "backoff" && state.attempt === 1,
677
+ );
678
+ expect(failed).toMatchObject({
679
+ generation: 0,
680
+ lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
681
+ });
682
+ expect(logs).toEqual([[log]]);
683
+ expect(Object.hasOwn(annotations[0]!, "errorType")).toBe(statusDefect);
684
+
685
+ yield* TestClock.adjust("999 millis");
686
+ expect(yield* Ref.get(harness.prepareCount)).toBe(1);
687
+ yield* TestClock.adjust("2 seconds");
688
+ yield* awaitState(supervisor.state, (state) => state.phase === "connected");
689
+ expect(yield* Ref.get(harness.prepareCount)).toBe(2);
690
+ }).pipe(Effect.provide(TestClock.layer())),
691
+ );
692
+
693
+ it.effect.each([
694
+ {
695
+ label: "is interrupted",
696
+ closed: Exit.interrupt(),
697
+ releaseDefect: false,
698
+ log: "Connection attempt failed with an unexpected interruption.",
699
+ },
700
+ {
701
+ label: "fails while interrupted",
702
+ closed: Exit.failCause(
703
+ Cause.combine(Cause.fail(transient("Session closed.")), Cause.interrupt()),
704
+ ),
705
+ releaseDefect: false,
706
+ log: null,
707
+ },
708
+ {
709
+ label: "closes while its release defects",
710
+ closed: Exit.fail(transient("Session closed.")),
711
+ releaseDefect: true,
712
+ log: "Connection release failed with an unexpected defect.",
713
+ },
714
+ ])("reconnects soon after a stable session $label", ({ closed, releaseDefect, log }) =>
715
Effect.gen(function* () {
716
const harness = yield* makeHarness({
717
prepare: (attempt) =>
718
attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
719
+ release: (session) =>
720
+ releaseDefect && session === 1
721
+ ? Effect.die(new Error("Session release defect."))
722
+ : Effect.void,
723
});
724
+ const logs: unknown[] = [];
725
const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
726
initiallyDesired: true,
614
- }).pipe(Effect.provide(harness.dependencies));
727
+ }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))));
728
729
yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
730
+ yield* TestClock.adjust("2 seconds");
731
+ yield* awaitState(
732
+ supervisor.state,
733
+ (state) => state.phase === "connected" && state.generation === 1,
734
+ );
735
+ yield* TestClock.adjust("1 minute");
736
+ yield* harness.closeLatestSession(closed);
737
+ const lost = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
738
+ expect(logs).toEqual(log === null ? [] : [[log]]);
739
+ expect(lost).toMatchObject({ phase: "backoff", generation: 1 });
740
+ expect(lost.retryAt).toBeLessThanOrEqual((yield* Clock.currentTimeMillis) + 250);
741
+
742
+ yield* TestClock.adjust("250 millis");
743
+ const replaced = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
744
+ expect(replaced.generation).toBe(2);
745
+ }).pipe(Effect.provide(TestClock.layer())),
746
+ );
747
+
748
+ it.effect(
749
+ "does not reuse a live session's generation when its attempt defects after going live",
750
+ () =>
751
+ Effect.gen(function* () {
752
+ const harness = yield* makeHarness({
753
+ prepare: (attempt) =>
754
+ attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
755
+ });
756
+ let logCalls = 0;
757
+ const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
758
+ initiallyDesired: true,
759
+ }).pipe(
760
+ Effect.provide(
761
+ Layer.merge(
762
+ harness.dependencies,
763
+ Logger.layer(
764
+ [
765
+ // Logging the lost session's interruption throws, so the attempt itself defects.
766
+ Logger.make(() => {
767
+ logCalls += 1;
768
+ if (logCalls === 1) throw new Error("Logger defect.");
769
+ }),
770
+ ],
771
+ { mergeWithExisting: false },
772
+ ),
773
+ ),
774
+ ),
775
+ );
776
+
777
+ yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
778
+ yield* TestClock.adjust("2 seconds");
779
+ yield* awaitState(
780
+ supervisor.state,
781
+ (state) => state.phase === "connected" && state.generation === 1,
782
+ );
783
+ yield* harness.closeLatestSession(Exit.interrupt());
784
+ const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
785
+ expect(backoff).toMatchObject({ generation: 1, lastFailure: null });
786
+
787
+ yield* TestClock.adjust("4 seconds");
788
+ const replaced = yield* awaitState(
789
+ supervisor.state,
790
+ (state) => state.phase === "connected",
791
+ );
792
+ expect(replaced.generation).toBe(2);
793
+ }).pipe(Effect.provide(TestClock.layer())),
794
+ );
795
+
796
+ it.effect("restarts the loop after a defect outside an attempt", () =>
797
+ Effect.gen(function* () {
798
+ const harness = yield* makeHarness();
799
+ const logs: unknown[] = [];
800
+ let randomReads = 0;
801
+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
802
+ initiallyDesired: true,
803
+ }).pipe(
804
+ Effect.provideService(Random.Random, {
805
+ // Both session losses below die here, so the second reaches the restart handler again.
806
+ nextDoubleUnsafe: () => {
807
+ randomReads += 1;
808
+ if (randomReads <= 2) throw new Error("Random defect.");
809
+ return 1 - Number.EPSILON;
810
+ },
811
+ nextIntUnsafe: () => 0,
812
+ }),
813
+ Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
814
+ );
815
+
816
+ yield* awaitState(
817
+ supervisor.state,
818
+ (state) => state.phase === "connected" && state.generation === 1,
819
+ );
820
+ yield* harness.closeLatestSession();
821
+ const restarted = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
822
+ expect(logs).toEqual([["Connection loop failed with an unexpected defect."]]);
823
+ expect(restarted).toMatchObject({
824
+ attempt: 1,
825
+ generation: 1,
826
+ lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
827
+ retryAt: (yield* Clock.currentTimeMillis) + 300_000,
828
+ });
829
+
830
+ yield* TestClock.adjust("299999 millis");
831
+ expect(yield* Ref.get(harness.sessionCount)).toBe(1);
832
+ yield* TestClock.adjust("1 millis");
833
+ const reconnected = yield* awaitState(
834
+ supervisor.state,
835
+ (state) => state.phase === "connected",
836
+ );
837
+ expect(reconnected.generation).toBe(2);
838
+
839
+ yield* harness.setNetworkStatus("offline");
840
+ yield* harness.closeLatestSession();
841
+ const offline = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
842
+ expect(offline).toMatchObject({
843
+ phase: "offline",
844
+ attempt: 1,
845
+ lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
846
+ });
847
+
848
yield* supervisor.retryNow;
618
- yield* awaitState(supervisor.state, (state) => state.phase === "connected");
849
+ const retried = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
850
+ expect(retried.generation).toBe(3);
851
+ }).pipe(Effect.provide(TestClock.layer())),
852
+ );
853
620
- expect(yield* Ref.get(harness.prepareCount)).toBe(2);
621
- }),
854
+ it.effect.each(["connecting", "connected"] as const)(
855
+ "stops attempting when its scope closes while %s",
856
+ (phase) =>
857
+ Effect.gen(function* () {
858
+ const preparing = yield* Deferred.make<void>();
859
+ const harness = yield* makeHarness({
860
+ prepare: () =>
861
+ Deferred.succeed(preparing, undefined).pipe(
862
+ Effect.andThen(
863
+ phase === "connected" ? Effect.succeed(PREPARED_CONNECTION) : Effect.never,
864
+ ),
865
+ ),
866
+ });
867
+ const logs: unknown[] = [];
868
+ const scope = yield* Scope.make();
869
+ const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
870
+ initiallyDesired: true,
871
+ }).pipe(
872
+ Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
873
+ Scope.provide(scope),
874
+ );
875
+
876
+ yield* Deferred.await(preparing);
877
+ yield* awaitState(supervisor.state, (state) => state.phase === phase);
878
+ // Yielding between finalizers lets the loop run if the queue shuts down before its interrupt.
879
+ yield* Scope.close(scope, Exit.void).pipe(
880
+ Effect.provideService(References.MaxOpsBeforeYield, 1),
881
+ );
882
+ yield* TestClock.adjust("10 minutes");
883
+ expect(logs).toEqual([]);
884
+ expect(yield* Ref.get(harness.prepareCount)).toBe(1);
885
+ }).pipe(Effect.provide(TestClock.layer())),
886
);
887
624
- it.effect("keeps blocked failures idle until an external signal requests another attempt", () =>
888
+ it.effect("explicit retry interrupts the current backoff", () =>
889
Effect.gen(function* () {
890
const harness = yield* makeHarness({
891
prepare: (attempt) =>
628
- attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION),
892
+ attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
893
});
894
const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
895
initiallyDesired: true,
896
}).pipe(Effect.provide(harness.dependencies));
897
634
- yield* awaitState(supervisor.state, (state) => state.phase === "blocked");
635
- yield* TestClock.adjust("1 hour");
636
- expect(yield* Ref.get(harness.prepareCount)).toBe(1);
637
-
898
+ yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
899
yield* supervisor.retryNow;
900
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
901
+
902
expect(yield* Ref.get(harness.prepareCount)).toBe(2);
641
- }).pipe(Effect.provide(TestClock.layer())),
903
+ }),
904
+ );
905
+
906
+ it.effect.each([
907
+ { label: "fails", cause: Cause.fail(blocked()) },
908
+ {
909
+ label: "fails while interrupted",
910
+ cause: Cause.combine(Cause.fail(blocked()), Cause.interrupt()),
911
+ },
912
+ ])(
913
+ "keeps a blocked host that $label idle until an external signal requests another attempt",
914
+ ({ cause }) =>
915
+ Effect.gen(function* () {
916
+ const harness = yield* makeHarness();
917
+ const driver = yield* ConnectionDriver.ConnectionDriver.pipe(
918
+ Effect.provide(harness.dependencies),
919
+ );
920
+ const connects = yield* Ref.make(0);
921
+ const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
922
+ initiallyDesired: true,
923
+ }).pipe(
924
+ // The driver keeps only the typed error of a failed route, so the cause is mixed here.
925
+ Effect.provideService(ConnectionDriver.ConnectionDriver, {
926
+ ...driver,
927
+ connect: (entry, reportProgress) =>
928
+ Ref.updateAndGet(connects, (count) => count + 1).pipe(
929
+ Effect.flatMap((count) =>
930
+ count === 1 ? Effect.failCause(cause) : driver.connect(entry, reportProgress),
931
+ ),
932
+ ),
933
+ }),
934
+ Effect.provide(harness.dependencies),
935
+ );
936
+
937
+ const failed = yield* awaitState(supervisor.state, (state) => state.lastFailure !== null);
938
+ expect(failed).toMatchObject({
939
+ phase: "blocked",
940
+ lastFailure: { _tag: "ConnectionBlockedError", detail: "Authentication required." },
941
+ });
942
+ yield* TestClock.adjust("1 hour");
943
+ expect(yield* Ref.get(connects)).toBe(1);
944
+
945
+ yield* supervisor.retryNow;
946
+ yield* awaitState(supervisor.state, (state) => state.phase === "connected");
947
+ expect(yield* Ref.get(connects)).toBe(2);
948
+ }).pipe(Effect.provide(TestClock.layer())),
949
);
950
951
it.effect("retries a blocked connection when platform credentials change", () =>
packages/client-runtime/src/connection/supervisor.ts
+123
-90
@@ -149,16 +149,6 @@ type EstablishmentEvent =
149
| { readonly _tag: "Interrupted"; readonly resetRetry: boolean }
150
| { readonly _tag: "TimedOut" };
151
152
-function exitUnlessInterrupted<A, E, R>(
153
- effect: Effect.Effect<A, E, R>,
154
-): Effect.Effect<Exit.Exit<A, E>, never, R> {
155
- return Effect.matchCauseEffect(effect, {
156
- onFailure: (cause) =>
157
- Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(Exit.failCause(cause)),
158
- onSuccess: (value) => Effect.succeed(Exit.succeed(value)),
159
- });
160
-}
161
-
152
export interface EnvironmentSupervisorOptions {
153
readonly initiallyDesired?: boolean;
154
readonly learnRoutes?: (input: {
@@ -247,37 +237,27 @@ function connectingState(
237
};
238
}
239
250
-function failureFromExit<A>(
240
+function failureFromCause(
241
target: ConnectionTarget,
252
- exit: Exit.Exit<A, TracedAttemptFailure>,
242
+ cause: Cause.Cause<TracedAttemptFailure>,
243
established: boolean,
244
stable: boolean,
255
-): AttemptOutcome {
256
- if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) {
257
- return { _tag: "Interrupted", established, stable, resetRetry: false };
258
- }
259
- const typedFailure = exit.cause.reasons.find(Cause.isFailReason);
260
- if (typedFailure) {
261
- return {
262
- _tag: "Failure",
263
- established,
264
- stable,
265
- failure: typedFailure.error,
266
- };
267
- }
245
+) {
246
return {
247
_tag: "Failure",
248
established,
249
stable,
272
- failure: {
273
- error: new ConnectionTransientError({
274
- reason: "transport",
275
- ...translatableDetail(
276
- msg("{host} connection failed unexpectedly.", { host: target.label }),
277
- ),
278
- }),
279
- attemptSpan: Option.none(),
280
- },
250
+ failure: cause.reasons.find(Cause.isFailReason)?.error ?? unexpectedFailure(target),
251
+ } satisfies AttemptOutcome;
252
+}
253
+
254
+function unexpectedFailure(target: ConnectionTarget): TracedAttemptFailure {
255
+ return {
256
+ error: new ConnectionTransientError({
257
+ reason: "transport",
258
+ ...translatableDetail(msg("{host} connection failed unexpectedly.", { host: target.label })),
259
+ }),
260
+ attemptSpan: Option.none(),
261
};
262
}
263
@@ -496,9 +476,27 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
476
}),
477
);
478
479
+ const logUnexpectedFailure = (
480
+ failed: "attempt" | "release" | "loop",
481
+ cause: Cause.Cause<TracedAttemptFailure>,
482
+ ) => {
483
+ const died = cause.reasons.find(Cause.isDieReason);
484
+ if (!died && cause.reasons.some(Cause.isFailReason)) return Effect.void;
485
+ return Effect.logError(
486
+ `Connection ${failed} failed with an unexpected ${died ? "defect" : "interruption"}.`,
487
+ ).pipe(
488
+ Effect.annotateLogs({
489
+ "environment.id": target.environmentId,
490
+ "environment.label": target.label,
491
+ "cause.reason_count": cause.reasons.length,
492
+ ...(died ? safeErrorLogAttributes(died.defect) : {}),
493
+ }),
494
+ );
495
+ };
496
+
497
const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* (
498
attempt: number,
501
- generation: number,
499
+ nextGeneration: number,
500
lastFailure: ConnectionAttemptError | null,
501
progress: ConnectionDriver.ConnectionDriverProgress,
502
) {
@@ -506,13 +504,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
504
yield* SubscriptionRef.set(prepared, Option.some(progress.prepared));
505
}
506
yield* setState(
509
- connectingState(yield* Ref.get(intent), generation, attempt, lastFailure, progress.stage),
507
+ connectingState(yield* Ref.get(intent), nextGeneration, attempt, lastFailure, progress.stage),
508
);
509
});
510
511
const establishConnection = Effect.fnUntraced(function* (
512
attempt: number,
515
- generation: number,
513
+ nextGeneration: number,
514
lastFailure: ConnectionAttemptError | null,
515
synchronizedRoutes: Ref.Ref<number>,
516
) {
@@ -520,7 +518,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
518
(progress.stage === "synchronizing"
519
? Ref.update(synchronizedRoutes, (count) => count + 1)
520
: Effect.void
523
- ).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))),
521
+ ).pipe(Effect.andThen(reportProgress(attempt, nextGeneration, lastFailure, progress))),
522
);
523
});
524
@@ -559,7 +557,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
557
Scope.Scope
558
>,
559
attempt: number,
562
- generation: number,
560
+ nextGeneration: number,
561
pendingRetry: Option.Option<PendingRetryTrace>,
562
) => {
563
// The span must end with the ConnectionAttemptError itself: the exporter
@@ -572,7 +570,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
570
"environment.id": target.environmentId,
571
"environment.target.kind": target._tag,
572
"connection.attempt": attempt,
575
- "connection.generation": generation,
573
+ "connection.generation": nextGeneration,
574
"connection.retry.failure_count": Option.match(pendingRetry, {
575
onNone: () => 0,
576
onSome: (retry) => retry.failureCount,
@@ -599,20 +597,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
597
598
const establishTracedConnection = Effect.fnUntraced(function* (
599
attempt: number,
602
- generation: number,
600
+ nextGeneration: number,
601
lastFailure: ConnectionAttemptError | null,
602
pendingRetry: Option.Option<PendingRetryTrace>,
603
synchronizedRoutes: Ref.Ref<number>,
604
) {
605
if (usesRelay) {
606
return yield* traceRelayEstablishment(
609
- establishConnection(attempt, generation, lastFailure, synchronizedRoutes),
607
+ establishConnection(attempt, nextGeneration, lastFailure, synchronizedRoutes),
608
attempt,
611
- generation,
609
+ nextGeneration,
610
pendingRetry,
611
);
612
}
615
- return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe(
613
+ return yield* establishConnection(
614
+ attempt,
615
+ nextGeneration,
616
+ lastFailure,
617
+ synchronizedRoutes,
618
+ ).pipe(
619
Effect.map((lease) => ({
620
attemptSpan: Option.none<Tracer.Span>(),
621
lease,
@@ -946,9 +949,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
949
}
950
});
951
952
+ // Advanced as a session goes live and kept across loop restarts, so no two sessions share one.
953
+ let generation = 0;
954
const runAttempt = Effect.fnUntraced(function* (
955
attempt: number,
951
- generation: number,
956
+ nextGeneration: number,
957
lastFailure: ConnectionAttemptError | null,
958
pendingRetry: Option.Option<PendingRetryTrace>,
959
ignoreOffline: boolean,
@@ -957,10 +962,10 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
962
yield* SubscriptionRef.set(prepared, Option.none());
963
const synchronizedRoutes = yield* Ref.make(0);
964
const establishment = yield* Effect.raceAllFirst([
960
- exitUnlessInterrupted(
965
+ Effect.exit(
966
establishTracedConnection(
967
attempt,
963
- generation,
968
+ nextGeneration,
969
lastFailure,
970
pendingRetry,
971
synchronizedRoutes,
@@ -1031,22 +1036,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1036
} satisfies AttemptOutcome;
1037
}
1038
if (Exit.isFailure(establishment.exit)) {
1034
- const isUnexpectedDefect =
1035
- !Cause.hasInterruptsOnly(establishment.exit.cause) &&
1036
- !establishment.exit.cause.reasons.some(Cause.isFailReason);
1037
- const outcome = failureFromExit(target, establishment.exit, false, false);
1038
- if (isUnexpectedDefect) {
1039
- const defect = establishment.exit.cause.reasons.find(Cause.isDieReason)?.defect;
1040
- yield* Effect.logError("Connection attempt failed with an unexpected defect.").pipe(
1041
- Effect.annotateLogs({
1042
- "environment.id": target.environmentId,
1043
- "environment.label": target.label,
1044
- "cause.reason_count": establishment.exit.cause.reasons.length,
1045
- ...safeErrorLogAttributes(defect),
1046
- }),
1047
- );
1048
- }
1049
- return outcome;
1039
+ yield* logUnexpectedFailure("attempt", establishment.exit.cause);
1040
+ return failureFromCause(target, establishment.exit.cause, false, false);
1041
}
1042
1043
const active = establishment.exit.value;
@@ -1075,7 +1066,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1066
const connectedLease: ConnectedLease = {
1067
...active.lease,
1068
activeRoute: routes.find((route) => connectionRouteId(route.target) === activeRouteId),
1078
- generation,
1069
+ generation: nextGeneration,
1070
};
1071
const refusedRouteIds = yield* Ref.updateAndGet(refusedRoutes, (known) =>
1072
mergeRefusedRoutes(
@@ -1086,6 +1077,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1077
),
1078
);
1079
const connectedAt = yield* Clock.currentTimeMillis;
1080
+ generation = nextGeneration;
1081
yield* SubscriptionRef.set(prepared, Option.some(connectedLease.prepared));
1082
yield* SubscriptionRef.set(session, Option.some(connectedLease.session));
1083
if (options?.learnRoutes !== undefined) {
@@ -1097,7 +1089,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1089
phase: "connected",
1090
stage: null,
1091
attempt,
1100
- generation,
1092
+ generation: nextGeneration,
1093
lastFailure: null,
1094
retryAt: null,
1095
refusedRouteIds,
@@ -1116,7 +1108,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1108
attemptSpan: active.attemptSpan,
1109
})),
1110
),
1119
- ).pipe(exitUnlessInterrupted);
1111
+ ).pipe(Effect.exit);
1112
const connectedForMs = (yield* Clock.currentTimeMillis) - connectedAt;
1113
if (Exit.isSuccess(connectedExit)) {
1114
return {
@@ -1126,27 +1118,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1118
resetRetry: connectedExit.value,
1119
} satisfies AttemptOutcome;
1120
}
1129
- const outcome = failureFromExit(
1121
+ yield* logUnexpectedFailure("attempt", connectedExit.cause);
1122
+ const outcome = failureFromCause(
1123
target,
1131
- connectedExit,
1124
+ connectedExit.cause,
1125
true,
1126
connectedForMs >= BACKOFF_RESET_AFTER_MS,
1127
);
1135
- if (outcome._tag === "Failure") {
1136
- yield* Effect.void.pipe(
1137
- Effect.withSpan("EnvironmentSupervisor.connectionLost", {
1138
- root: true,
1139
- attributes: {
1140
- "environment.id": target.environmentId,
1141
- "environment.label": target.label,
1142
- "environment.target.kind": target._tag,
1143
- "connection.connected_ms": connectedForMs,
1144
- "connection.failure.reason": outcome.failure.error.reason,
1145
- "connection.failure.detail": outcome.failure.error.detail,
1146
- },
1147
- }),
1148
- );
1149
- }
1128
+ yield* Effect.void.pipe(
1129
+ Effect.withSpan("EnvironmentSupervisor.connectionLost", {
1130
+ root: true,
1131
+ attributes: {
1132
+ "environment.id": target.environmentId,
1133
+ "environment.label": target.label,
1134
+ "environment.target.kind": target._tag,
1135
+ "connection.connected_ms": connectedForMs,
1136
+ "connection.failure.reason": outcome.failure.error.reason,
1137
+ "connection.failure.detail": outcome.failure.error.detail,
1138
+ },
1139
+ }),
1140
+ );
1141
return outcome;
1142
}, Effect.ensuring(clearLease));
1143
@@ -1185,7 +1176,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1176
// Approval waits count as attempts but not toward backoff, so a later outage starts low.
1177
let backoffFailures = 0;
1178
let approvalWaits = 0;
1188
- let generation = 0;
1179
let latestFailure: ConnectionAttemptError | null = null;
1180
let pendingRetry = Option.none<PendingRetryTrace>();
1181
const resetRetryLadder = () => {
@@ -1229,13 +1219,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1219
1220
const attempt = failureCount + 1;
1221
const nextGeneration = generation + 1;
1232
- const outcome: AttemptOutcome = yield* Effect.scoped(
1233
- runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline),
1222
+ const outcome: AttemptOutcome = yield* Effect.acquireUseRelease(
1223
+ Scope.make(),
1224
+ (scope) =>
1225
+ runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline).pipe(
1226
+ Scope.provide(scope),
1227
+ Effect.catchCause((cause) =>
1228
+ logUnexpectedFailure("attempt", cause).pipe(
1229
+ Effect.as(failureFromCause(target, cause, generation === nextGeneration, false)),
1230
+ ),
1231
+ ),
1232
+ ),
1233
+ (scope, exit) =>
1234
+ Scope.close(scope, exit).pipe(
1235
+ Effect.catchCause((cause) => logUnexpectedFailure("release", cause)),
1236
+ ),
1237
);
1238
replacing = false;
1239
const failedProbe = yield* Ref.getAndSet(probeUnanswered, false);
1240
if (outcome.established) {
1238
- generation = nextGeneration;
1241
// A session came up, so no earlier failure describes the connection.
1242
// Losing a live session is not a failed connection either, for example
1243
// a socket the OS killed while the app was suspended: its replacement
@@ -1327,6 +1329,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1329
}
1330
});
1331
1332
+ // Registered before the fibers below so closing the scope interrupts them first.
1333
+ // A loop still running when the queue shuts down would report that as a failure.
1334
+ yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1335
yield* connectivity.changes.pipe(
1336
Stream.runForEach((network) =>
1337
Ref.modify(intent, (current) => {
@@ -1362,7 +1367,37 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1367
Effect.forever,
1368
Effect.forkScoped,
1369
);
1365
- yield* run().pipe(Effect.forkScoped);
1370
+ // A restarted loop has forgotten its retry ladder, so it waits out the
1371
+ // ladder's ceiling unless a signal cuts the wait short.
1372
+ yield* run().pipe(
1373
+ Effect.catchCause((cause) =>
1374
+ Effect.gen(function* () {
1375
+ yield* logUnexpectedFailure("loop", cause);
1376
+ const currentIntent = yield* Ref.get(intent);
1377
+ const lastFailure = unexpectedFailure(target).error;
1378
+ yield* setState(
1379
+ !currentIntent.desired
1380
+ ? availableState(currentIntent, generation)
1381
+ : currentIntent.network === "offline"
1382
+ ? offlineState(currentIntent, generation, 1, lastFailure)
1383
+ : {
1384
+ desired: true,
1385
+ network: currentIntent.network,
1386
+ phase: "backoff",
1387
+ stage: null,
1388
+ attempt: 1,
1389
+ generation,
1390
+ lastFailure,
1391
+ retryAt: (yield* Clock.currentTimeMillis) + RETRY_MAX_DELAY_MS,
1392
+ refusedRouteIds: NO_REFUSED_ROUTES,
1393
+ },
1394
+ );
1395
+ yield* waitForRetrySignal(RETRY_MAX_DELAY_MS);
1396
+ }),
1397
+ ),
1398
+ Effect.forever,
1399
+ Effect.forkScoped,
1400
+ );
1401
1402
const connect = Ref.update(intent, (current) => ({
1403
...current,
@@ -1385,8 +1420,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1420
Effect.withSpan("EnvironmentSupervisor.retryNow"),
1421
);
1422
1388
- yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1389
-
1423
return EnvironmentSupervisor.of({
1424
target,
1425
state,
scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts
new
+85
@@ -0,0 +1,85 @@
1
+import { expect } from "e2e";
2
+import * as Effect from "effect/Effect";
3
+
4
+import { runPromise, test, WEB } from "./t3.ts";
5
+
6
+interface Fault {
7
+ readonly failConfigStreamWhileInterrupted: () => boolean;
8
+}
9
+
10
+test(
11
+ "The app reconnects after its config stream fails while interrupted",
12
+ WEB,
13
+ async ({ t3, app, screen, browser }) => {
14
+ const project = await t3.project("demo");
15
+ const thread = await t3.thread({
16
+ project,
17
+ title: "Earlier work",
18
+ messages: ["Print the JSON"],
19
+ model: "command-output",
20
+ });
21
+ // Injected fault: the page hands the app one server frame that ends the config
22
+ // subscription with a typed failure and an interruption in the same cause.
23
+ await browser.addInitScript(() => {
24
+ let deliver: (() => void) | null = null;
25
+ const NativeWebSocket = window.WebSocket;
26
+ window.WebSocket = class extends NativeWebSocket {
27
+ override send(data: Parameters<WebSocket["send"]>[0]) {
28
+ if (typeof data === "string" && data.includes('"tag":"subscribeServerConfig"')) {
29
+ const requestId = (JSON.parse(data) as { id: number | string }).id;
30
+ deliver = () =>
31
+ this.dispatchEvent(
32
+ new MessageEvent("message", {
33
+ data: JSON.stringify({
34
+ _tag: "Exit",
35
+ requestId,
36
+ exit: {
37
+ _tag: "Failure",
38
+ cause: [
39
+ {
40
+ _tag: "Fail",
41
+ error: {
42
+ _tag: "KeybindingsConfigParseError",
43
+ configPath: "/demo/keybindings.json",
44
+ detail: "Injected failure.",
45
+ },
46
+ },
47
+ { _tag: "Interrupt", fiberId: 1 },
48
+ ],
49
+ },
50
+ }),
51
+ }),
52
+ );
53
+ }
54
+ super.send(data);
55
+ }
56
+ };
57
+ (window as unknown as { __fault: Fault }).__fault = {
58
+ failConfigStreamWhileInterrupted: () => {
59
+ deliver?.();
60
+ return deliver !== null;
61
+ },
62
+ };
63
+ });
64
+ await browser.emulateMedia({ colorScheme: "dark", reducedMotion: "reduce" });
65
+ await t3.open(thread);
66
+
67
+ const reply = screen.getByText("The command printed one JSON line.");
68
+ await expect(reply.first()).toBeVisible();
69
+ await app.screenshot("1-connected");
70
+
71
+ const delivered = await browser.evaluate(() =>
72
+ (window as unknown as { __fault: Fault }).__fault.failConfigStreamWhileInterrupted(),
73
+ );
74
+ expect(delivered).toBe(true);
75
+ // A live loop reconnects well within each wait; a dead one never does.
76
+ await runPromise(Effect.sleep("5 seconds"));
77
+ await app.screenshot("2-five-seconds-after-the-failure");
78
+
79
+ await screen.getByRole("textbox").fill("Are you still there?");
80
+ await screen.getByRole("textbox").press("Enter");
81
+ await runPromise(Effect.sleep("3 seconds"));
82
+ await app.screenshot("3-after-pressing-enter");
83
+ await expect(reply.nth(1)).toBeVisible();
84
+ },
85
+);