@devries-cameron20 / t3code Mirror · read-only / commits / 38bac7091a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

fix(client-runtime): an unexpected failure no longer stops the reconnect loop for good (#39420)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Viktor Leukocyte committed Nov 19, 2026 at 23:54 UTC 38bac7091a71eb6e01b63b70f6b1bae7441ed5bc
3 files changed +531 -106
packages/client-runtime/src/connection/supervisor.test.ts
+323 -16
@@ -6,14 +6,19 @@ import {
6 } from "@t3tools/contracts";
7 import { describe, expect, it } from "@effect/vitest";
8 import { RelayClientTracer } from "@t3tools/shared/relayTracing";
9 +import * as Cause from "effect/Cause";
10 import * as Deferred from "effect/Deferred";
11 import * as Effect from "effect/Effect";
12 +import * as Exit from "effect/Exit";
13 import * as Fiber from "effect/Fiber";
14 import * as Layer from "effect/Layer";
15 +import * as Logger from "effect/Logger";
16 import * as Option from "effect/Option";
17 import * as Queue from "effect/Queue";
18 import * as Random from "effect/Random";
19 import * as Ref from "effect/Ref";
20 +import * as References from "effect/References";
21 +import * as Scope from "effect/Scope";
22 import * as Stream from "effect/Stream";
23 import * as SubscriptionRef from "effect/SubscriptionRef";
24 import * as Clock from "effect/Clock";
@@ -167,6 +172,18 @@ function awaitState(
172 );
173 }
174
175 +function recordLogs(logs: Array<unknown>, annotations?: Array<unknown>) {
176 + return Logger.layer(
177 + [
178 + Logger.make(({ message, fiber }) => {
179 + logs.push(message);
180 + annotations?.push(fiber.getRef(References.CurrentLogAnnotations));
181 + }),
182 + ],
183 + { mergeWithExisting: false },
184 + );
185 +}
186 +
187 const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: {
188 readonly networkStatus?: NetworkStatus;
189 readonly prepare?: (
@@ -174,6 +191,7 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
191 target: ConnectionTarget,
192 ) => Effect.Effect<PreparedConnection, ConnectionAttemptError>;
193 readonly ready?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
194 + readonly release?: (attempt: number) => Effect.Effect<void>;
195 readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
196 readonly configEvents?: Stream.Stream<ServerConfigStreamEvent> | undefined;
197 readonly initialConfig?: (attempt: number) => Effect.Effect<ServerConfig, ConnectionAttemptError>;
@@ -245,7 +263,10 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
263 inboundFrames: options?.inboundFrames ?? Effect.succeed(0),
264 closed: Deferred.await(closed),
265 } satisfies RpcSession.RpcSession),
248 - () => Ref.update(releaseCount, (count) => count + 1),
266 + () =>
267 + Ref.update(releaseCount, (count) => count + 1).pipe(
268 + Effect.andThen(options?.release?.(attempt) ?? Effect.void),
269 + ),
270 );
271
272 yield* reportProgress({ stage: "synchronizing", prepared });
@@ -308,12 +329,12 @@ const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?:
329 reason,
330 })),
331 closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* (
311 - error = transient("Session closed."),
332 + exit: Exit.Exit<never, ConnectionTransientError> = Exit.fail(transient("Session closed.")),
333 ) {
334 const sessions = yield* Ref.get(closedSessions);
335 const latest = sessions.at(-1);
336 if (latest) {
316 - yield* Deferred.fail(latest, error);
337 + yield* Deferred.done(latest, exit);
338 }
339 }),
340 };
@@ -581,9 +602,11 @@ describe("EnvironmentSupervisor", () => {
602 ? Effect.die(new Error("Native transport defect."))
603 : Effect.succeed(PREPARED_CONNECTION),
604 });
605 + const logs: unknown[] = [];
606 + const annotations: unknown[] = [];
607 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
608 initiallyDesired: true,
586 - }).pipe(Effect.provide(harness.dependencies));
609 + }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))));
610
611 const failed = yield* awaitState(
612 supervisor.state,
@@ -596,6 +619,10 @@ describe("EnvironmentSupervisor", () => {
619 message: "Test environment connection failed unexpectedly.",
620 },
621 });
622 + expect(logs).toEqual([["Connection attempt failed with an unexpected defect."]]);
623 + expect(annotations).toMatchObject([
624 + { "environment.id": RELAY_TARGET.environmentId, errorName: "Error" },
625 + ]);
626
627 yield* TestClock.adjust("3 seconds");
628 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
@@ -603,42 +630,322 @@ describe("EnvironmentSupervisor", () => {
630 }).pipe(Effect.provide(TestClock.layer())),
631 );
632
606 - it.effect("explicit retry interrupts the current backoff", () =>
633 + it.effect.each([
634 + {
635 + label: "the attempt's network-status read dies",
636 + firstPrepare: Effect.never,
637 + statusDefect: true,
638 + log: "Connection attempt failed with an unexpected defect.",
639 + },
640 + {
641 + label: "an attempt interrupts itself",
642 + firstPrepare: Effect.interrupt,
643 + statusDefect: false,
644 + log: "Connection attempt failed with an unexpected interruption.",
645 + },
646 + ])("retries after $label", ({ firstPrepare, statusDefect, log }) =>
647 + Effect.gen(function* () {
648 + const harness = yield* makeHarness({
649 + prepare: (attempt) => (attempt === 1 ? firstPrepare : Effect.succeed(PREPARED_CONNECTION)),
650 + });
651 + const statusReads = yield* Ref.make(0);
652 + const logs: unknown[] = [];
653 + const annotations: Array<Record<string, unknown>> = [];
654 + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
655 + initiallyDesired: true,
656 + }).pipe(
657 + Effect.provideService(
658 + Connectivity.Connectivity,
659 + Connectivity.Connectivity.of({
660 + // The supervisor reads once at startup; the second read belongs to the first attempt.
661 + status: Ref.updateAndGet(statusReads, (count) => count + 1).pipe(
662 + Effect.flatMap((count) =>
663 + statusDefect && count === 2
664 + ? Effect.die(new Error("Platform connectivity defect."))
665 + : Effect.succeed("online" as const),
666 + ),
667 + ),
668 + changes: Stream.never,
669 + }),
670 + ),
671 + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))),
672 + );
673 +
674 + const failed = yield* awaitState(
675 + supervisor.state,
676 + (state) => state.phase === "backoff" && state.attempt === 1,
677 + );
678 + expect(failed).toMatchObject({
679 + generation: 0,
680 + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
681 + });
682 + expect(logs).toEqual([[log]]);
683 + expect(Object.hasOwn(annotations[0]!, "errorType")).toBe(statusDefect);
684 +
685 + yield* TestClock.adjust("999 millis");
686 + expect(yield* Ref.get(harness.prepareCount)).toBe(1);
687 + yield* TestClock.adjust("2 seconds");
688 + yield* awaitState(supervisor.state, (state) => state.phase === "connected");
689 + expect(yield* Ref.get(harness.prepareCount)).toBe(2);
690 + }).pipe(Effect.provide(TestClock.layer())),
691 + );
692 +
693 + it.effect.each([
694 + {
695 + label: "is interrupted",
696 + closed: Exit.interrupt(),
697 + releaseDefect: false,
698 + log: "Connection attempt failed with an unexpected interruption.",
699 + },
700 + {
701 + label: "fails while interrupted",
702 + closed: Exit.failCause(
703 + Cause.combine(Cause.fail(transient("Session closed.")), Cause.interrupt()),
704 + ),
705 + releaseDefect: false,
706 + log: null,
707 + },
708 + {
709 + label: "closes while its release defects",
710 + closed: Exit.fail(transient("Session closed.")),
711 + releaseDefect: true,
712 + log: "Connection release failed with an unexpected defect.",
713 + },
714 + ])("reconnects soon after a stable session $label", ({ closed, releaseDefect, log }) =>
715 Effect.gen(function* () {
716 const harness = yield* makeHarness({
717 prepare: (attempt) =>
718 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
719 + release: (session) =>
720 + releaseDefect && session === 1
721 + ? Effect.die(new Error("Session release defect."))
722 + : Effect.void,
723 });
724 + const logs: unknown[] = [];
725 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
726 initiallyDesired: true,
614 - }).pipe(Effect.provide(harness.dependencies));
727 + }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))));
728
729 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
730 + yield* TestClock.adjust("2 seconds");
731 + yield* awaitState(
732 + supervisor.state,
733 + (state) => state.phase === "connected" && state.generation === 1,
734 + );
735 + yield* TestClock.adjust("1 minute");
736 + yield* harness.closeLatestSession(closed);
737 + const lost = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
738 + expect(logs).toEqual(log === null ? [] : [[log]]);
739 + expect(lost).toMatchObject({ phase: "backoff", generation: 1 });
740 + expect(lost.retryAt).toBeLessThanOrEqual((yield* Clock.currentTimeMillis) + 250);
741 +
742 + yield* TestClock.adjust("250 millis");
743 + const replaced = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
744 + expect(replaced.generation).toBe(2);
745 + }).pipe(Effect.provide(TestClock.layer())),
746 + );
747 +
748 + it.effect(
749 + "does not reuse a live session's generation when its attempt defects after going live",
750 + () =>
751 + Effect.gen(function* () {
752 + const harness = yield* makeHarness({
753 + prepare: (attempt) =>
754 + attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
755 + });
756 + let logCalls = 0;
757 + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
758 + initiallyDesired: true,
759 + }).pipe(
760 + Effect.provide(
761 + Layer.merge(
762 + harness.dependencies,
763 + Logger.layer(
764 + [
765 + // Logging the lost session's interruption throws, so the attempt itself defects.
766 + Logger.make(() => {
767 + logCalls += 1;
768 + if (logCalls === 1) throw new Error("Logger defect.");
769 + }),
770 + ],
771 + { mergeWithExisting: false },
772 + ),
773 + ),
774 + ),
775 + );
776 +
777 + yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
778 + yield* TestClock.adjust("2 seconds");
779 + yield* awaitState(
780 + supervisor.state,
781 + (state) => state.phase === "connected" && state.generation === 1,
782 + );
783 + yield* harness.closeLatestSession(Exit.interrupt());
784 + const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
785 + expect(backoff).toMatchObject({ generation: 1, lastFailure: null });
786 +
787 + yield* TestClock.adjust("4 seconds");
788 + const replaced = yield* awaitState(
789 + supervisor.state,
790 + (state) => state.phase === "connected",
791 + );
792 + expect(replaced.generation).toBe(2);
793 + }).pipe(Effect.provide(TestClock.layer())),
794 + );
795 +
796 + it.effect("restarts the loop after a defect outside an attempt", () =>
797 + Effect.gen(function* () {
798 + const harness = yield* makeHarness();
799 + const logs: unknown[] = [];
800 + let randomReads = 0;
801 + const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
802 + initiallyDesired: true,
803 + }).pipe(
804 + Effect.provideService(Random.Random, {
805 + // Both session losses below die here, so the second reaches the restart handler again.
806 + nextDoubleUnsafe: () => {
807 + randomReads += 1;
808 + if (randomReads <= 2) throw new Error("Random defect.");
809 + return 1 - Number.EPSILON;
810 + },
811 + nextIntUnsafe: () => 0,
812 + }),
813 + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
814 + );
815 +
816 + yield* awaitState(
817 + supervisor.state,
818 + (state) => state.phase === "connected" && state.generation === 1,
819 + );
820 + yield* harness.closeLatestSession();
821 + const restarted = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
822 + expect(logs).toEqual([["Connection loop failed with an unexpected defect."]]);
823 + expect(restarted).toMatchObject({
824 + attempt: 1,
825 + generation: 1,
826 + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
827 + retryAt: (yield* Clock.currentTimeMillis) + 300_000,
828 + });
829 +
830 + yield* TestClock.adjust("299999 millis");
831 + expect(yield* Ref.get(harness.sessionCount)).toBe(1);
832 + yield* TestClock.adjust("1 millis");
833 + const reconnected = yield* awaitState(
834 + supervisor.state,
835 + (state) => state.phase === "connected",
836 + );
837 + expect(reconnected.generation).toBe(2);
838 +
839 + yield* harness.setNetworkStatus("offline");
840 + yield* harness.closeLatestSession();
841 + const offline = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
842 + expect(offline).toMatchObject({
843 + phase: "offline",
844 + attempt: 1,
845 + lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
846 + });
847 +
848 yield* supervisor.retryNow;
618 - yield* awaitState(supervisor.state, (state) => state.phase === "connected");
849 + const retried = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
850 + expect(retried.generation).toBe(3);
851 + }).pipe(Effect.provide(TestClock.layer())),
852 + );
853
620 - expect(yield* Ref.get(harness.prepareCount)).toBe(2);
621 - }),
854 + it.effect.each(["connecting", "connected"] as const)(
855 + "stops attempting when its scope closes while %s",
856 + (phase) =>
857 + Effect.gen(function* () {
858 + const preparing = yield* Deferred.make<void>();
859 + const harness = yield* makeHarness({
860 + prepare: () =>
861 + Deferred.succeed(preparing, undefined).pipe(
862 + Effect.andThen(
863 + phase === "connected" ? Effect.succeed(PREPARED_CONNECTION) : Effect.never,
864 + ),
865 + ),
866 + });
867 + const logs: unknown[] = [];
868 + const scope = yield* Scope.make();
869 + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
870 + initiallyDesired: true,
871 + }).pipe(
872 + Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
873 + Scope.provide(scope),
874 + );
875 +
876 + yield* Deferred.await(preparing);
877 + yield* awaitState(supervisor.state, (state) => state.phase === phase);
878 + // Yielding between finalizers lets the loop run if the queue shuts down before its interrupt.
879 + yield* Scope.close(scope, Exit.void).pipe(
880 + Effect.provideService(References.MaxOpsBeforeYield, 1),
881 + );
882 + yield* TestClock.adjust("10 minutes");
883 + expect(logs).toEqual([]);
884 + expect(yield* Ref.get(harness.prepareCount)).toBe(1);
885 + }).pipe(Effect.provide(TestClock.layer())),
886 );
887
624 - it.effect("keeps blocked failures idle until an external signal requests another attempt", () =>
888 + it.effect("explicit retry interrupts the current backoff", () =>
889 Effect.gen(function* () {
890 const harness = yield* makeHarness({
891 prepare: (attempt) =>
628 - attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION),
892 + attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
893 });
894 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
895 initiallyDesired: true,
896 }).pipe(Effect.provide(harness.dependencies));
897
634 - yield* awaitState(supervisor.state, (state) => state.phase === "blocked");
635 - yield* TestClock.adjust("1 hour");
636 - expect(yield* Ref.get(harness.prepareCount)).toBe(1);
637 -
898 + yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
899 yield* supervisor.retryNow;
900 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
901 +
902 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
641 - }).pipe(Effect.provide(TestClock.layer())),
903 + }),
904 + );
905 +
906 + it.effect.each([
907 + { label: "fails", cause: Cause.fail(blocked()) },
908 + {
909 + label: "fails while interrupted",
910 + cause: Cause.combine(Cause.fail(blocked()), Cause.interrupt()),
911 + },
912 + ])(
913 + "keeps a blocked host that $label idle until an external signal requests another attempt",
914 + ({ cause }) =>
915 + Effect.gen(function* () {
916 + const harness = yield* makeHarness();
917 + const driver = yield* ConnectionDriver.ConnectionDriver.pipe(
918 + Effect.provide(harness.dependencies),
919 + );
920 + const connects = yield* Ref.make(0);
921 + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
922 + initiallyDesired: true,
923 + }).pipe(
924 + // The driver keeps only the typed error of a failed route, so the cause is mixed here.
925 + Effect.provideService(ConnectionDriver.ConnectionDriver, {
926 + ...driver,
927 + connect: (entry, reportProgress) =>
928 + Ref.updateAndGet(connects, (count) => count + 1).pipe(
929 + Effect.flatMap((count) =>
930 + count === 1 ? Effect.failCause(cause) : driver.connect(entry, reportProgress),
931 + ),
932 + ),
933 + }),
934 + Effect.provide(harness.dependencies),
935 + );
936 +
937 + const failed = yield* awaitState(supervisor.state, (state) => state.lastFailure !== null);
938 + expect(failed).toMatchObject({
939 + phase: "blocked",
940 + lastFailure: { _tag: "ConnectionBlockedError", detail: "Authentication required." },
941 + });
942 + yield* TestClock.adjust("1 hour");
943 + expect(yield* Ref.get(connects)).toBe(1);
944 +
945 + yield* supervisor.retryNow;
946 + yield* awaitState(supervisor.state, (state) => state.phase === "connected");
947 + expect(yield* Ref.get(connects)).toBe(2);
948 + }).pipe(Effect.provide(TestClock.layer())),
949 );
950
951 it.effect("retries a blocked connection when platform credentials change", () =>
packages/client-runtime/src/connection/supervisor.ts
+123 -90
@@ -149,16 +149,6 @@ type EstablishmentEvent =
149 | { readonly _tag: "Interrupted"; readonly resetRetry: boolean }
150 | { readonly _tag: "TimedOut" };
151
152 -function exitUnlessInterrupted<A, E, R>(
153 - effect: Effect.Effect<A, E, R>,
154 -): Effect.Effect<Exit.Exit<A, E>, never, R> {
155 - return Effect.matchCauseEffect(effect, {
156 - onFailure: (cause) =>
157 - Cause.hasInterrupts(cause) ? Effect.interrupt : Effect.succeed(Exit.failCause(cause)),
158 - onSuccess: (value) => Effect.succeed(Exit.succeed(value)),
159 - });
160 -}
161 -
152 export interface EnvironmentSupervisorOptions {
153 readonly initiallyDesired?: boolean;
154 readonly learnRoutes?: (input: {
@@ -247,37 +237,27 @@ function connectingState(
237 };
238 }
239
250 -function failureFromExit<A>(
240 +function failureFromCause(
241 target: ConnectionTarget,
252 - exit: Exit.Exit<A, TracedAttemptFailure>,
242 + cause: Cause.Cause<TracedAttemptFailure>,
243 established: boolean,
244 stable: boolean,
255 -): AttemptOutcome {
256 - if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) {
257 - return { _tag: "Interrupted", established, stable, resetRetry: false };
258 - }
259 - const typedFailure = exit.cause.reasons.find(Cause.isFailReason);
260 - if (typedFailure) {
261 - return {
262 - _tag: "Failure",
263 - established,
264 - stable,
265 - failure: typedFailure.error,
266 - };
267 - }
245 +) {
246 return {
247 _tag: "Failure",
248 established,
249 stable,
272 - failure: {
273 - error: new ConnectionTransientError({
274 - reason: "transport",
275 - ...translatableDetail(
276 - msg("{host} connection failed unexpectedly.", { host: target.label }),
277 - ),
278 - }),
279 - attemptSpan: Option.none(),
280 - },
250 + failure: cause.reasons.find(Cause.isFailReason)?.error ?? unexpectedFailure(target),
251 + } satisfies AttemptOutcome;
252 +}
253 +
254 +function unexpectedFailure(target: ConnectionTarget): TracedAttemptFailure {
255 + return {
256 + error: new ConnectionTransientError({
257 + reason: "transport",
258 + ...translatableDetail(msg("{host} connection failed unexpectedly.", { host: target.label })),
259 + }),
260 + attemptSpan: Option.none(),
261 };
262 }
263
@@ -496,9 +476,27 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
476 }),
477 );
478
479 + const logUnexpectedFailure = (
480 + failed: "attempt" | "release" | "loop",
481 + cause: Cause.Cause<TracedAttemptFailure>,
482 + ) => {
483 + const died = cause.reasons.find(Cause.isDieReason);
484 + if (!died && cause.reasons.some(Cause.isFailReason)) return Effect.void;
485 + return Effect.logError(
486 + `Connection ${failed} failed with an unexpected ${died ? "defect" : "interruption"}.`,
487 + ).pipe(
488 + Effect.annotateLogs({
489 + "environment.id": target.environmentId,
490 + "environment.label": target.label,
491 + "cause.reason_count": cause.reasons.length,
492 + ...(died ? safeErrorLogAttributes(died.defect) : {}),
493 + }),
494 + );
495 + };
496 +
497 const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* (
498 attempt: number,
501 - generation: number,
499 + nextGeneration: number,
500 lastFailure: ConnectionAttemptError | null,
501 progress: ConnectionDriver.ConnectionDriverProgress,
502 ) {
@@ -506,13 +504,13 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
504 yield* SubscriptionRef.set(prepared, Option.some(progress.prepared));
505 }
506 yield* setState(
509 - connectingState(yield* Ref.get(intent), generation, attempt, lastFailure, progress.stage),
507 + connectingState(yield* Ref.get(intent), nextGeneration, attempt, lastFailure, progress.stage),
508 );
509 });
510
511 const establishConnection = Effect.fnUntraced(function* (
512 attempt: number,
515 - generation: number,
513 + nextGeneration: number,
514 lastFailure: ConnectionAttemptError | null,
515 synchronizedRoutes: Ref.Ref<number>,
516 ) {
@@ -520,7 +518,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
518 (progress.stage === "synchronizing"
519 ? Ref.update(synchronizedRoutes, (count) => count + 1)
520 : Effect.void
523 - ).pipe(Effect.andThen(reportProgress(attempt, generation, lastFailure, progress))),
521 + ).pipe(Effect.andThen(reportProgress(attempt, nextGeneration, lastFailure, progress))),
522 );
523 });
524
@@ -559,7 +557,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
557 Scope.Scope
558 >,
559 attempt: number,
562 - generation: number,
560 + nextGeneration: number,
561 pendingRetry: Option.Option<PendingRetryTrace>,
562 ) => {
563 // The span must end with the ConnectionAttemptError itself: the exporter
@@ -572,7 +570,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
570 "environment.id": target.environmentId,
571 "environment.target.kind": target._tag,
572 "connection.attempt": attempt,
575 - "connection.generation": generation,
573 + "connection.generation": nextGeneration,
574 "connection.retry.failure_count": Option.match(pendingRetry, {
575 onNone: () => 0,
576 onSome: (retry) => retry.failureCount,
@@ -599,20 +597,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
597
598 const establishTracedConnection = Effect.fnUntraced(function* (
599 attempt: number,
602 - generation: number,
600 + nextGeneration: number,
601 lastFailure: ConnectionAttemptError | null,
602 pendingRetry: Option.Option<PendingRetryTrace>,
603 synchronizedRoutes: Ref.Ref<number>,
604 ) {
605 if (usesRelay) {
606 return yield* traceRelayEstablishment(
609 - establishConnection(attempt, generation, lastFailure, synchronizedRoutes),
607 + establishConnection(attempt, nextGeneration, lastFailure, synchronizedRoutes),
608 attempt,
611 - generation,
609 + nextGeneration,
610 pendingRetry,
611 );
612 }
615 - return yield* establishConnection(attempt, generation, lastFailure, synchronizedRoutes).pipe(
613 + return yield* establishConnection(
614 + attempt,
615 + nextGeneration,
616 + lastFailure,
617 + synchronizedRoutes,
618 + ).pipe(
619 Effect.map((lease) => ({
620 attemptSpan: Option.none<Tracer.Span>(),
621 lease,
@@ -946,9 +949,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
949 }
950 });
951
952 + // Advanced as a session goes live and kept across loop restarts, so no two sessions share one.
953 + let generation = 0;
954 const runAttempt = Effect.fnUntraced(function* (
955 attempt: number,
951 - generation: number,
956 + nextGeneration: number,
957 lastFailure: ConnectionAttemptError | null,
958 pendingRetry: Option.Option<PendingRetryTrace>,
959 ignoreOffline: boolean,
@@ -957,10 +962,10 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
962 yield* SubscriptionRef.set(prepared, Option.none());
963 const synchronizedRoutes = yield* Ref.make(0);
964 const establishment = yield* Effect.raceAllFirst([
960 - exitUnlessInterrupted(
965 + Effect.exit(
966 establishTracedConnection(
967 attempt,
963 - generation,
968 + nextGeneration,
969 lastFailure,
970 pendingRetry,
971 synchronizedRoutes,
@@ -1031,22 +1036,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1036 } satisfies AttemptOutcome;
1037 }
1038 if (Exit.isFailure(establishment.exit)) {
1034 - const isUnexpectedDefect =
1035 - !Cause.hasInterruptsOnly(establishment.exit.cause) &&
1036 - !establishment.exit.cause.reasons.some(Cause.isFailReason);
1037 - const outcome = failureFromExit(target, establishment.exit, false, false);
1038 - if (isUnexpectedDefect) {
1039 - const defect = establishment.exit.cause.reasons.find(Cause.isDieReason)?.defect;
1040 - yield* Effect.logError("Connection attempt failed with an unexpected defect.").pipe(
1041 - Effect.annotateLogs({
1042 - "environment.id": target.environmentId,
1043 - "environment.label": target.label,
1044 - "cause.reason_count": establishment.exit.cause.reasons.length,
1045 - ...safeErrorLogAttributes(defect),
1046 - }),
1047 - );
1048 - }
1049 - return outcome;
1039 + yield* logUnexpectedFailure("attempt", establishment.exit.cause);
1040 + return failureFromCause(target, establishment.exit.cause, false, false);
1041 }
1042
1043 const active = establishment.exit.value;
@@ -1075,7 +1066,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1066 const connectedLease: ConnectedLease = {
1067 ...active.lease,
1068 activeRoute: routes.find((route) => connectionRouteId(route.target) === activeRouteId),
1078 - generation,
1069 + generation: nextGeneration,
1070 };
1071 const refusedRouteIds = yield* Ref.updateAndGet(refusedRoutes, (known) =>
1072 mergeRefusedRoutes(
@@ -1086,6 +1077,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1077 ),
1078 );
1079 const connectedAt = yield* Clock.currentTimeMillis;
1080 + generation = nextGeneration;
1081 yield* SubscriptionRef.set(prepared, Option.some(connectedLease.prepared));
1082 yield* SubscriptionRef.set(session, Option.some(connectedLease.session));
1083 if (options?.learnRoutes !== undefined) {
@@ -1097,7 +1089,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1089 phase: "connected",
1090 stage: null,
1091 attempt,
1100 - generation,
1092 + generation: nextGeneration,
1093 lastFailure: null,
1094 retryAt: null,
1095 refusedRouteIds,
@@ -1116,7 +1108,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1108 attemptSpan: active.attemptSpan,
1109 })),
1110 ),
1119 - ).pipe(exitUnlessInterrupted);
1111 + ).pipe(Effect.exit);
1112 const connectedForMs = (yield* Clock.currentTimeMillis) - connectedAt;
1113 if (Exit.isSuccess(connectedExit)) {
1114 return {
@@ -1126,27 +1118,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1118 resetRetry: connectedExit.value,
1119 } satisfies AttemptOutcome;
1120 }
1129 - const outcome = failureFromExit(
1121 + yield* logUnexpectedFailure("attempt", connectedExit.cause);
1122 + const outcome = failureFromCause(
1123 target,
1131 - connectedExit,
1124 + connectedExit.cause,
1125 true,
1126 connectedForMs >= BACKOFF_RESET_AFTER_MS,
1127 );
1135 - if (outcome._tag === "Failure") {
1136 - yield* Effect.void.pipe(
1137 - Effect.withSpan("EnvironmentSupervisor.connectionLost", {
1138 - root: true,
1139 - attributes: {
1140 - "environment.id": target.environmentId,
1141 - "environment.label": target.label,
1142 - "environment.target.kind": target._tag,
1143 - "connection.connected_ms": connectedForMs,
1144 - "connection.failure.reason": outcome.failure.error.reason,
1145 - "connection.failure.detail": outcome.failure.error.detail,
1146 - },
1147 - }),
1148 - );
1149 - }
1128 + yield* Effect.void.pipe(
1129 + Effect.withSpan("EnvironmentSupervisor.connectionLost", {
1130 + root: true,
1131 + attributes: {
1132 + "environment.id": target.environmentId,
1133 + "environment.label": target.label,
1134 + "environment.target.kind": target._tag,
1135 + "connection.connected_ms": connectedForMs,
1136 + "connection.failure.reason": outcome.failure.error.reason,
1137 + "connection.failure.detail": outcome.failure.error.detail,
1138 + },
1139 + }),
1140 + );
1141 return outcome;
1142 }, Effect.ensuring(clearLease));
1143
@@ -1185,7 +1176,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1176 // Approval waits count as attempts but not toward backoff, so a later outage starts low.
1177 let backoffFailures = 0;
1178 let approvalWaits = 0;
1188 - let generation = 0;
1179 let latestFailure: ConnectionAttemptError | null = null;
1180 let pendingRetry = Option.none<PendingRetryTrace>();
1181 const resetRetryLadder = () => {
@@ -1229,13 +1219,25 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1219
1220 const attempt = failureCount + 1;
1221 const nextGeneration = generation + 1;
1232 - const outcome: AttemptOutcome = yield* Effect.scoped(
1233 - runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline),
1222 + const outcome: AttemptOutcome = yield* Effect.acquireUseRelease(
1223 + Scope.make(),
1224 + (scope) =>
1225 + runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline).pipe(
1226 + Scope.provide(scope),
1227 + Effect.catchCause((cause) =>
1228 + logUnexpectedFailure("attempt", cause).pipe(
1229 + Effect.as(failureFromCause(target, cause, generation === nextGeneration, false)),
1230 + ),
1231 + ),
1232 + ),
1233 + (scope, exit) =>
1234 + Scope.close(scope, exit).pipe(
1235 + Effect.catchCause((cause) => logUnexpectedFailure("release", cause)),
1236 + ),
1237 );
1238 replacing = false;
1239 const failedProbe = yield* Ref.getAndSet(probeUnanswered, false);
1240 if (outcome.established) {
1238 - generation = nextGeneration;
1241 // A session came up, so no earlier failure describes the connection.
1242 // Losing a live session is not a failed connection either, for example
1243 // a socket the OS killed while the app was suspended: its replacement
@@ -1327,6 +1329,9 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1329 }
1330 });
1331
1332 + // Registered before the fibers below so closing the scope interrupts them first.
1333 + // A loop still running when the queue shuts down would report that as a failure.
1334 + yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1335 yield* connectivity.changes.pipe(
1336 Stream.runForEach((network) =>
1337 Ref.modify(intent, (current) => {
@@ -1362,7 +1367,37 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1367 Effect.forever,
1368 Effect.forkScoped,
1369 );
1365 - yield* run().pipe(Effect.forkScoped);
1370 + // A restarted loop has forgotten its retry ladder, so it waits out the
1371 + // ladder's ceiling unless a signal cuts the wait short.
1372 + yield* run().pipe(
1373 + Effect.catchCause((cause) =>
1374 + Effect.gen(function* () {
1375 + yield* logUnexpectedFailure("loop", cause);
1376 + const currentIntent = yield* Ref.get(intent);
1377 + const lastFailure = unexpectedFailure(target).error;
1378 + yield* setState(
1379 + !currentIntent.desired
1380 + ? availableState(currentIntent, generation)
1381 + : currentIntent.network === "offline"
1382 + ? offlineState(currentIntent, generation, 1, lastFailure)
1383 + : {
1384 + desired: true,
1385 + network: currentIntent.network,
1386 + phase: "backoff",
1387 + stage: null,
1388 + attempt: 1,
1389 + generation,
1390 + lastFailure,
1391 + retryAt: (yield* Clock.currentTimeMillis) + RETRY_MAX_DELAY_MS,
1392 + refusedRouteIds: NO_REFUSED_ROUTES,
1393 + },
1394 + );
1395 + yield* waitForRetrySignal(RETRY_MAX_DELAY_MS);
1396 + }),
1397 + ),
1398 + Effect.forever,
1399 + Effect.forkScoped,
1400 + );
1401
1402 const connect = Ref.update(intent, (current) => ({
1403 ...current,
@@ -1385,8 +1420,6 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1420 Effect.withSpan("EnvironmentSupervisor.retryNow"),
1421 );
1422
1388 - yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1389 -
1423 return EnvironmentSupervisor.of({
1424 target,
1425 state,
scripts/scenarios/reconnect-survives-config-stream-failing-while-interrupted.e2e.ts new
+85
@@ -0,0 +1,85 @@
1 +import { expect } from "e2e";
2 +import * as Effect from "effect/Effect";
3 +
4 +import { runPromise, test, WEB } from "./t3.ts";
5 +
6 +interface Fault {
7 + readonly failConfigStreamWhileInterrupted: () => boolean;
8 +}
9 +
10 +test(
11 + "The app reconnects after its config stream fails while interrupted",
12 + WEB,
13 + async ({ t3, app, screen, browser }) => {
14 + const project = await t3.project("demo");
15 + const thread = await t3.thread({
16 + project,
17 + title: "Earlier work",
18 + messages: ["Print the JSON"],
19 + model: "command-output",
20 + });
21 + // Injected fault: the page hands the app one server frame that ends the config
22 + // subscription with a typed failure and an interruption in the same cause.
23 + await browser.addInitScript(() => {
24 + let deliver: (() => void) | null = null;
25 + const NativeWebSocket = window.WebSocket;
26 + window.WebSocket = class extends NativeWebSocket {
27 + override send(data: Parameters<WebSocket["send"]>[0]) {
28 + if (typeof data === "string" && data.includes('"tag":"subscribeServerConfig"')) {
29 + const requestId = (JSON.parse(data) as { id: number | string }).id;
30 + deliver = () =>
31 + this.dispatchEvent(
32 + new MessageEvent("message", {
33 + data: JSON.stringify({
34 + _tag: "Exit",
35 + requestId,
36 + exit: {
37 + _tag: "Failure",
38 + cause: [
39 + {
40 + _tag: "Fail",
41 + error: {
42 + _tag: "KeybindingsConfigParseError",
43 + configPath: "/demo/keybindings.json",
44 + detail: "Injected failure.",
45 + },
46 + },
47 + { _tag: "Interrupt", fiberId: 1 },
48 + ],
49 + },
50 + }),
51 + }),
52 + );
53 + }
54 + super.send(data);
55 + }
56 + };
57 + (window as unknown as { __fault: Fault }).__fault = {
58 + failConfigStreamWhileInterrupted: () => {
59 + deliver?.();
60 + return deliver !== null;
61 + },
62 + };
63 + });
64 + await browser.emulateMedia({ colorScheme: "dark", reducedMotion: "reduce" });
65 + await t3.open(thread);
66 +
67 + const reply = screen.getByText("The command printed one JSON line.");
68 + await expect(reply.first()).toBeVisible();
69 + await app.screenshot("1-connected");
70 +
71 + const delivered = await browser.evaluate(() =>
72 + (window as unknown as { __fault: Fault }).__fault.failConfigStreamWhileInterrupted(),
73 + );
74 + expect(delivered).toBe(true);
75 + // A live loop reconnects well within each wait; a dead one never does.
76 + await runPromise(Effect.sleep("5 seconds"));
77 + await app.screenshot("2-five-seconds-after-the-failure");
78 +
79 + await screen.getByRole("textbox").fill("Are you still there?");
80 + await screen.getByRole("textbox").press("Enter");
81 + await runPromise(Effect.sleep("3 seconds"));
82 + await app.screenshot("3-after-pressing-enter");
83 + await expect(reply.nth(1)).toBeVisible();
84 + },
85 +);