The best way to control Claude Code, Codex and more from any computer or phone
fix(client-runtime): connection supervisor restarts its background listeners after a defect (#39431)
Julius Marminge committed
Nov 19, 2026 at 16:38 UTC
003d53ace1f5255eaca52486e35d58fbd299e22e
2 files changed
+176
-30
packages/client-runtime/src/connection/supervisor.test.ts
+120
@@ -2936,3 +2936,123 @@ describe("EnvironmentSupervisor routes", () => {
2936
}),
2937
);
2938
});
2939
+
2940
+describe("EnvironmentSupervisor background fibers", () => {
2941
+ it.effect("hears the network return after its connectivity stream died", () =>
2942
+ Effect.gen(function* () {
2943
+ const harness = yield* makeHarness();
2944
+ const status = yield* SubscriptionRef.make<NetworkStatus>("offline");
2945
+ const subscriptions = yield* Ref.make(0);
2946
+ const dying = Connectivity.Connectivity.of({
2947
+ status: SubscriptionRef.get(status),
2948
+ changes: Stream.unwrap(
2949
+ Ref.updateAndGet(subscriptions, (count) => count + 1).pipe(
2950
+ Effect.map((count) =>
2951
+ count === 1
2952
+ ? Stream.die(new Error("platform connectivity stream died"))
2953
+ : SubscriptionRef.changes(status),
2954
+ ),
2955
+ ),
2956
+ ),
2957
+ });
2958
+ const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
2959
+ initiallyDesired: true,
2960
+ }).pipe(
2961
+ Effect.provideService(Connectivity.Connectivity, dying),
2962
+ Effect.provide(harness.dependencies),
2963
+ );
2964
+ yield* TestClock.adjust("10 minutes");
2965
+ expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("offline");
2966
+
2967
+ yield* SubscriptionRef.set(status, "online");
2968
+ yield* TestClock.adjust("10 minutes");
2969
+ expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
2970
+ }).pipe(Effect.provide(TestClock.layer())),
2971
+ );
2972
+
2973
+ it.effect("replaces a stalled session on return to the app after its wakeup stream died", () =>
2974
+ Effect.gen(function* () {
2975
+ const harness = yield* makeHarness({ probe: () => Effect.never });
2976
+ const returns = yield* SubscriptionRef.make(0);
2977
+ const subscriptions = yield* Ref.make(0);
2978
+ const dying = ConnectionWakeups.ConnectionWakeups.of({
2979
+ changes: Stream.unwrap(
2980
+ Ref.updateAndGet(subscriptions, (count) => count + 1).pipe(
2981
+ Effect.map((count) =>
2982
+ count === 1
2983
+ ? Stream.die(new Error("platform wakeup stream died"))
2984
+ : SubscriptionRef.changes(returns).pipe(
2985
+ Stream.drop(1),
2986
+ Stream.map(() => "application-active" as const),
2987
+ ),
2988
+ ),
2989
+ ),
2990
+ ),
2991
+ });
2992
+ const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
2993
+ initiallyDesired: true,
2994
+ }).pipe(
2995
+ Effect.provideService(ConnectionWakeups.ConnectionWakeups, dying),
2996
+ Effect.provide(harness.dependencies),
2997
+ );
2998
+ yield* TestClock.adjust("10 minutes");
2999
+ expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
3000
+ phase: "connected",
3001
+ generation: 1,
3002
+ });
3003
+
3004
+ // The user returns to the app; the session's probe never answers.
3005
+ yield* SubscriptionRef.update(returns, (count) => count + 1);
3006
+ yield* TestClock.adjust("15 seconds");
3007
+ yield* TestClock.adjust("15 seconds");
3008
+ yield* TestClock.adjust("10 minutes");
3009
+ expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
3010
+ phase: "connected",
3011
+ generation: 2,
3012
+ });
3013
+ }).pipe(Effect.provide(TestClock.layer())),
3014
+ );
3015
+
3016
+ it.effect("still moves to a better route after one route check crashed", () =>
3017
+ Effect.gen(function* () {
3018
+ const lanReachable = yield* Ref.make(false);
3019
+ const preflights = yield* Ref.make(0);
3020
+ const harness = yield* makeHarness({
3021
+ checkRoute: (route) =>
3022
+ route.target._tag === "BearerConnectionTarget"
3023
+ ? Ref.get(lanReachable).pipe(
3024
+ Effect.map((reachable) => (reachable ? "answered" : "silent")),
3025
+ )
3026
+ : Effect.succeed("unchecked"),
3027
+ preflight: () =>
3028
+ Ref.updateAndGet(preflights, (count) => count + 1).pipe(
3029
+ Effect.flatMap((count) =>
3030
+ count === 1
3031
+ ? Effect.die(new Error("route check threw"))
3032
+ : Ref.get(lanReachable).pipe(
3033
+ Effect.map((reachable) => (reachable ? "ready" : "silent")),
3034
+ ),
3035
+ ),
3036
+ ),
3037
+ prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
3038
+ });
3039
+ const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
3040
+ initiallyDesired: true,
3041
+ }).pipe(Effect.provide(harness.dependencies));
3042
+ yield* awaitState(
3043
+ supervisor.state,
3044
+ (state) => state.phase === "connected" && state.generation === 1,
3045
+ );
3046
+ yield* TestClock.adjust("60 seconds");
3047
+ expect(yield* Ref.get(preflights)).toBeGreaterThanOrEqual(1);
3048
+
3049
+ yield* Ref.set(lanReachable, true);
3050
+ for (let minute = 0; minute < 60; minute += 1) {
3051
+ yield* TestClock.adjust("60 seconds");
3052
+ }
3053
+ expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
3054
+ "BearerConnectionTarget",
3055
+ );
3056
+ }).pipe(Effect.provide(TestClock.layer())),
3057
+ );
3058
+});
packages/client-runtime/src/connection/supervisor.ts
+56
-30
@@ -1332,40 +1332,66 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
1332
// Registered before the fibers below so closing the scope interrupts them first.
1333
// A loop still running when the queue shuts down would report that as a failure.
1334
yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1335
- yield* connectivity.changes.pipe(
1336
- Stream.runForEach((network) =>
1337
- Ref.modify(intent, (current) => {
1338
- const next = targetNetwork(network);
1339
- return current.network === next
1340
- ? [false, current]
1341
- : ([true, { ...current, network: next }] as const);
1342
- }).pipe(
1343
- Effect.flatMap((changed) =>
1344
- changed
1345
- ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe(
1346
- Effect.andThen(signal({ _tag: "NetworkChanged", network })),
1347
- )
1348
- : Effect.void,
1349
- ),
1335
+ // A defect in a background listener would otherwise end it for good, silently.
1336
+ // Untraced: a span here would stay open for the supervisor's whole lifetime.
1337
+ const keepRunning = Effect.fnUntraced(
1338
+ function* (task: string, effect: Effect.Effect<void>, onRestart?: Effect.Effect<void>) {
1339
+ let current = effect;
1340
+ while (true) {
1341
+ const exit = yield* Effect.exit(current);
1342
+ if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
1343
+ yield* Effect.logError("Connection supervisor task failed; restarting it.").pipe(
1344
+ Effect.annotateLogs({
1345
+ "environment.id": target.environmentId,
1346
+ "environment.label": target.label,
1347
+ task,
1348
+ ...safeErrorLogAttributes(exit.cause.reasons.find(Cause.isDieReason)?.defect),
1349
+ }),
1350
+ );
1351
+ yield* Effect.sleep(RETRY_BASE_DELAY_MS);
1352
+ current = onRestart ? onRestart.pipe(Effect.andThen(effect)) : effect;
1353
+ }
1354
+ },
1355
+ (self) => Effect.forkScoped(self),
1356
+ );
1357
+
1358
+ const applyNetwork = (network: NetworkStatus) =>
1359
+ Ref.modify(intent, (current) => {
1360
+ const next = targetNetwork(network);
1361
+ return current.network === next
1362
+ ? [false, current]
1363
+ : ([true, { ...current, network: next }] as const);
1364
+ }).pipe(
1365
+ Effect.flatMap((changed) =>
1366
+ changed
1367
+ ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe(
1368
+ Effect.andThen(signal({ _tag: "NetworkChanged", network })),
1369
+ )
1370
+ : Effect.void,
1371
),
1351
- ),
1352
- Effect.forkScoped,
1372
+ );
1373
+ // A restarted listener may have missed changes while it was down.
1374
+ yield* keepRunning(
1375
+ "network-listener",
1376
+ connectivity.changes.pipe(Stream.runForEach(applyNetwork)),
1377
+ connectivity.status.pipe(Effect.flatMap(applyNetwork)),
1378
);
1354
- yield* wakeups.changes.pipe(
1355
- Stream.filter(ConnectionWakeups.reachesConnectionSupervisor),
1356
- Stream.runForEach((reason) =>
1357
- // A long suspension can hide a network change the platform never reported.
1358
- (reason === "network-changed" || reason === "application-active-long"
1359
- ? forgetSilentRoutes
1360
- : Effect.void
1361
- ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))),
1379
+ yield* keepRunning(
1380
+ "wakeup-listener",
1381
+ wakeups.changes.pipe(
1382
+ Stream.filter(ConnectionWakeups.reachesConnectionSupervisor),
1383
+ Stream.runForEach((reason) =>
1384
+ // A long suspension can hide a network change the platform never reported.
1385
+ (reason === "network-changed" || reason === "application-active-long"
1386
+ ? forgetSilentRoutes
1387
+ : Effect.void
1388
+ ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))),
1389
+ ),
1390
),
1363
- Effect.forkScoped,
1391
);
1365
- yield* Queue.take(betterRouteChecks).pipe(
1366
- Effect.flatMap(checkBetterRoutes),
1367
- Effect.forever,
1368
- Effect.forkScoped,
1392
+ yield* keepRunning(
1393
+ "better-route-checker",
1394
+ Queue.take(betterRouteChecks).pipe(Effect.flatMap(checkBetterRoutes), Effect.forever),
1395
);
1396
// A restarted loop has forgotten its retry ladder, so it waits out the
1397
// ladder's ceiling unless a signal cuts the wait short.