@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 1,458 lines 54.1 KB
Raw
1 import { msg } from "gt-i18n";
2 import { translatableDetail } from "../errors/translatable.ts";
3 import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
4 import * as Cause from "effect/Cause";
5 import * as Clock from "effect/Clock";
6 import * as Context from "effect/Context";
7 import * as Duration from "effect/Duration";
8 import * as Effect from "effect/Effect";
9 import * as Exit from "effect/Exit";
10 import * as Fiber from "effect/Fiber";
11 import * as Option from "effect/Option";
12 import * as Queue from "effect/Queue";
13 import * as Random from "effect/Random";
14 import * as Ref from "effect/Ref";
15 import * as Scope from "effect/Scope";
16 import * as Stream from "effect/Stream";
17 import * as SubscriptionRef from "effect/SubscriptionRef";
18 import * as Tracer from "effect/Tracer";
19
20 import type { ConnectionCatalogEntry, ConnectionRoute } from "./catalog.ts";
21 import * as Connectivity from "./connectivity.ts";
22 import * as ConnectionDriver from "./driver.ts";
23 import {
24 type ConnectionAttemptError,
25 type ConnectionTarget,
26 ConnectionTransientError,
27 type NetworkStatus,
28 NO_REFUSED_ROUTES,
29 type PreparedConnection,
30 type SupervisorConnectionState,
31 } from "./model.ts";
32 import * as RpcSession from "../rpc/session.ts";
33 import { safeErrorLogAttributes } from "../errors/safeLog.ts";
34 import { NETWORK_BLOCKING_HINT } from "../errors/network.ts";
35 import * as ConnectionWakeups from "./wakeups.ts";
36 import {
37 connectionRouteId,
38 connectionRouteKind,
39 connectionRoutes,
40 type ConnectionRouteKind,
41 entryWithRoutes,
42 insertRoute,
43 type ReportedEndpoint,
44 } from "./routes.ts";
45
46 const RETRY_BASE_DELAY_MS = 1_000;
47 const RETRY_MAX_DELAY_MS = 300_000;
48 const SESSION_LOSS_RETRY_DELAY_MS = 250;
49 // Someone is approving this device on the host; check back soon rather than climbing the ladder.
50 const APPROVAL_PENDING_RETRY_DELAY_MS = 4_000;
51 // A request lives 10 minutes; past that nobody is approving, so a forgotten device backs off.
52 const APPROVAL_PENDING_FAST_RETRIES = 150;
53 // A cold server with a large database can take longer than 15 seconds to
54 // answer, and that is not a reason to drop a working connection.
55 const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
56 const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT);
57 const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds");
58 const CONNECTION_PROBE_TIMEOUT = "15 seconds";
59 const QUICK_CONNECTION_PROBE_TIMEOUT = "3 seconds";
60 // After a long suspension pooled connections are often dead too, so the host
61 // check can hang with the socket. A replacement is cheaper than the wait.
62 const LONG_RESUME_PROBE_TIMEOUT = "1500 millis";
63 // A probe still unanswered this long checks the host with a fresh request. If
64 // the host answers while the socket stays silent, the socket is dead.
65 const SILENT_SOCKET_CHECK_AFTER = "500 millis";
66 const SILENT_SOCKET_RECHECK_EVERY = "1 second";
67 const FRAME_CHECK_EVERY = "100 millis";
68 // Off the LAN one lost packet can stall a live socket for a second, so a remote
69 // socket must stay silent this long, except after a long resume. It must end
70 // before QUICK_CONNECTION_PROBE_TIMEOUT so a resume replaces a silent socket
71 // instead of timing out.
72 const REMOTE_SILENT_SOCKET_AFTER = "2500 millis";
73 const BACKOFF_RESET_AFTER_MS = 30_000;
74 const BETTER_ROUTE_CHECK_INTERVAL_MS = 60_000;
75 // Each check of a dead route can wait out the full probe timeout, so a route
76 // that keeps failing is checked half as often each time, until the network
77 // changes and it may answer again.
78 const SILENT_ROUTE_BACKOFF_MAX_MS = 1_800_000;
79 // A better route that answered the check but then failed to connect is not
80 // tried again for this long, so a flaky LAN cannot bounce the connection.
81 const BETTER_ROUTE_COOLDOWN_MS = 300_000;
82
83 interface SupervisorIntent {
84 readonly desired: boolean;
85 readonly network: NetworkStatus;
86 }
87
88 type SupervisorSignal =
89 | { readonly _tag: "ConnectRequested" }
90 | { readonly _tag: "DisconnectRequested" }
91 | { readonly _tag: "RetryRequested" }
92 | { readonly _tag: "NetworkChanged"; readonly network: NetworkStatus }
93 | { readonly _tag: "Wakeup"; readonly reason: ConnectionWakeups.ConnectionWakeup }
94 | { readonly _tag: "BetterRouteAvailable"; readonly routeId: string };
95
96 interface PendingRetryTrace {
97 readonly previousAttempt: Tracer.Span;
98 readonly failureCount: number;
99 readonly delayMs: number;
100 readonly reason: ConnectionAttemptError["reason"];
101 }
102
103 type ConnectedLease = ConnectionDriver.EnvironmentConnectionLease & {
104 readonly activeRoute: ConnectionRoute | undefined;
105 readonly generation: number;
106 };
107
108 function mergeRefusedRoutes(
109 known: ReadonlyArray<string>,
110 refused: ReadonlyArray<string>,
111 keep: (routeId: string) => boolean,
112 ): ReadonlyArray<string> {
113 const next = [...new Set([...known, ...refused])].filter(keep);
114 return next.length === known.length && next.every((id, index) => id === known[index])
115 ? known
116 : next;
117 }
118
119 interface TracedAttemptFailure {
120 readonly error: ConnectionAttemptError;
121 readonly attemptSpan: Option.Option<Tracer.Span>;
122 }
123
124 type AttemptOutcome =
125 | {
126 readonly _tag: "Interrupted";
127 readonly established: boolean;
128 readonly stable: boolean;
129 readonly resetRetry: boolean;
130 }
131 | {
132 readonly _tag: "Failure";
133 readonly established: boolean;
134 readonly stable: boolean;
135 readonly failure: TracedAttemptFailure;
136 };
137
138 type EstablishmentEvent =
139 | {
140 readonly _tag: "Completed";
141 readonly exit: Exit.Exit<
142 {
143 readonly attemptSpan: Option.Option<Tracer.Span>;
144 readonly lease: ConnectionDriver.EnvironmentConnectionLease;
145 },
146 TracedAttemptFailure
147 >;
148 }
149 | { readonly _tag: "Interrupted"; readonly resetRetry: boolean }
150 | { readonly _tag: "TimedOut" };
151
152 export interface EnvironmentSupervisorOptions {
153 readonly initiallyDesired?: boolean;
154 readonly learnRoutes?: (input: {
155 readonly activeRoute: ConnectionRoute;
156 readonly reported: ReadonlyArray<ReportedEndpoint>;
157 }) => Effect.Effect<Option.Option<ConnectionCatalogEntry>>;
158 }
159
160 function retryDelayMs(failureCount: number, random: number): number {
161 const ceiling = Math.min(RETRY_MAX_DELAY_MS, RETRY_BASE_DELAY_MS * 2 ** (failureCount + 1));
162 return Math.round(ceiling / 2 + (ceiling / 2) * random);
163 }
164
165 interface SilentRoute {
166 readonly failures: number;
167 readonly until: bigint;
168 }
169
170 function backOffSilentRoute(previous: SilentRoute | undefined, now: bigint): SilentRoute {
171 const failures = (previous?.failures ?? 0) + 1;
172 const delayMs = Math.min(
173 SILENT_ROUTE_BACKOFF_MAX_MS,
174 BETTER_ROUTE_CHECK_INTERVAL_MS * 2 ** failures,
175 );
176 return { failures, until: now + BigInt(delayMs) * 1_000_000n };
177 }
178
179 function annotateTarget(target: ConnectionTarget) {
180 return Effect.annotateCurrentSpan({
181 "environment.id": target.environmentId,
182 "environment.label": target.label,
183 "environment.target.kind": target._tag,
184 });
185 }
186
187 function availableState(intent: SupervisorIntent, generation: number): SupervisorConnectionState {
188 return {
189 desired: false,
190 network: intent.network,
191 phase: "available",
192 stage: null,
193 attempt: 0,
194 generation,
195 lastFailure: null,
196 retryAt: null,
197 refusedRouteIds: NO_REFUSED_ROUTES,
198 };
199 }
200
201 function offlineState(
202 intent: SupervisorIntent,
203 generation: number,
204 attempt: number,
205 lastFailure: ConnectionAttemptError | null,
206 ): SupervisorConnectionState {
207 return {
208 desired: true,
209 network: intent.network,
210 phase: "offline",
211 stage: null,
212 attempt,
213 generation,
214 lastFailure,
215 retryAt: null,
216 refusedRouteIds: NO_REFUSED_ROUTES,
217 };
218 }
219
220 function connectingState(
221 intent: SupervisorIntent,
222 generation: number,
223 attempt: number,
224 lastFailure: ConnectionAttemptError | null,
225 stage: SupervisorConnectionState["stage"] = "preparing",
226 ): SupervisorConnectionState {
227 return {
228 desired: true,
229 network: intent.network,
230 phase: "connecting",
231 stage,
232 attempt,
233 generation,
234 lastFailure,
235 retryAt: null,
236 refusedRouteIds: NO_REFUSED_ROUTES,
237 };
238 }
239
240 function failureFromCause(
241 target: ConnectionTarget,
242 cause: Cause.Cause<TracedAttemptFailure>,
243 established: boolean,
244 stable: boolean,
245 ) {
246 return {
247 _tag: "Failure",
248 established,
249 stable,
250 failure: cause.reasons.find(Cause.isFailReason)?.error ?? unexpectedFailure(target),
251 } satisfies AttemptOutcome;
252 }
253
254 function unexpectedFailure(target: ConnectionTarget): TracedAttemptFailure {
255 return {
256 error: new ConnectionTransientError({
257 reason: "transport",
258 ...translatableDetail(msg("{host} connection failed unexpectedly.", { host: target.label })),
259 }),
260 attemptSpan: Option.none(),
261 };
262 }
263
264 export class EnvironmentSupervisor extends Context.Service<
265 EnvironmentSupervisor,
266 {
267 readonly target: ConnectionTarget;
268 readonly state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>;
269 readonly session: SubscriptionRef.SubscriptionRef<Option.Option<RpcSession.RpcSession>>;
270 readonly prepared: SubscriptionRef.SubscriptionRef<Option.Option<PreparedConnection>>;
271 readonly connect: Effect.Effect<void>;
272 readonly disconnect: Effect.Effect<void>;
273 readonly retryNow: Effect.Effect<void>;
274 }
275 >()("@t3tools/client-runtime/connection/supervisor/EnvironmentSupervisor") {}
276
277 export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
278 entry: ConnectionCatalogEntry,
279 options?: EnvironmentSupervisorOptions,
280 ): Effect.fn.Return<
281 EnvironmentSupervisor["Service"],
282 never,
283 | Connectivity.Connectivity
284 | ConnectionDriver.ConnectionDriver
285 | Scope.Scope
286 | ConnectionWakeups.ConnectionWakeups
287 > {
288 const target = entry.target;
289 const usesRelay = connectionRoutes(entry).some(
290 (route) => route.target._tag === "RelayConnectionTarget",
291 );
292 const setupTimeout = msg("{host} did not respond during connection setup.", {
293 host: target.label,
294 });
295 const setupTimeoutDetail = usesRelay
296 ? translatableDetail(setupTimeout, NETWORK_BLOCKING_HINT)
297 : translatableDetail(setupTimeout);
298 yield* annotateTarget(target);
299
300 const connectivity = yield* Connectivity.Connectivity;
301 const driver = yield* ConnectionDriver.ConnectionDriver;
302 const wakeups = yield* ConnectionWakeups.ConnectionWakeups;
303 // The primary environment served this client or runs beside it on loopback,
304 // so the device reporting offline (often wrongly) never parks it. The report
305 // still probes its live session, like any other network change.
306 const targetNetwork = (network: NetworkStatus): NetworkStatus =>
307 network === "offline" && target._tag === "PrimaryConnectionTarget" ? "unknown" : network;
308 const initialIntent: SupervisorIntent = {
309 desired: options?.initiallyDesired ?? false,
310 network: targetNetwork(yield* connectivity.status),
311 };
312 const intent = yield* Ref.make(initialIntent);
313 const signals = yield* Queue.unbounded<SupervisorSignal>();
314 const resetRetryState = yield* Ref.make(false);
315 const probeUnanswered = yield* Ref.make(false);
316 const state = yield* SubscriptionRef.make<SupervisorConnectionState>(
317 !initialIntent.desired
318 ? availableState(initialIntent, 0)
319 : initialIntent.network === "offline"
320 ? offlineState(initialIntent, 0, 0, null)
321 : connectingState(initialIntent, 0, 1, null),
322 );
323 const session = yield* SubscriptionRef.make<Option.Option<RpcSession.RpcSession>>(Option.none());
324 const prepared = yield* SubscriptionRef.make<Option.Option<PreparedConnection>>(Option.none());
325 // Learned routes arrive while a session is live, so the route list is read
326 // fresh by each attempt and check instead of being fixed at creation.
327 const currentEntry = yield* Ref.make(entry);
328 const currentRoutes = Ref.get(currentEntry).pipe(Effect.map(connectionRoutes));
329 const preferredRouteId = yield* Ref.make(Option.none<string>());
330 const routeCooldowns = yield* Ref.make<ReadonlyMap<string, bigint>>(new Map());
331 // Outlives sessions so a reconnect shows a refusal seen earlier; a route leaves once it connects.
332 const refusedRoutes = yield* Ref.make(NO_REFUSED_ROUTES);
333 // The generation keeps a check that started before a reset from recording
334 // its stale failures after it.
335 const silentRoutes = yield* Ref.make({
336 generation: 0,
337 routes: new Map<string, SilentRoute>() as ReadonlyMap<string, SilentRoute>,
338 });
339 const forgetSilentRoutes = Ref.update(silentRoutes, ({ generation }) => ({
340 generation: generation + 1,
341 routes: new Map(),
342 }));
343
344 const attemptEntry = Effect.gen(function* () {
345 const latest = yield* Ref.get(currentEntry);
346 const routes = connectionRoutes(latest);
347 const preferred = yield* Ref.getAndSet(preferredRouteId, Option.none());
348 if (Option.isNone(preferred)) return latest;
349 const route = routes.find(
350 (candidate) => connectionRouteId(candidate.target) === preferred.value,
351 );
352 return route === undefined
353 ? latest
354 : entryWithRoutes(latest, [route, ...routes.filter((candidate) => candidate !== route)]);
355 });
356
357 const learnRoutesFrom = Effect.fnUntraced(function* (
358 learn: NonNullable<EnvironmentSupervisorOptions["learnRoutes"]>,
359 lease: ConnectedLease,
360 ) {
361 const activeRoute = lease.activeRoute;
362 if (activeRoute === undefined) return;
363 yield* lease.session.subscribeServerConfig().pipe(
364 Stream.filter((event) => event.type === "snapshot"),
365 Stream.map((event) => event.config.directEndpoints),
366 Stream.filter((reported) => reported !== undefined),
367 Stream.changesWith((previous, next) => JSON.stringify(previous) === JSON.stringify(next)),
368 Stream.runForEach((reported) =>
369 Effect.gen(function* () {
370 const updated = yield* learn({ activeRoute, reported });
371 if (Option.isSome(updated)) {
372 yield* Ref.set(currentEntry, updated.value);
373 yield* requestBetterRouteCheck(lease);
374 }
375 }),
376 ),
377 Effect.ignoreCause({ log: true }),
378 );
379 });
380
381 const routeIndex = (lease: ConnectedLease) =>
382 currentRoutes.pipe(
383 Effect.map((routes) => {
384 const activeId = connectionRouteId(lease.prepared.target);
385 const index = routes.findIndex((route) => connectionRouteId(route.target) === activeId);
386 return index === -1 && lease.activeRoute !== undefined
387 ? insertRoute(routes, lease.activeRoute).findIndex(
388 (route) => connectionRouteId(route.target) === activeId,
389 )
390 : index;
391 }),
392 );
393
394 const checkBetterRoutes = Effect.fnUntraced(function* (lease: ConnectedLease) {
395 const latest = yield* Ref.get(currentEntry);
396 const current = yield* routeIndex(lease);
397 if (current <= 0) return;
398 const now = yield* Clock.monotonicTimeNanos;
399 const cooldowns = yield* Ref.get(routeCooldowns);
400 const silent = yield* Ref.get(silentRoutes);
401 const better = connectionRoutes(latest)
402 .slice(0, current)
403 .filter((route) => {
404 const id = connectionRouteId(route.target);
405 return (cooldowns.get(id) ?? 0n) <= now && (silent.routes.get(id)?.until ?? 0n) <= now;
406 });
407 const results = yield* Effect.forEach(
408 better,
409 (route: ConnectionRoute) => driver.preflight(latest, route),
410 { concurrency: "unbounded" },
411 );
412 yield* Ref.update(silentRoutes, (previous) => {
413 if (previous.generation !== silent.generation) return previous;
414 const routes = new Map(previous.routes);
415 for (const [position, route] of better.entries()) {
416 const id = connectionRouteId(route.target);
417 if (results[position] !== "silent") routes.delete(id);
418 else routes.set(id, backOffSilentRoute(previous.routes.get(id), now));
419 }
420 return { generation: previous.generation, routes };
421 });
422 const live = yield* SubscriptionRef.get(session);
423 if (Option.isNone(live) || live.value !== lease.session) return;
424 const refused = better
425 .filter((_route, position) => results[position] === "refused")
426 .map((route) => connectionRouteId(route.target));
427 if (refused.length > 0) {
428 const known = yield* Ref.updateAndGet(refusedRoutes, (current) =>
429 mergeRefusedRoutes(current, refused, () => true),
430 );
431 yield* SubscriptionRef.updateSome(state, (current) =>
432 current.phase === "connected" &&
433 current.generation === lease.generation &&
434 current.refusedRouteIds !== known
435 ? Option.some({ ...current, refusedRouteIds: known })
436 : Option.none(),
437 );
438 }
439 const index = results.indexOf("ready");
440 if (index === -1) return;
441 yield* signal({
442 _tag: "BetterRouteAvailable",
443 routeId: connectionRouteId(better[index]!.target),
444 });
445 });
446
447 const betterRouteChecks = yield* Queue.dropping<ConnectedLease>(1);
448 const requestBetterRouteCheck = (lease: ConnectedLease) =>
449 routeIndex(lease).pipe(
450 Effect.flatMap((index) =>
451 index > 0 ? Queue.offer(betterRouteChecks, lease).pipe(Effect.asVoid) : Effect.void,
452 ),
453 );
454
455 const clearLease = Effect.all(
456 [SubscriptionRef.set(session, Option.none()), SubscriptionRef.set(prepared, Option.none())],
457 { discard: true },
458 );
459
460 const setState = Effect.fn("EnvironmentSupervisor.setState")(function* (
461 next: SupervisorConnectionState,
462 ) {
463 yield* SubscriptionRef.set(state, next);
464 });
465
466 const signal = Effect.fn("EnvironmentSupervisor.signal")(function* (next: SupervisorSignal) {
467 yield* Queue.offer(signals, next);
468 });
469
470 const logManagedRelayAccountChange = Effect.logInfo(
471 "Managed relay account changed; restarting the environment connection.",
472 ).pipe(
473 Effect.annotateLogs({
474 "environment.id": target.environmentId,
475 "environment.label": target.label,
476 }),
477 );
478
479 const logUnexpectedFailure = (
480 failed: "attempt" | "release" | "loop",
481 cause: Cause.Cause<TracedAttemptFailure>,
482 ) => {
483 const died = cause.reasons.find(Cause.isDieReason);
484 if (!died && cause.reasons.some(Cause.isFailReason)) return Effect.void;
485 return Effect.logError(
486 `Connection ${failed} failed with an unexpected ${died ? "defect" : "interruption"}.`,
487 ).pipe(
488 Effect.annotateLogs({
489 "environment.id": target.environmentId,
490 "environment.label": target.label,
491 "cause.reason_count": cause.reasons.length,
492 ...(died ? safeErrorLogAttributes(died.defect) : {}),
493 }),
494 );
495 };
496
497 const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* (
498 attempt: number,
499 nextGeneration: number,
500 lastFailure: ConnectionAttemptError | null,
501 progress: ConnectionDriver.ConnectionDriverProgress,
502 ) {
503 if ("prepared" in progress) {
504 yield* SubscriptionRef.set(prepared, Option.some(progress.prepared));
505 }
506 yield* setState(
507 connectingState(yield* Ref.get(intent), nextGeneration, attempt, lastFailure, progress.stage),
508 );
509 });
510
511 const establishConnection = Effect.fnUntraced(function* (
512 attempt: number,
513 nextGeneration: number,
514 lastFailure: ConnectionAttemptError | null,
515 synchronizedRoutes: Ref.Ref<number>,
516 ) {
517 return yield* driver.connect(yield* attemptEntry, (progress) =>
518 (progress.stage === "synchronizing"
519 ? Ref.update(synchronizedRoutes, (count) => count + 1)
520 : Effect.void
521 ).pipe(Effect.andThen(reportProgress(attempt, nextGeneration, lastFailure, progress))),
522 );
523 });
524
525 // Low-cardinality dimensions for time-to-connected dashboards; the error
526 // detail stays in the span status, never in an attribute.
527 const annotateAttemptOutcome = Effect.fnUntraced(function* (
528 exit: Exit.Exit<ConnectionDriver.EnvironmentConnectionLease, ConnectionAttemptError>,
529 ) {
530 if (Exit.isSuccess(exit)) {
531 const leaseTarget = exit.value.prepared.target;
532 const route = (yield* currentRoutes).find(
533 (candidate) => connectionRouteId(candidate.target) === connectionRouteId(leaseTarget),
534 );
535 return yield* Effect.annotateCurrentSpan({
536 "connection.outcome": "connected",
537 "connection.route": connectionRouteKind(
538 route ?? { target: leaseTarget, profile: Option.none() },
539 ),
540 });
541 }
542 if (Cause.hasInterruptsOnly(exit.cause)) {
543 return yield* Effect.annotateCurrentSpan("connection.outcome", "interrupted");
544 }
545 const failure = exit.cause.reasons.find(Cause.isFailReason)?.error;
546 yield* Effect.annotateCurrentSpan({
547 "connection.outcome": "failed",
548 "connection.failure.tag": failure?._tag ?? "defect",
549 "connection.failure.reason": failure?.reason ?? "defect",
550 });
551 });
552
553 const traceRelayEstablishment = (
554 effect: Effect.Effect<
555 ConnectionDriver.EnvironmentConnectionLease,
556 ConnectionAttemptError,
557 Scope.Scope
558 >,
559 attempt: number,
560 nextGeneration: number,
561 pendingRetry: Option.Option<PendingRetryTrace>,
562 ) => {
563 // The span must end with the ConnectionAttemptError itself: the exporter
564 // can only describe an Error-like failure, and a TracedAttemptFailure
565 // wrapper reached it as "[object Object]". The wrapper is added outside.
566 let attemptSpan = Option.none<Tracer.Span>();
567 const traced = Effect.gen(function* () {
568 attemptSpan = Option.some(yield* Effect.currentSpan.pipe(Effect.orDie));
569 yield* Effect.annotateCurrentSpan({
570 "environment.id": target.environmentId,
571 "environment.target.kind": target._tag,
572 "connection.attempt": attempt,
573 "connection.generation": nextGeneration,
574 "connection.retry.failure_count": Option.match(pendingRetry, {
575 onNone: () => 0,
576 onSome: (retry) => retry.failureCount,
577 }),
578 });
579 return yield* effect.pipe(Effect.onExit(annotateAttemptOutcome));
580 }).pipe(Effect.withSpan("relay.connection.attempt", { root: true }));
581
582 return Option.match(pendingRetry, {
583 onNone: () => traced,
584 onSome: (retry) =>
585 traced.pipe(
586 Effect.linkSpans(retry.previousAttempt, {
587 "connection.retry.delay_ms": retry.delayMs,
588 "connection.retry.reason": retry.reason,
589 }),
590 ),
591 }).pipe(
592 Effect.map((lease) => ({ attemptSpan, lease })),
593 Effect.mapError((error): TracedAttemptFailure => ({ error, attemptSpan })),
594 withRelayClientTracing,
595 );
596 };
597
598 const establishTracedConnection = Effect.fnUntraced(function* (
599 attempt: number,
600 nextGeneration: number,
601 lastFailure: ConnectionAttemptError | null,
602 pendingRetry: Option.Option<PendingRetryTrace>,
603 synchronizedRoutes: Ref.Ref<number>,
604 ) {
605 if (usesRelay) {
606 return yield* traceRelayEstablishment(
607 establishConnection(attempt, nextGeneration, lastFailure, synchronizedRoutes),
608 attempt,
609 nextGeneration,
610 pendingRetry,
611 );
612 }
613 return yield* establishConnection(
614 attempt,
615 nextGeneration,
616 lastFailure,
617 synchronizedRoutes,
618 ).pipe(
619 Effect.map((lease) => ({
620 attemptSpan: Option.none<Tracer.Span>(),
621 lease,
622 })),
623 Effect.mapError((error): TracedAttemptFailure => ({
624 error,
625 attemptSpan: Option.none(),
626 })),
627 );
628 });
629
630 const waitForEstablishmentInterrupt = Effect.fnUntraced(function* (ignoreOffline: boolean) {
631 // A primary attempt keeps running through an offline report, so the
632 // network returning restarts it in case it stalled meanwhile.
633 let deviceOffline = (yield* connectivity.status) === "offline";
634 for (;;) {
635 const next = yield* Queue.take(signals);
636 switch (next._tag) {
637 case "DisconnectRequested":
638 return false;
639 case "RetryRequested":
640 // Once the loop has taken this retry's flag, the running attempt is
641 // the one the retry asked for.
642 if (yield* Ref.get(resetRetryState)) {
643 return false;
644 }
645 break;
646 case "NetworkChanged":
647 if (target._tag !== "PrimaryConnectionTarget") {
648 if (next.network === "offline" && !ignoreOffline) {
649 return false;
650 }
651 } else if (next.network === "offline") {
652 deviceOffline = true;
653 } else if (deviceOffline) {
654 return true;
655 }
656 break;
657 case "ConnectRequested":
658 case "BetterRouteAvailable":
659 break;
660 case "Wakeup":
661 // An attempt that ran into a long suspension is likely stuck.
662 if (next.reason === "application-active-long") {
663 return true;
664 }
665 if (next.reason === "credentials-changed" && usesRelay) {
666 yield* logManagedRelayAccountChange;
667 return false;
668 }
669 break;
670 }
671 }
672 });
673
674 const isRelayLease = (lease: ConnectionDriver.EnvironmentConnectionLease) =>
675 lease.prepared.target._tag === "RelayConnectionTarget";
676
677 const connectedLeaseEnd = Effect.fnUntraced(function* (
678 next: SupervisorSignal,
679 lease: ConnectionDriver.EnvironmentConnectionLease,
680 ) {
681 if (next._tag === "DisconnectRequested") {
682 return "end" as const;
683 }
684 if (next._tag === "BetterRouteAvailable") {
685 yield* Ref.set(preferredRouteId, Option.some(next.routeId));
686 return "reset" as const;
687 }
688 // Only a session over T3 Connect holds the old account's credential.
689 if (next._tag === "Wakeup" && next.reason === "credentials-changed" && isRelayLease(lease)) {
690 yield* logManagedRelayAccountChange;
691 return "end" as const;
692 }
693 return undefined;
694 });
695
696 const leaseRouteKind = (lease: ConnectionDriver.EnvironmentConnectionLease) =>
697 currentRoutes.pipe(
698 Effect.map((routes) => {
699 const route = routes.find(
700 (candidate) =>
701 connectionRouteId(candidate.target) === connectionRouteId(lease.prepared.target),
702 );
703 return route === undefined ? undefined : connectionRouteKind(route);
704 }),
705 );
706
707 const isLongResume = (signal: SupervisorSignal) =>
708 signal._tag === "Wakeup" && signal.reason === "application-active-long";
709
710 const answersQuickly = (routeKind: ConnectionRouteKind | undefined) =>
711 routeKind === "loopback" || routeKind === "lan";
712
713 const probeTimeoutFor = (
714 next: SupervisorSignal,
715 routeKind: ConnectionRouteKind | undefined,
716 ): Duration.Input | undefined => {
717 switch (next._tag) {
718 case "RetryRequested":
719 return QUICK_CONNECTION_PROBE_TIMEOUT;
720 case "NetworkChanged":
721 return next.network === "offline" ? QUICK_CONNECTION_PROBE_TIMEOUT : undefined;
722 case "Wakeup":
723 if (next.reason === "application-active") {
724 return CONNECTION_PROBE_TIMEOUT;
725 }
726 // After a long suspension the data on screen is stale, so a slow
727 // answer reconnects as quickly as a missing one, even on a remote
728 // route where a reconnect takes longer than the wait it replaces.
729 if (next.reason === "application-active-long") {
730 return LONG_RESUME_PROBE_TIMEOUT;
731 }
732 if (next.reason !== "application-active-probe" && next.reason !== "network-changed") {
733 return undefined;
734 }
735 // Only loopback and LAN answer within 3 seconds reliably. A tailnet
736 // over DERP, a VPN, or the relay can spike past it on a live socket;
737 // a dead one is still caught early by the silent socket check.
738 return answersQuickly(routeKind)
739 ? QUICK_CONNECTION_PROBE_TIMEOUT
740 : CONNECTION_PROBE_TIMEOUT;
741 case "ConnectRequested":
742 case "DisconnectRequested":
743 case "BetterRouteAvailable":
744 return undefined;
745 }
746 };
747
748 const isForegroundProbe = (next: SupervisorSignal) =>
749 next._tag === "Wakeup" && next.reason === "application-active";
750
751 // Ends "silent" when the host answers a fresh request while the socket
752 // delivers nothing, as when iOS kills a suspended socket without a close
753 // event, and "alive" once the socket delivers a frame. A slow or
754 // unreachable host never ends it.
755 const awaitSilentSocket = Effect.fn("EnvironmentSupervisor.awaitSilentSocket")(function* (
756 lease: ConnectedLease,
757 routeKind: ConnectionRouteKind | undefined,
758 longResume: boolean,
759 ) {
760 const startedAt = yield* Clock.monotonicTimeNanos;
761 yield* Effect.sleep(SILENT_SOCKET_CHECK_AFTER);
762 const configLoaded = yield* lease.session.initialConfig.pipe(
763 Effect.as(true),
764 Effect.catchCause(() => Effect.succeed(false)),
765 );
766 if (!configLoaded) return yield* Effect.never;
767 // Frames that queued while the app was suspended have drained by now, so
768 // a later frame shows the path still works.
769 const framesBefore = yield* lease.session.inboundFrames;
770 const socketSilent = lease.session.inboundFrames.pipe(
771 Effect.map((frames) => frames === framesBefore),
772 );
773 // The session only counts frames, so a host check slower than the probe
774 // deadline would otherwise hide a frame until after the socket is replaced.
775 const frameArrives = Effect.sleep(FRAME_CHECK_EVERY).pipe(
776 Effect.andThen(socketSilent),
777 Effect.repeat({ while: (silent) => silent }),
778 Effect.as("alive" as const),
779 );
780 const hostAnswersWhileSilent = Effect.gen(function* () {
781 // A radio still waking up or a stale pooled connection can fail one check.
782 while ((yield* driver.checkHost(lease.prepared)) !== "answered") {
783 yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY);
784 }
785 if (!answersQuickly(routeKind) && !longResume) {
786 const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt;
787 const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor;
788 if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining));
789 }
790 return (yield* socketSilent) ? ("silent" as const) : ("alive" as const);
791 });
792 return yield* Effect.race(hostAnswersWhileSilent, frameArrives);
793 });
794
795 // Returning to the app, an explicit retry, and the network reporting offline
796 // all probe the live session instead of replacing it, so a healthy socket is
797 // not torn down (the offline report is often wrong, for example for a
798 // loopback server).
799 const monitorConnectedLease = Effect.fnUntraced(function* (lease: ConnectedLease) {
800 // A probe answers an explicit retry here, so the retry must not also reset
801 // the backoff of a later, unrelated failure.
802 const takeSignal = Queue.take(signals).pipe(
803 Effect.tap((next) =>
804 next._tag === "RetryRequested" ? Ref.set(resetRetryState, false) : Effect.void,
805 ),
806 );
807 // Ticks always run: the check skips the preferred route itself, and the
808 // route list can grow while connected (learned routes).
809 yield* Stream.tick(Duration.millis(BETTER_ROUTE_CHECK_INTERVAL_MS)).pipe(
810 Stream.drop(1),
811 Stream.runForEach(() => requestBetterRouteCheck(lease)),
812 Effect.forkScoped,
813 );
814 const routeKind = yield* leaseRouteKind(lease);
815 for (;;) {
816 const next = yield* takeSignal;
817 const end = yield* connectedLeaseEnd(next, lease);
818 if (end !== undefined) {
819 return end === "reset";
820 }
821 if (
822 (next._tag === "NetworkChanged" && next.network !== "offline") ||
823 (next._tag === "Wakeup" && ConnectionWakeups.resetsRetryBackoff(next.reason))
824 ) {
825 yield* requestBetterRouteCheck(lease);
826 }
827 const probeTimeout = probeTimeoutFor(next, routeKind);
828 if (probeTimeout === undefined) {
829 continue;
830 }
831 yield* Ref.set(probeUnanswered, true);
832 let probe = yield* Effect.forkChild(lease.session.probe);
833 // A busy server can miss one foreground probe while the socket is still
834 // healthy, so the first miss asks again and only the follow-up's miss
835 // replaces the session.
836 let toleratesMiss = isForegroundProbe(next);
837 let silentSocket: Fiber.Fiber<"silent" | "alive"> | undefined;
838 // A signal during a running check, such as a network change that kills
839 // the socket, needs a fresh check only if the running one sees a frame.
840 let recheck = false;
841 let longResume = isLongResume(next);
842 const startSilentSocketCheck = Effect.gen(function* () {
843 silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind, longResume));
844 recheck = false;
845 });
846 // The web and desktop foreground probe allows a busy host 15 seconds,
847 // and a busy host can answer a fresh request before a queued probe.
848 const armSilentSocketCheck = Effect.fnUntraced(function* (signal: SupervisorSignal) {
849 if (isForegroundProbe(signal)) return;
850 if (silentSocket !== undefined) {
851 recheck = true;
852 return;
853 }
854 yield* startSilentSocketCheck;
855 });
856 yield* armSilentSocketCheck(next);
857 const stopProbe = Effect.suspend(() =>
858 Effect.all(
859 [Fiber.interrupt(probe), silentSocket ? Fiber.interrupt(silentSocket) : Effect.void],
860 { discard: true },
861 ),
862 );
863 let longResumeStartedAt = yield* Clock.monotonicTimeNanos;
864 let deadline = longResumeStartedAt + Duration.toNanosUnsafe(probeTimeout);
865 for (;;) {
866 const remaining = deadline - (yield* Clock.monotonicTimeNanos);
867 const probeEvent = yield* Effect.raceAllFirst([
868 Fiber.await(probe).pipe(
869 Effect.map((exit) => ({ _tag: "ProbeCompleted" as const, exit })),
870 ),
871 takeSignal.pipe(Effect.map((signal) => ({ _tag: "Signal" as const, signal }))),
872 Effect.sleep(Duration.nanos(remaining > 0n ? remaining : 0n)).pipe(
873 Effect.as({ _tag: "TimedOut" as const }),
874 ),
875 (silentSocket ? Fiber.join(silentSocket) : Effect.never).pipe(
876 Effect.map((check) =>
877 check === "silent"
878 ? { _tag: "SocketSilent" as const }
879 : { _tag: "SocketAlive" as const },
880 ),
881 ),
882 ]);
883 if (probeEvent._tag === "SocketAlive") {
884 silentSocket = undefined;
885 // A frame shows the socket still works, so its probe gets the usual resume time.
886 if (longResume) {
887 const aliveDeadline =
888 longResumeStartedAt + Duration.toNanosUnsafe(QUICK_CONNECTION_PROBE_TIMEOUT);
889 if (aliveDeadline > deadline) deadline = aliveDeadline;
890 }
891 if (recheck) yield* startSilentSocketCheck;
892 continue;
893 }
894 if (probeEvent._tag === "SocketSilent") {
895 // The host is reachable, so replace the socket at once, even while
896 // the network reports offline.
897 yield* stopProbe;
898 return true;
899 }
900 if (probeEvent._tag === "TimedOut") {
901 if (toleratesMiss) {
902 toleratesMiss = false;
903 yield* Fiber.interrupt(probe);
904 probe = yield* Effect.forkChild(lease.session.probe);
905 deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout);
906 continue;
907 }
908 yield* stopProbe;
909 // A long suspension leaves the socket suspect even when the network
910 // reports offline, so it is replaced like a silent one.
911 if (longResume) return true;
912 return yield* new ConnectionTransientError({
913 reason: "timeout",
914 ...translatableDetail(
915 msg("{host} did not respond to a connection health check.", {
916 host: target.label,
917 }),
918 ),
919 });
920 }
921 if (probeEvent._tag === "ProbeCompleted") {
922 if (silentSocket) yield* Fiber.interrupt(silentSocket);
923 if (Exit.isSuccess(probeEvent.exit)) {
924 yield* Ref.set(probeUnanswered, false);
925 }
926 yield* probeEvent.exit;
927 break;
928 }
929 const endDuringProbe = yield* connectedLeaseEnd(probeEvent.signal, lease);
930 if (endDuringProbe !== undefined) {
931 yield* stopProbe;
932 return endDuringProbe === "reset";
933 }
934 const signalTimeout = probeTimeoutFor(probeEvent.signal, routeKind);
935 if (signalTimeout !== undefined) {
936 // A retry or network signal wants a definite answer, even when its
937 // own deadline lands after the foreground one.
938 if (!isForegroundProbe(probeEvent.signal)) toleratesMiss = false;
939 if (isLongResume(probeEvent.signal) && !longResume) {
940 longResume = true;
941 longResumeStartedAt = yield* Clock.monotonicTimeNanos;
942 }
943 yield* armSilentSocketCheck(probeEvent.signal);
944 const signalDeadline =
945 (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout);
946 if (signalDeadline < deadline) deadline = signalDeadline;
947 }
948 }
949 }
950 });
951
952 // Advanced as a session goes live and kept across loop restarts, so no two sessions share one.
953 let generation = 0;
954 const runAttempt = Effect.fnUntraced(function* (
955 attempt: number,
956 nextGeneration: number,
957 lastFailure: ConnectionAttemptError | null,
958 pendingRetry: Option.Option<PendingRetryTrace>,
959 ignoreOffline: boolean,
960 ) {
961 const switchingTo = yield* Ref.get(preferredRouteId);
962 yield* SubscriptionRef.set(prepared, Option.none());
963 const synchronizedRoutes = yield* Ref.make(0);
964 const establishment = yield* Effect.raceAllFirst([
965 Effect.exit(
966 establishTracedConnection(
967 attempt,
968 nextGeneration,
969 lastFailure,
970 pendingRetry,
971 synchronizedRoutes,
972 ),
973 ).pipe(
974 Effect.map((exit): EstablishmentEvent => ({
975 _tag: "Completed",
976 exit,
977 })),
978 ),
979 waitForEstablishmentInterrupt(ignoreOffline).pipe(
980 Effect.map((resetRetry): EstablishmentEvent => ({
981 _tag: "Interrupted",
982 resetRetry,
983 })),
984 ),
985 // Each route may use the full setup time before the next is tried, and
986 // each route that reaches synchronizing adds its own snapshot budget, so
987 // a route that fails late does not eat the next route's time.
988 Effect.gen(function* () {
989 const routes = connectionRoutes(yield* Ref.get(currentEntry));
990 const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routes.length));
991 // The desktop bounds SSH preparation, which may download a server
992 // release, so for SSH-only entries the clock stops while preparing.
993 const pausesWhilePreparing = routes.every((route) => connectionRouteKind(route) === "ssh");
994 const preparing = (expected: boolean) =>
995 SubscriptionRef.changes(state).pipe(
996 Stream.filter((current) => (current.stage === "preparing") === expected),
997 Stream.runHead,
998 );
999 let spent = 0n;
1000 while (true) {
1001 if (pausesWhilePreparing) yield* preparing(false);
1002 const synchronized = BigInt(yield* Ref.get(synchronizedRoutes));
1003 const remaining =
1004 setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout) - spent;
1005 if (remaining <= 0n) {
1006 return { _tag: "TimedOut" } as const satisfies EstablishmentEvent;
1007 }
1008 const resumedAt = yield* Clock.monotonicTimeNanos;
1009 const sleep = Effect.sleep(Duration.nanos(remaining));
1010 yield* pausesWhilePreparing ? Effect.raceFirst(sleep, preparing(true)) : sleep;
1011 spent += (yield* Clock.monotonicTimeNanos) - resumedAt;
1012 }
1013 }),
1014 ]);
1015
1016 if (establishment._tag === "Interrupted") {
1017 return {
1018 _tag: "Interrupted",
1019 established: false,
1020 stable: false,
1021 resetRetry: establishment.resetRetry,
1022 } satisfies AttemptOutcome;
1023 }
1024 if (establishment._tag === "TimedOut") {
1025 return {
1026 _tag: "Failure",
1027 established: false,
1028 stable: false,
1029 failure: {
1030 error: new ConnectionTransientError({
1031 reason: "timeout",
1032 ...setupTimeoutDetail,
1033 }),
1034 attemptSpan: Option.none(),
1035 },
1036 } satisfies AttemptOutcome;
1037 }
1038 if (Exit.isFailure(establishment.exit)) {
1039 yield* logUnexpectedFailure("attempt", establishment.exit.cause);
1040 return failureFromCause(target, establishment.exit.cause, false, false);
1041 }
1042
1043 const active = establishment.exit.value;
1044 if (Option.isSome(switchingTo)) {
1045 const landed = connectionRouteId(active.lease.prepared.target);
1046 if (landed !== switchingTo.value) {
1047 const until =
1048 (yield* Clock.monotonicTimeNanos) + BigInt(BETTER_ROUTE_COOLDOWN_MS) * 1_000_000n;
1049 yield* Ref.update(routeCooldowns, (current) =>
1050 new Map(current).set(switchingTo.value, until),
1051 );
1052 }
1053 }
1054 const currentIntent = yield* Ref.get(intent);
1055 if (!currentIntent.desired || (currentIntent.network === "offline" && !ignoreOffline)) {
1056 return {
1057 _tag: "Interrupted",
1058 established: false,
1059 stable: false,
1060 resetRetry: false,
1061 } satisfies AttemptOutcome;
1062 }
1063
1064 const routes = yield* currentRoutes;
1065 const activeRouteId = connectionRouteId(active.lease.prepared.target);
1066 const connectedLease: ConnectedLease = {
1067 ...active.lease,
1068 activeRoute: routes.find((route) => connectionRouteId(route.target) === activeRouteId),
1069 generation: nextGeneration,
1070 };
1071 const refusedRouteIds = yield* Ref.updateAndGet(refusedRoutes, (known) =>
1072 mergeRefusedRoutes(
1073 known,
1074 active.lease.refusedRouteIds,
1075 (id) =>
1076 id !== activeRouteId && routes.some((route) => connectionRouteId(route.target) === id),
1077 ),
1078 );
1079 const connectedAt = yield* Clock.currentTimeMillis;
1080 generation = nextGeneration;
1081 yield* SubscriptionRef.set(prepared, Option.some(connectedLease.prepared));
1082 yield* SubscriptionRef.set(session, Option.some(connectedLease.session));
1083 if (options?.learnRoutes !== undefined) {
1084 yield* learnRoutesFrom(options.learnRoutes, connectedLease).pipe(Effect.forkScoped);
1085 }
1086 yield* setState({
1087 desired: true,
1088 network: currentIntent.network,
1089 phase: "connected",
1090 stage: null,
1091 attempt,
1092 generation: nextGeneration,
1093 lastFailure: null,
1094 retryAt: null,
1095 refusedRouteIds,
1096 });
1097
1098 const connectedExit = yield* Effect.raceFirst(
1099 connectedLease.session.closed.pipe(
1100 Effect.mapError((error): TracedAttemptFailure => ({
1101 error,
1102 attemptSpan: active.attemptSpan,
1103 })),
1104 ),
1105 monitorConnectedLease(connectedLease).pipe(
1106 Effect.mapError((error): TracedAttemptFailure => ({
1107 error,
1108 attemptSpan: active.attemptSpan,
1109 })),
1110 ),
1111 ).pipe(Effect.exit);
1112 const connectedForMs = (yield* Clock.currentTimeMillis) - connectedAt;
1113 if (Exit.isSuccess(connectedExit)) {
1114 return {
1115 _tag: "Interrupted",
1116 established: true,
1117 stable: connectedForMs >= BACKOFF_RESET_AFTER_MS,
1118 resetRetry: connectedExit.value,
1119 } satisfies AttemptOutcome;
1120 }
1121 yield* logUnexpectedFailure("attempt", connectedExit.cause);
1122 const outcome = failureFromCause(
1123 target,
1124 connectedExit.cause,
1125 true,
1126 connectedForMs >= BACKOFF_RESET_AFTER_MS,
1127 );
1128 yield* Effect.void.pipe(
1129 Effect.withSpan("EnvironmentSupervisor.connectionLost", {
1130 root: true,
1131 attributes: {
1132 "environment.id": target.environmentId,
1133 "environment.label": target.label,
1134 "environment.target.kind": target._tag,
1135 "connection.connected_ms": connectedForMs,
1136 "connection.failure.reason": outcome.failure.error.reason,
1137 "connection.failure.detail": outcome.failure.error.detail,
1138 },
1139 }),
1140 );
1141 return outcome;
1142 }, Effect.ensuring(clearLease));
1143
1144 const waitForRetrySignal = Effect.fnUntraced(function* (delayMs: number) {
1145 // @effect-diagnostics-next-line raceFirstWithSleepToTimeout:off - the sleep is the retry delay (false), not a timeout around the signal loop
1146 return yield* Effect.raceFirst(
1147 Effect.sleep(delayMs).pipe(Effect.as(false)),
1148 Effect.gen(function* () {
1149 for (;;) {
1150 const next = yield* Queue.take(signals);
1151 switch (next._tag) {
1152 case "Wakeup":
1153 return ConnectionWakeups.resetsRetryBackoff(next.reason);
1154 case "ConnectRequested":
1155 case "DisconnectRequested":
1156 case "RetryRequested":
1157 case "NetworkChanged":
1158 return false;
1159 case "BetterRouteAvailable":
1160 break;
1161 }
1162 }
1163 }),
1164 );
1165 });
1166
1167 const waitForSignal = Queue.take(signals).pipe(
1168 Effect.repeat({ while: (next) => next._tag === "BetterRouteAvailable" }),
1169 Effect.map(
1170 (next) => next._tag === "Wakeup" && ConnectionWakeups.resetsRetryBackoff(next.reason),
1171 ),
1172 );
1173
1174 const run = Effect.fnUntraced(function* () {
1175 let failureCount = 0;
1176 // Approval waits count as attempts but not toward backoff, so a later outage starts low.
1177 let backoffFailures = 0;
1178 let approvalWaits = 0;
1179 let latestFailure: ConnectionAttemptError | null = null;
1180 let pendingRetry = Option.none<PendingRetryTrace>();
1181 const resetRetryLadder = () => {
1182 failureCount = 0;
1183 backoffFailures = 0;
1184 approvalWaits = 0;
1185 pendingRetry = Option.none();
1186 };
1187 let replacing = false;
1188
1189 for (;;) {
1190 const retryRequested = yield* Ref.getAndSet(resetRetryState, false);
1191 if (retryRequested) {
1192 failureCount = 0;
1193 backoffFailures = 0;
1194 approvalWaits = 0;
1195 latestFailure = null;
1196 pendingRetry = Option.none();
1197 }
1198 const currentIntent = yield* Ref.get(intent);
1199 if (!currentIntent.desired) {
1200 resetRetryLadder();
1201 latestFailure = null;
1202 yield* clearLease;
1203 yield* setState(availableState(currentIntent, generation));
1204 yield* waitForSignal;
1205 continue;
1206 }
1207 // The platform's offline report can stay wrong after the network is back,
1208 // so an explicit retry still tries instead of waiting for it to change.
1209 const ignoreOffline = replacing || retryRequested;
1210 if (currentIntent.network === "offline" && !ignoreOffline) {
1211 yield* clearLease;
1212 yield* setState(offlineState(currentIntent, generation, failureCount + 1, latestFailure));
1213 const applicationActivated = yield* waitForSignal;
1214 if (applicationActivated) {
1215 resetRetryLadder();
1216 }
1217 continue;
1218 }
1219
1220 const attempt = failureCount + 1;
1221 const nextGeneration = generation + 1;
1222 const outcome: AttemptOutcome = yield* Effect.acquireUseRelease(
1223 Scope.make(),
1224 (scope) =>
1225 runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline).pipe(
1226 Scope.provide(scope),
1227 Effect.catchCause((cause) =>
1228 logUnexpectedFailure("attempt", cause).pipe(
1229 Effect.as(failureFromCause(target, cause, generation === nextGeneration, false)),
1230 ),
1231 ),
1232 ),
1233 (scope, exit) =>
1234 Scope.close(scope, exit).pipe(
1235 Effect.catchCause((cause) => logUnexpectedFailure("release", cause)),
1236 ),
1237 );
1238 replacing = false;
1239 const failedProbe = yield* Ref.getAndSet(probeUnanswered, false);
1240 if (outcome.established) {
1241 // A session came up, so no earlier failure describes the connection.
1242 // Losing a live session is not a failed connection either, for example
1243 // a socket the OS killed while the app was suspended: its replacement
1244 // reports no failure until it fails itself.
1245 latestFailure = null;
1246 if (outcome.stable) {
1247 resetRetryLadder();
1248 }
1249 }
1250 if (outcome._tag === "Interrupted") {
1251 if (outcome.resetRetry) {
1252 resetRetryLadder();
1253 replacing = true;
1254 }
1255 continue;
1256 }
1257
1258 const attemptSpan: Option.Option<Tracer.Span> = outcome.failure.attemptSpan;
1259 const error: ConnectionAttemptError = outcome.failure.error;
1260 if (!outcome.established) {
1261 latestFailure = error;
1262 }
1263 if (error._tag === "ConnectionBlockedError") {
1264 const blockedIntent = yield* Ref.get(intent);
1265 yield* setState({
1266 desired: blockedIntent.desired,
1267 network: blockedIntent.network,
1268 phase: "blocked",
1269 stage: null,
1270 attempt,
1271 generation,
1272 lastFailure: error,
1273 retryAt: null,
1274 refusedRouteIds: NO_REFUSED_ROUTES,
1275 });
1276 const applicationActivated = yield* waitForSignal;
1277 if (applicationActivated) {
1278 resetRetryLadder();
1279 }
1280 continue;
1281 }
1282
1283 if (failedProbe) {
1284 resetRetryLadder();
1285 yield* setState(connectingState(yield* Ref.get(intent), generation, 1, latestFailure));
1286 continue;
1287 }
1288
1289 // The first loss of a live session retries almost at once; the ladder
1290 // still counts it, so a host that keeps dropping sessions backs off.
1291 const firstLoss = outcome.established && failureCount === 0;
1292 failureCount += 1;
1293 if (error.reason === "approval-pending") approvalWaits += 1;
1294 const awaitingApproval =
1295 error.reason === "approval-pending" && approvalWaits <= APPROVAL_PENDING_FAST_RETRIES;
1296 if (!awaitingApproval) backoffFailures += 1;
1297 const delayMs = awaitingApproval
1298 ? APPROVAL_PENDING_RETRY_DELAY_MS
1299 : firstLoss
1300 ? Math.round(SESSION_LOSS_RETRY_DELAY_MS * (0.5 + (yield* Random.next) / 2))
1301 : error.reason === "approval-pending"
1302 ? Math.max(
1303 APPROVAL_PENDING_RETRY_DELAY_MS,
1304 retryDelayMs(backoffFailures - 1, yield* Random.next),
1305 )
1306 : retryDelayMs(backoffFailures - 1, yield* Random.next);
1307 pendingRetry = Option.map(attemptSpan, (previousAttempt) => ({
1308 previousAttempt,
1309 failureCount,
1310 delayMs,
1311 reason: error.reason,
1312 }));
1313 const failedIntent = yield* Ref.get(intent);
1314 yield* setState({
1315 desired: failedIntent.desired,
1316 network: failedIntent.network,
1317 phase: "backoff",
1318 stage: null,
1319 attempt,
1320 generation,
1321 lastFailure: latestFailure,
1322 retryAt: (yield* Clock.currentTimeMillis) + delayMs,
1323 refusedRouteIds: NO_REFUSED_ROUTES,
1324 });
1325 const applicationActivated = yield* waitForRetrySignal(delayMs);
1326 if (applicationActivated) {
1327 resetRetryLadder();
1328 }
1329 }
1330 });
1331
1332 // Registered before the fibers below so closing the scope interrupts them first.
1333 // A loop still running when the queue shuts down would report that as a failure.
1334 yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease)));
1335 // A defect in a background listener would otherwise end it for good, silently.
1336 // Untraced: a span here would stay open for the supervisor's whole lifetime.
1337 const keepRunning = Effect.fnUntraced(
1338 function* (task: string, effect: Effect.Effect<void>, onRestart?: Effect.Effect<void>) {
1339 let current = effect;
1340 while (true) {
1341 const exit = yield* Effect.exit(current);
1342 if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit;
1343 yield* Effect.logError("Connection supervisor task failed; restarting it.").pipe(
1344 Effect.annotateLogs({
1345 "environment.id": target.environmentId,
1346 "environment.label": target.label,
1347 task,
1348 ...safeErrorLogAttributes(exit.cause.reasons.find(Cause.isDieReason)?.defect),
1349 }),
1350 );
1351 yield* Effect.sleep(RETRY_BASE_DELAY_MS);
1352 current = onRestart ? onRestart.pipe(Effect.andThen(effect)) : effect;
1353 }
1354 },
1355 (self) => Effect.forkScoped(self),
1356 );
1357
1358 const applyNetwork = (network: NetworkStatus) =>
1359 Ref.modify(intent, (current) => {
1360 const next = targetNetwork(network);
1361 return current.network === next
1362 ? [false, current]
1363 : ([true, { ...current, network: next }] as const);
1364 }).pipe(
1365 Effect.flatMap((changed) =>
1366 changed
1367 ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe(
1368 Effect.andThen(signal({ _tag: "NetworkChanged", network })),
1369 )
1370 : Effect.void,
1371 ),
1372 );
1373 // A restarted listener may have missed changes while it was down.
1374 yield* keepRunning(
1375 "network-listener",
1376 connectivity.changes.pipe(Stream.runForEach(applyNetwork)),
1377 connectivity.status.pipe(Effect.flatMap(applyNetwork)),
1378 );
1379 yield* keepRunning(
1380 "wakeup-listener",
1381 wakeups.changes.pipe(
1382 Stream.filter(ConnectionWakeups.reachesConnectionSupervisor),
1383 Stream.runForEach((reason) =>
1384 // A long suspension can hide a network change the platform never reported.
1385 (reason === "network-changed" || reason === "application-active-long"
1386 ? forgetSilentRoutes
1387 : Effect.void
1388 ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))),
1389 ),
1390 ),
1391 );
1392 yield* keepRunning(
1393 "better-route-checker",
1394 Queue.take(betterRouteChecks).pipe(Effect.flatMap(checkBetterRoutes), Effect.forever),
1395 );
1396 // A restarted loop has forgotten its retry ladder, so it waits out the
1397 // ladder's ceiling unless a signal cuts the wait short.
1398 yield* run().pipe(
1399 Effect.catchCause((cause) =>
1400 Effect.gen(function* () {
1401 yield* logUnexpectedFailure("loop", cause);
1402 const currentIntent = yield* Ref.get(intent);
1403 const lastFailure = unexpectedFailure(target).error;
1404 yield* setState(
1405 !currentIntent.desired
1406 ? availableState(currentIntent, generation)
1407 : currentIntent.network === "offline"
1408 ? offlineState(currentIntent, generation, 1, lastFailure)
1409 : {
1410 desired: true,
1411 network: currentIntent.network,
1412 phase: "backoff",
1413 stage: null,
1414 attempt: 1,
1415 generation,
1416 lastFailure,
1417 retryAt: (yield* Clock.currentTimeMillis) + RETRY_MAX_DELAY_MS,
1418 refusedRouteIds: NO_REFUSED_ROUTES,
1419 },
1420 );
1421 yield* waitForRetrySignal(RETRY_MAX_DELAY_MS);
1422 }),
1423 ),
1424 Effect.forever,
1425 Effect.forkScoped,
1426 );
1427
1428 const connect = Ref.update(intent, (current) => ({
1429 ...current,
1430 desired: true,
1431 })).pipe(
1432 Effect.andThen(signal({ _tag: "ConnectRequested" })),
1433 Effect.withSpan("EnvironmentSupervisor.connect"),
1434 );
1435
1436 const disconnect = Ref.update(intent, (current) => ({
1437 ...current,
1438 desired: false,
1439 })).pipe(
1440 Effect.andThen(signal({ _tag: "DisconnectRequested" })),
1441 Effect.withSpan("EnvironmentSupervisor.disconnect"),
1442 );
1443
1444 const retryNow = Ref.set(resetRetryState, true).pipe(
1445 Effect.andThen(signal({ _tag: "RetryRequested" })),
1446 Effect.withSpan("EnvironmentSupervisor.retryNow"),
1447 );
1448
1449 return EnvironmentSupervisor.of({
1450 target,
1451 state,
1452 session,
1453 prepared,
1454 connect,
1455 disconnect,
1456 retryNow,
1457 });
1458 });