The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { msg } from "gt-i18n"; |
| 2 | import { translatableDetail } from "../errors/translatable.ts"; |
| 3 | import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; |
| 4 | import * as Cause from "effect/Cause"; |
| 5 | import * as Clock from "effect/Clock"; |
| 6 | import * as Context from "effect/Context"; |
| 7 | import * as Duration from "effect/Duration"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Exit from "effect/Exit"; |
| 10 | import * as Fiber from "effect/Fiber"; |
| 11 | import * as Option from "effect/Option"; |
| 12 | import * as Queue from "effect/Queue"; |
| 13 | import * as Random from "effect/Random"; |
| 14 | import * as Ref from "effect/Ref"; |
| 15 | import * as Scope from "effect/Scope"; |
| 16 | import * as Stream from "effect/Stream"; |
| 17 | import * as SubscriptionRef from "effect/SubscriptionRef"; |
| 18 | import * as Tracer from "effect/Tracer"; |
| 19 | |
| 20 | import type { ConnectionCatalogEntry, ConnectionRoute } from "./catalog.ts"; |
| 21 | import * as Connectivity from "./connectivity.ts"; |
| 22 | import * as ConnectionDriver from "./driver.ts"; |
| 23 | import { |
| 24 | type ConnectionAttemptError, |
| 25 | type ConnectionTarget, |
| 26 | ConnectionTransientError, |
| 27 | type NetworkStatus, |
| 28 | NO_REFUSED_ROUTES, |
| 29 | type PreparedConnection, |
| 30 | type SupervisorConnectionState, |
| 31 | } from "./model.ts"; |
| 32 | import * as RpcSession from "../rpc/session.ts"; |
| 33 | import { safeErrorLogAttributes } from "../errors/safeLog.ts"; |
| 34 | import { NETWORK_BLOCKING_HINT } from "../errors/network.ts"; |
| 35 | import * as ConnectionWakeups from "./wakeups.ts"; |
| 36 | import { |
| 37 | connectionRouteId, |
| 38 | connectionRouteKind, |
| 39 | connectionRoutes, |
| 40 | type ConnectionRouteKind, |
| 41 | entryWithRoutes, |
| 42 | insertRoute, |
| 43 | type ReportedEndpoint, |
| 44 | } from "./routes.ts"; |
| 45 | |
| 46 | const RETRY_BASE_DELAY_MS = 1_000; |
| 47 | const RETRY_MAX_DELAY_MS = 300_000; |
| 48 | const SESSION_LOSS_RETRY_DELAY_MS = 250; |
| 49 | // Someone is approving this device on the host; check back soon rather than climbing the ladder. |
| 50 | const APPROVAL_PENDING_RETRY_DELAY_MS = 4_000; |
| 51 | // A request lives 10 minutes; past that nobody is approving, so a forgotten device backs off. |
| 52 | const APPROVAL_PENDING_FAST_RETRIES = 150; |
| 53 | // A cold server with a large database can take longer than 15 seconds to |
| 54 | // answer, and that is not a reason to drop a working connection. |
| 55 | const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds"; |
| 56 | const establishmentTimeout = Duration.fromInputUnsafe(CONNECTION_ESTABLISHMENT_TIMEOUT); |
| 57 | const synchronizationTimeout = Duration.fromInputUnsafe("75 seconds"); |
| 58 | const CONNECTION_PROBE_TIMEOUT = "15 seconds"; |
| 59 | const QUICK_CONNECTION_PROBE_TIMEOUT = "3 seconds"; |
| 60 | // After a long suspension pooled connections are often dead too, so the host |
| 61 | // check can hang with the socket. A replacement is cheaper than the wait. |
| 62 | const LONG_RESUME_PROBE_TIMEOUT = "1500 millis"; |
| 63 | // A probe still unanswered this long checks the host with a fresh request. If |
| 64 | // the host answers while the socket stays silent, the socket is dead. |
| 65 | const SILENT_SOCKET_CHECK_AFTER = "500 millis"; |
| 66 | const SILENT_SOCKET_RECHECK_EVERY = "1 second"; |
| 67 | const FRAME_CHECK_EVERY = "100 millis"; |
| 68 | // Off the LAN one lost packet can stall a live socket for a second, so a remote |
| 69 | // socket must stay silent this long, except after a long resume. It must end |
| 70 | // before QUICK_CONNECTION_PROBE_TIMEOUT so a resume replaces a silent socket |
| 71 | // instead of timing out. |
| 72 | const REMOTE_SILENT_SOCKET_AFTER = "2500 millis"; |
| 73 | const BACKOFF_RESET_AFTER_MS = 30_000; |
| 74 | const BETTER_ROUTE_CHECK_INTERVAL_MS = 60_000; |
| 75 | // Each check of a dead route can wait out the full probe timeout, so a route |
| 76 | // that keeps failing is checked half as often each time, until the network |
| 77 | // changes and it may answer again. |
| 78 | const SILENT_ROUTE_BACKOFF_MAX_MS = 1_800_000; |
| 79 | // A better route that answered the check but then failed to connect is not |
| 80 | // tried again for this long, so a flaky LAN cannot bounce the connection. |
| 81 | const BETTER_ROUTE_COOLDOWN_MS = 300_000; |
| 82 | |
| 83 | interface SupervisorIntent { |
| 84 | readonly desired: boolean; |
| 85 | readonly network: NetworkStatus; |
| 86 | } |
| 87 | |
| 88 | type SupervisorSignal = |
| 89 | | { readonly _tag: "ConnectRequested" } |
| 90 | | { readonly _tag: "DisconnectRequested" } |
| 91 | | { readonly _tag: "RetryRequested" } |
| 92 | | { readonly _tag: "NetworkChanged"; readonly network: NetworkStatus } |
| 93 | | { readonly _tag: "Wakeup"; readonly reason: ConnectionWakeups.ConnectionWakeup } |
| 94 | | { readonly _tag: "BetterRouteAvailable"; readonly routeId: string }; |
| 95 | |
| 96 | interface PendingRetryTrace { |
| 97 | readonly previousAttempt: Tracer.Span; |
| 98 | readonly failureCount: number; |
| 99 | readonly delayMs: number; |
| 100 | readonly reason: ConnectionAttemptError["reason"]; |
| 101 | } |
| 102 | |
| 103 | type ConnectedLease = ConnectionDriver.EnvironmentConnectionLease & { |
| 104 | readonly activeRoute: ConnectionRoute | undefined; |
| 105 | readonly generation: number; |
| 106 | }; |
| 107 | |
| 108 | function mergeRefusedRoutes( |
| 109 | known: ReadonlyArray<string>, |
| 110 | refused: ReadonlyArray<string>, |
| 111 | keep: (routeId: string) => boolean, |
| 112 | ): ReadonlyArray<string> { |
| 113 | const next = [...new Set([...known, ...refused])].filter(keep); |
| 114 | return next.length === known.length && next.every((id, index) => id === known[index]) |
| 115 | ? known |
| 116 | : next; |
| 117 | } |
| 118 | |
| 119 | interface TracedAttemptFailure { |
| 120 | readonly error: ConnectionAttemptError; |
| 121 | readonly attemptSpan: Option.Option<Tracer.Span>; |
| 122 | } |
| 123 | |
| 124 | type AttemptOutcome = |
| 125 | | { |
| 126 | readonly _tag: "Interrupted"; |
| 127 | readonly established: boolean; |
| 128 | readonly stable: boolean; |
| 129 | readonly resetRetry: boolean; |
| 130 | } |
| 131 | | { |
| 132 | readonly _tag: "Failure"; |
| 133 | readonly established: boolean; |
| 134 | readonly stable: boolean; |
| 135 | readonly failure: TracedAttemptFailure; |
| 136 | }; |
| 137 | |
| 138 | type EstablishmentEvent = |
| 139 | | { |
| 140 | readonly _tag: "Completed"; |
| 141 | readonly exit: Exit.Exit< |
| 142 | { |
| 143 | readonly attemptSpan: Option.Option<Tracer.Span>; |
| 144 | readonly lease: ConnectionDriver.EnvironmentConnectionLease; |
| 145 | }, |
| 146 | TracedAttemptFailure |
| 147 | >; |
| 148 | } |
| 149 | | { readonly _tag: "Interrupted"; readonly resetRetry: boolean } |
| 150 | | { readonly _tag: "TimedOut" }; |
| 151 | |
| 152 | export interface EnvironmentSupervisorOptions { |
| 153 | readonly initiallyDesired?: boolean; |
| 154 | readonly learnRoutes?: (input: { |
| 155 | readonly activeRoute: ConnectionRoute; |
| 156 | readonly reported: ReadonlyArray<ReportedEndpoint>; |
| 157 | }) => Effect.Effect<Option.Option<ConnectionCatalogEntry>>; |
| 158 | } |
| 159 | |
| 160 | function retryDelayMs(failureCount: number, random: number): number { |
| 161 | const ceiling = Math.min(RETRY_MAX_DELAY_MS, RETRY_BASE_DELAY_MS * 2 ** (failureCount + 1)); |
| 162 | return Math.round(ceiling / 2 + (ceiling / 2) * random); |
| 163 | } |
| 164 | |
| 165 | interface SilentRoute { |
| 166 | readonly failures: number; |
| 167 | readonly until: bigint; |
| 168 | } |
| 169 | |
| 170 | function backOffSilentRoute(previous: SilentRoute | undefined, now: bigint): SilentRoute { |
| 171 | const failures = (previous?.failures ?? 0) + 1; |
| 172 | const delayMs = Math.min( |
| 173 | SILENT_ROUTE_BACKOFF_MAX_MS, |
| 174 | BETTER_ROUTE_CHECK_INTERVAL_MS * 2 ** failures, |
| 175 | ); |
| 176 | return { failures, until: now + BigInt(delayMs) * 1_000_000n }; |
| 177 | } |
| 178 | |
| 179 | function annotateTarget(target: ConnectionTarget) { |
| 180 | return Effect.annotateCurrentSpan({ |
| 181 | "environment.id": target.environmentId, |
| 182 | "environment.label": target.label, |
| 183 | "environment.target.kind": target._tag, |
| 184 | }); |
| 185 | } |
| 186 | |
| 187 | function availableState(intent: SupervisorIntent, generation: number): SupervisorConnectionState { |
| 188 | return { |
| 189 | desired: false, |
| 190 | network: intent.network, |
| 191 | phase: "available", |
| 192 | stage: null, |
| 193 | attempt: 0, |
| 194 | generation, |
| 195 | lastFailure: null, |
| 196 | retryAt: null, |
| 197 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 198 | }; |
| 199 | } |
| 200 | |
| 201 | function offlineState( |
| 202 | intent: SupervisorIntent, |
| 203 | generation: number, |
| 204 | attempt: number, |
| 205 | lastFailure: ConnectionAttemptError | null, |
| 206 | ): SupervisorConnectionState { |
| 207 | return { |
| 208 | desired: true, |
| 209 | network: intent.network, |
| 210 | phase: "offline", |
| 211 | stage: null, |
| 212 | attempt, |
| 213 | generation, |
| 214 | lastFailure, |
| 215 | retryAt: null, |
| 216 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 217 | }; |
| 218 | } |
| 219 | |
| 220 | function connectingState( |
| 221 | intent: SupervisorIntent, |
| 222 | generation: number, |
| 223 | attempt: number, |
| 224 | lastFailure: ConnectionAttemptError | null, |
| 225 | stage: SupervisorConnectionState["stage"] = "preparing", |
| 226 | ): SupervisorConnectionState { |
| 227 | return { |
| 228 | desired: true, |
| 229 | network: intent.network, |
| 230 | phase: "connecting", |
| 231 | stage, |
| 232 | attempt, |
| 233 | generation, |
| 234 | lastFailure, |
| 235 | retryAt: null, |
| 236 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 237 | }; |
| 238 | } |
| 239 | |
| 240 | function failureFromCause( |
| 241 | target: ConnectionTarget, |
| 242 | cause: Cause.Cause<TracedAttemptFailure>, |
| 243 | established: boolean, |
| 244 | stable: boolean, |
| 245 | ) { |
| 246 | return { |
| 247 | _tag: "Failure", |
| 248 | established, |
| 249 | stable, |
| 250 | failure: cause.reasons.find(Cause.isFailReason)?.error ?? unexpectedFailure(target), |
| 251 | } satisfies AttemptOutcome; |
| 252 | } |
| 253 | |
| 254 | function unexpectedFailure(target: ConnectionTarget): TracedAttemptFailure { |
| 255 | return { |
| 256 | error: new ConnectionTransientError({ |
| 257 | reason: "transport", |
| 258 | ...translatableDetail(msg("{host} connection failed unexpectedly.", { host: target.label })), |
| 259 | }), |
| 260 | attemptSpan: Option.none(), |
| 261 | }; |
| 262 | } |
| 263 | |
| 264 | export class EnvironmentSupervisor extends Context.Service< |
| 265 | EnvironmentSupervisor, |
| 266 | { |
| 267 | readonly target: ConnectionTarget; |
| 268 | readonly state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>; |
| 269 | readonly session: SubscriptionRef.SubscriptionRef<Option.Option<RpcSession.RpcSession>>; |
| 270 | readonly prepared: SubscriptionRef.SubscriptionRef<Option.Option<PreparedConnection>>; |
| 271 | readonly connect: Effect.Effect<void>; |
| 272 | readonly disconnect: Effect.Effect<void>; |
| 273 | readonly retryNow: Effect.Effect<void>; |
| 274 | } |
| 275 | >()("@t3tools/client-runtime/connection/supervisor/EnvironmentSupervisor") {} |
| 276 | |
| 277 | export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( |
| 278 | entry: ConnectionCatalogEntry, |
| 279 | options?: EnvironmentSupervisorOptions, |
| 280 | ): Effect.fn.Return< |
| 281 | EnvironmentSupervisor["Service"], |
| 282 | never, |
| 283 | | Connectivity.Connectivity |
| 284 | | ConnectionDriver.ConnectionDriver |
| 285 | | Scope.Scope |
| 286 | | ConnectionWakeups.ConnectionWakeups |
| 287 | > { |
| 288 | const target = entry.target; |
| 289 | const usesRelay = connectionRoutes(entry).some( |
| 290 | (route) => route.target._tag === "RelayConnectionTarget", |
| 291 | ); |
| 292 | const setupTimeout = msg("{host} did not respond during connection setup.", { |
| 293 | host: target.label, |
| 294 | }); |
| 295 | const setupTimeoutDetail = usesRelay |
| 296 | ? translatableDetail(setupTimeout, NETWORK_BLOCKING_HINT) |
| 297 | : translatableDetail(setupTimeout); |
| 298 | yield* annotateTarget(target); |
| 299 | |
| 300 | const connectivity = yield* Connectivity.Connectivity; |
| 301 | const driver = yield* ConnectionDriver.ConnectionDriver; |
| 302 | const wakeups = yield* ConnectionWakeups.ConnectionWakeups; |
| 303 | // The primary environment served this client or runs beside it on loopback, |
| 304 | // so the device reporting offline (often wrongly) never parks it. The report |
| 305 | // still probes its live session, like any other network change. |
| 306 | const targetNetwork = (network: NetworkStatus): NetworkStatus => |
| 307 | network === "offline" && target._tag === "PrimaryConnectionTarget" ? "unknown" : network; |
| 308 | const initialIntent: SupervisorIntent = { |
| 309 | desired: options?.initiallyDesired ?? false, |
| 310 | network: targetNetwork(yield* connectivity.status), |
| 311 | }; |
| 312 | const intent = yield* Ref.make(initialIntent); |
| 313 | const signals = yield* Queue.unbounded<SupervisorSignal>(); |
| 314 | const resetRetryState = yield* Ref.make(false); |
| 315 | const probeUnanswered = yield* Ref.make(false); |
| 316 | const state = yield* SubscriptionRef.make<SupervisorConnectionState>( |
| 317 | !initialIntent.desired |
| 318 | ? availableState(initialIntent, 0) |
| 319 | : initialIntent.network === "offline" |
| 320 | ? offlineState(initialIntent, 0, 0, null) |
| 321 | : connectingState(initialIntent, 0, 1, null), |
| 322 | ); |
| 323 | const session = yield* SubscriptionRef.make<Option.Option<RpcSession.RpcSession>>(Option.none()); |
| 324 | const prepared = yield* SubscriptionRef.make<Option.Option<PreparedConnection>>(Option.none()); |
| 325 | // Learned routes arrive while a session is live, so the route list is read |
| 326 | // fresh by each attempt and check instead of being fixed at creation. |
| 327 | const currentEntry = yield* Ref.make(entry); |
| 328 | const currentRoutes = Ref.get(currentEntry).pipe(Effect.map(connectionRoutes)); |
| 329 | const preferredRouteId = yield* Ref.make(Option.none<string>()); |
| 330 | const routeCooldowns = yield* Ref.make<ReadonlyMap<string, bigint>>(new Map()); |
| 331 | // Outlives sessions so a reconnect shows a refusal seen earlier; a route leaves once it connects. |
| 332 | const refusedRoutes = yield* Ref.make(NO_REFUSED_ROUTES); |
| 333 | // The generation keeps a check that started before a reset from recording |
| 334 | // its stale failures after it. |
| 335 | const silentRoutes = yield* Ref.make({ |
| 336 | generation: 0, |
| 337 | routes: new Map<string, SilentRoute>() as ReadonlyMap<string, SilentRoute>, |
| 338 | }); |
| 339 | const forgetSilentRoutes = Ref.update(silentRoutes, ({ generation }) => ({ |
| 340 | generation: generation + 1, |
| 341 | routes: new Map(), |
| 342 | })); |
| 343 | |
| 344 | const attemptEntry = Effect.gen(function* () { |
| 345 | const latest = yield* Ref.get(currentEntry); |
| 346 | const routes = connectionRoutes(latest); |
| 347 | const preferred = yield* Ref.getAndSet(preferredRouteId, Option.none()); |
| 348 | if (Option.isNone(preferred)) return latest; |
| 349 | const route = routes.find( |
| 350 | (candidate) => connectionRouteId(candidate.target) === preferred.value, |
| 351 | ); |
| 352 | return route === undefined |
| 353 | ? latest |
| 354 | : entryWithRoutes(latest, [route, ...routes.filter((candidate) => candidate !== route)]); |
| 355 | }); |
| 356 | |
| 357 | const learnRoutesFrom = Effect.fnUntraced(function* ( |
| 358 | learn: NonNullable<EnvironmentSupervisorOptions["learnRoutes"]>, |
| 359 | lease: ConnectedLease, |
| 360 | ) { |
| 361 | const activeRoute = lease.activeRoute; |
| 362 | if (activeRoute === undefined) return; |
| 363 | yield* lease.session.subscribeServerConfig().pipe( |
| 364 | Stream.filter((event) => event.type === "snapshot"), |
| 365 | Stream.map((event) => event.config.directEndpoints), |
| 366 | Stream.filter((reported) => reported !== undefined), |
| 367 | Stream.changesWith((previous, next) => JSON.stringify(previous) === JSON.stringify(next)), |
| 368 | Stream.runForEach((reported) => |
| 369 | Effect.gen(function* () { |
| 370 | const updated = yield* learn({ activeRoute, reported }); |
| 371 | if (Option.isSome(updated)) { |
| 372 | yield* Ref.set(currentEntry, updated.value); |
| 373 | yield* requestBetterRouteCheck(lease); |
| 374 | } |
| 375 | }), |
| 376 | ), |
| 377 | Effect.ignoreCause({ log: true }), |
| 378 | ); |
| 379 | }); |
| 380 | |
| 381 | const routeIndex = (lease: ConnectedLease) => |
| 382 | currentRoutes.pipe( |
| 383 | Effect.map((routes) => { |
| 384 | const activeId = connectionRouteId(lease.prepared.target); |
| 385 | const index = routes.findIndex((route) => connectionRouteId(route.target) === activeId); |
| 386 | return index === -1 && lease.activeRoute !== undefined |
| 387 | ? insertRoute(routes, lease.activeRoute).findIndex( |
| 388 | (route) => connectionRouteId(route.target) === activeId, |
| 389 | ) |
| 390 | : index; |
| 391 | }), |
| 392 | ); |
| 393 | |
| 394 | const checkBetterRoutes = Effect.fnUntraced(function* (lease: ConnectedLease) { |
| 395 | const latest = yield* Ref.get(currentEntry); |
| 396 | const current = yield* routeIndex(lease); |
| 397 | if (current <= 0) return; |
| 398 | const now = yield* Clock.monotonicTimeNanos; |
| 399 | const cooldowns = yield* Ref.get(routeCooldowns); |
| 400 | const silent = yield* Ref.get(silentRoutes); |
| 401 | const better = connectionRoutes(latest) |
| 402 | .slice(0, current) |
| 403 | .filter((route) => { |
| 404 | const id = connectionRouteId(route.target); |
| 405 | return (cooldowns.get(id) ?? 0n) <= now && (silent.routes.get(id)?.until ?? 0n) <= now; |
| 406 | }); |
| 407 | const results = yield* Effect.forEach( |
| 408 | better, |
| 409 | (route: ConnectionRoute) => driver.preflight(latest, route), |
| 410 | { concurrency: "unbounded" }, |
| 411 | ); |
| 412 | yield* Ref.update(silentRoutes, (previous) => { |
| 413 | if (previous.generation !== silent.generation) return previous; |
| 414 | const routes = new Map(previous.routes); |
| 415 | for (const [position, route] of better.entries()) { |
| 416 | const id = connectionRouteId(route.target); |
| 417 | if (results[position] !== "silent") routes.delete(id); |
| 418 | else routes.set(id, backOffSilentRoute(previous.routes.get(id), now)); |
| 419 | } |
| 420 | return { generation: previous.generation, routes }; |
| 421 | }); |
| 422 | const live = yield* SubscriptionRef.get(session); |
| 423 | if (Option.isNone(live) || live.value !== lease.session) return; |
| 424 | const refused = better |
| 425 | .filter((_route, position) => results[position] === "refused") |
| 426 | .map((route) => connectionRouteId(route.target)); |
| 427 | if (refused.length > 0) { |
| 428 | const known = yield* Ref.updateAndGet(refusedRoutes, (current) => |
| 429 | mergeRefusedRoutes(current, refused, () => true), |
| 430 | ); |
| 431 | yield* SubscriptionRef.updateSome(state, (current) => |
| 432 | current.phase === "connected" && |
| 433 | current.generation === lease.generation && |
| 434 | current.refusedRouteIds !== known |
| 435 | ? Option.some({ ...current, refusedRouteIds: known }) |
| 436 | : Option.none(), |
| 437 | ); |
| 438 | } |
| 439 | const index = results.indexOf("ready"); |
| 440 | if (index === -1) return; |
| 441 | yield* signal({ |
| 442 | _tag: "BetterRouteAvailable", |
| 443 | routeId: connectionRouteId(better[index]!.target), |
| 444 | }); |
| 445 | }); |
| 446 | |
| 447 | const betterRouteChecks = yield* Queue.dropping<ConnectedLease>(1); |
| 448 | const requestBetterRouteCheck = (lease: ConnectedLease) => |
| 449 | routeIndex(lease).pipe( |
| 450 | Effect.flatMap((index) => |
| 451 | index > 0 ? Queue.offer(betterRouteChecks, lease).pipe(Effect.asVoid) : Effect.void, |
| 452 | ), |
| 453 | ); |
| 454 | |
| 455 | const clearLease = Effect.all( |
| 456 | [SubscriptionRef.set(session, Option.none()), SubscriptionRef.set(prepared, Option.none())], |
| 457 | { discard: true }, |
| 458 | ); |
| 459 | |
| 460 | const setState = Effect.fn("EnvironmentSupervisor.setState")(function* ( |
| 461 | next: SupervisorConnectionState, |
| 462 | ) { |
| 463 | yield* SubscriptionRef.set(state, next); |
| 464 | }); |
| 465 | |
| 466 | const signal = Effect.fn("EnvironmentSupervisor.signal")(function* (next: SupervisorSignal) { |
| 467 | yield* Queue.offer(signals, next); |
| 468 | }); |
| 469 | |
| 470 | const logManagedRelayAccountChange = Effect.logInfo( |
| 471 | "Managed relay account changed; restarting the environment connection.", |
| 472 | ).pipe( |
| 473 | Effect.annotateLogs({ |
| 474 | "environment.id": target.environmentId, |
| 475 | "environment.label": target.label, |
| 476 | }), |
| 477 | ); |
| 478 | |
| 479 | const logUnexpectedFailure = ( |
| 480 | failed: "attempt" | "release" | "loop", |
| 481 | cause: Cause.Cause<TracedAttemptFailure>, |
| 482 | ) => { |
| 483 | const died = cause.reasons.find(Cause.isDieReason); |
| 484 | if (!died && cause.reasons.some(Cause.isFailReason)) return Effect.void; |
| 485 | return Effect.logError( |
| 486 | `Connection ${failed} failed with an unexpected ${died ? "defect" : "interruption"}.`, |
| 487 | ).pipe( |
| 488 | Effect.annotateLogs({ |
| 489 | "environment.id": target.environmentId, |
| 490 | "environment.label": target.label, |
| 491 | "cause.reason_count": cause.reasons.length, |
| 492 | ...(died ? safeErrorLogAttributes(died.defect) : {}), |
| 493 | }), |
| 494 | ); |
| 495 | }; |
| 496 | |
| 497 | const reportProgress = Effect.fn("EnvironmentSupervisor.reportProgress")(function* ( |
| 498 | attempt: number, |
| 499 | nextGeneration: number, |
| 500 | lastFailure: ConnectionAttemptError | null, |
| 501 | progress: ConnectionDriver.ConnectionDriverProgress, |
| 502 | ) { |
| 503 | if ("prepared" in progress) { |
| 504 | yield* SubscriptionRef.set(prepared, Option.some(progress.prepared)); |
| 505 | } |
| 506 | yield* setState( |
| 507 | connectingState(yield* Ref.get(intent), nextGeneration, attempt, lastFailure, progress.stage), |
| 508 | ); |
| 509 | }); |
| 510 | |
| 511 | const establishConnection = Effect.fnUntraced(function* ( |
| 512 | attempt: number, |
| 513 | nextGeneration: number, |
| 514 | lastFailure: ConnectionAttemptError | null, |
| 515 | synchronizedRoutes: Ref.Ref<number>, |
| 516 | ) { |
| 517 | return yield* driver.connect(yield* attemptEntry, (progress) => |
| 518 | (progress.stage === "synchronizing" |
| 519 | ? Ref.update(synchronizedRoutes, (count) => count + 1) |
| 520 | : Effect.void |
| 521 | ).pipe(Effect.andThen(reportProgress(attempt, nextGeneration, lastFailure, progress))), |
| 522 | ); |
| 523 | }); |
| 524 | |
| 525 | // Low-cardinality dimensions for time-to-connected dashboards; the error |
| 526 | // detail stays in the span status, never in an attribute. |
| 527 | const annotateAttemptOutcome = Effect.fnUntraced(function* ( |
| 528 | exit: Exit.Exit<ConnectionDriver.EnvironmentConnectionLease, ConnectionAttemptError>, |
| 529 | ) { |
| 530 | if (Exit.isSuccess(exit)) { |
| 531 | const leaseTarget = exit.value.prepared.target; |
| 532 | const route = (yield* currentRoutes).find( |
| 533 | (candidate) => connectionRouteId(candidate.target) === connectionRouteId(leaseTarget), |
| 534 | ); |
| 535 | return yield* Effect.annotateCurrentSpan({ |
| 536 | "connection.outcome": "connected", |
| 537 | "connection.route": connectionRouteKind( |
| 538 | route ?? { target: leaseTarget, profile: Option.none() }, |
| 539 | ), |
| 540 | }); |
| 541 | } |
| 542 | if (Cause.hasInterruptsOnly(exit.cause)) { |
| 543 | return yield* Effect.annotateCurrentSpan("connection.outcome", "interrupted"); |
| 544 | } |
| 545 | const failure = exit.cause.reasons.find(Cause.isFailReason)?.error; |
| 546 | yield* Effect.annotateCurrentSpan({ |
| 547 | "connection.outcome": "failed", |
| 548 | "connection.failure.tag": failure?._tag ?? "defect", |
| 549 | "connection.failure.reason": failure?.reason ?? "defect", |
| 550 | }); |
| 551 | }); |
| 552 | |
| 553 | const traceRelayEstablishment = ( |
| 554 | effect: Effect.Effect< |
| 555 | ConnectionDriver.EnvironmentConnectionLease, |
| 556 | ConnectionAttemptError, |
| 557 | Scope.Scope |
| 558 | >, |
| 559 | attempt: number, |
| 560 | nextGeneration: number, |
| 561 | pendingRetry: Option.Option<PendingRetryTrace>, |
| 562 | ) => { |
| 563 | // The span must end with the ConnectionAttemptError itself: the exporter |
| 564 | // can only describe an Error-like failure, and a TracedAttemptFailure |
| 565 | // wrapper reached it as "[object Object]". The wrapper is added outside. |
| 566 | let attemptSpan = Option.none<Tracer.Span>(); |
| 567 | const traced = Effect.gen(function* () { |
| 568 | attemptSpan = Option.some(yield* Effect.currentSpan.pipe(Effect.orDie)); |
| 569 | yield* Effect.annotateCurrentSpan({ |
| 570 | "environment.id": target.environmentId, |
| 571 | "environment.target.kind": target._tag, |
| 572 | "connection.attempt": attempt, |
| 573 | "connection.generation": nextGeneration, |
| 574 | "connection.retry.failure_count": Option.match(pendingRetry, { |
| 575 | onNone: () => 0, |
| 576 | onSome: (retry) => retry.failureCount, |
| 577 | }), |
| 578 | }); |
| 579 | return yield* effect.pipe(Effect.onExit(annotateAttemptOutcome)); |
| 580 | }).pipe(Effect.withSpan("relay.connection.attempt", { root: true })); |
| 581 | |
| 582 | return Option.match(pendingRetry, { |
| 583 | onNone: () => traced, |
| 584 | onSome: (retry) => |
| 585 | traced.pipe( |
| 586 | Effect.linkSpans(retry.previousAttempt, { |
| 587 | "connection.retry.delay_ms": retry.delayMs, |
| 588 | "connection.retry.reason": retry.reason, |
| 589 | }), |
| 590 | ), |
| 591 | }).pipe( |
| 592 | Effect.map((lease) => ({ attemptSpan, lease })), |
| 593 | Effect.mapError((error): TracedAttemptFailure => ({ error, attemptSpan })), |
| 594 | withRelayClientTracing, |
| 595 | ); |
| 596 | }; |
| 597 | |
| 598 | const establishTracedConnection = Effect.fnUntraced(function* ( |
| 599 | attempt: number, |
| 600 | nextGeneration: number, |
| 601 | lastFailure: ConnectionAttemptError | null, |
| 602 | pendingRetry: Option.Option<PendingRetryTrace>, |
| 603 | synchronizedRoutes: Ref.Ref<number>, |
| 604 | ) { |
| 605 | if (usesRelay) { |
| 606 | return yield* traceRelayEstablishment( |
| 607 | establishConnection(attempt, nextGeneration, lastFailure, synchronizedRoutes), |
| 608 | attempt, |
| 609 | nextGeneration, |
| 610 | pendingRetry, |
| 611 | ); |
| 612 | } |
| 613 | return yield* establishConnection( |
| 614 | attempt, |
| 615 | nextGeneration, |
| 616 | lastFailure, |
| 617 | synchronizedRoutes, |
| 618 | ).pipe( |
| 619 | Effect.map((lease) => ({ |
| 620 | attemptSpan: Option.none<Tracer.Span>(), |
| 621 | lease, |
| 622 | })), |
| 623 | Effect.mapError((error): TracedAttemptFailure => ({ |
| 624 | error, |
| 625 | attemptSpan: Option.none(), |
| 626 | })), |
| 627 | ); |
| 628 | }); |
| 629 | |
| 630 | const waitForEstablishmentInterrupt = Effect.fnUntraced(function* (ignoreOffline: boolean) { |
| 631 | // A primary attempt keeps running through an offline report, so the |
| 632 | // network returning restarts it in case it stalled meanwhile. |
| 633 | let deviceOffline = (yield* connectivity.status) === "offline"; |
| 634 | for (;;) { |
| 635 | const next = yield* Queue.take(signals); |
| 636 | switch (next._tag) { |
| 637 | case "DisconnectRequested": |
| 638 | return false; |
| 639 | case "RetryRequested": |
| 640 | // Once the loop has taken this retry's flag, the running attempt is |
| 641 | // the one the retry asked for. |
| 642 | if (yield* Ref.get(resetRetryState)) { |
| 643 | return false; |
| 644 | } |
| 645 | break; |
| 646 | case "NetworkChanged": |
| 647 | if (target._tag !== "PrimaryConnectionTarget") { |
| 648 | if (next.network === "offline" && !ignoreOffline) { |
| 649 | return false; |
| 650 | } |
| 651 | } else if (next.network === "offline") { |
| 652 | deviceOffline = true; |
| 653 | } else if (deviceOffline) { |
| 654 | return true; |
| 655 | } |
| 656 | break; |
| 657 | case "ConnectRequested": |
| 658 | case "BetterRouteAvailable": |
| 659 | break; |
| 660 | case "Wakeup": |
| 661 | // An attempt that ran into a long suspension is likely stuck. |
| 662 | if (next.reason === "application-active-long") { |
| 663 | return true; |
| 664 | } |
| 665 | if (next.reason === "credentials-changed" && usesRelay) { |
| 666 | yield* logManagedRelayAccountChange; |
| 667 | return false; |
| 668 | } |
| 669 | break; |
| 670 | } |
| 671 | } |
| 672 | }); |
| 673 | |
| 674 | const isRelayLease = (lease: ConnectionDriver.EnvironmentConnectionLease) => |
| 675 | lease.prepared.target._tag === "RelayConnectionTarget"; |
| 676 | |
| 677 | const connectedLeaseEnd = Effect.fnUntraced(function* ( |
| 678 | next: SupervisorSignal, |
| 679 | lease: ConnectionDriver.EnvironmentConnectionLease, |
| 680 | ) { |
| 681 | if (next._tag === "DisconnectRequested") { |
| 682 | return "end" as const; |
| 683 | } |
| 684 | if (next._tag === "BetterRouteAvailable") { |
| 685 | yield* Ref.set(preferredRouteId, Option.some(next.routeId)); |
| 686 | return "reset" as const; |
| 687 | } |
| 688 | // Only a session over T3 Connect holds the old account's credential. |
| 689 | if (next._tag === "Wakeup" && next.reason === "credentials-changed" && isRelayLease(lease)) { |
| 690 | yield* logManagedRelayAccountChange; |
| 691 | return "end" as const; |
| 692 | } |
| 693 | return undefined; |
| 694 | }); |
| 695 | |
| 696 | const leaseRouteKind = (lease: ConnectionDriver.EnvironmentConnectionLease) => |
| 697 | currentRoutes.pipe( |
| 698 | Effect.map((routes) => { |
| 699 | const route = routes.find( |
| 700 | (candidate) => |
| 701 | connectionRouteId(candidate.target) === connectionRouteId(lease.prepared.target), |
| 702 | ); |
| 703 | return route === undefined ? undefined : connectionRouteKind(route); |
| 704 | }), |
| 705 | ); |
| 706 | |
| 707 | const isLongResume = (signal: SupervisorSignal) => |
| 708 | signal._tag === "Wakeup" && signal.reason === "application-active-long"; |
| 709 | |
| 710 | const answersQuickly = (routeKind: ConnectionRouteKind | undefined) => |
| 711 | routeKind === "loopback" || routeKind === "lan"; |
| 712 | |
| 713 | const probeTimeoutFor = ( |
| 714 | next: SupervisorSignal, |
| 715 | routeKind: ConnectionRouteKind | undefined, |
| 716 | ): Duration.Input | undefined => { |
| 717 | switch (next._tag) { |
| 718 | case "RetryRequested": |
| 719 | return QUICK_CONNECTION_PROBE_TIMEOUT; |
| 720 | case "NetworkChanged": |
| 721 | return next.network === "offline" ? QUICK_CONNECTION_PROBE_TIMEOUT : undefined; |
| 722 | case "Wakeup": |
| 723 | if (next.reason === "application-active") { |
| 724 | return CONNECTION_PROBE_TIMEOUT; |
| 725 | } |
| 726 | // After a long suspension the data on screen is stale, so a slow |
| 727 | // answer reconnects as quickly as a missing one, even on a remote |
| 728 | // route where a reconnect takes longer than the wait it replaces. |
| 729 | if (next.reason === "application-active-long") { |
| 730 | return LONG_RESUME_PROBE_TIMEOUT; |
| 731 | } |
| 732 | if (next.reason !== "application-active-probe" && next.reason !== "network-changed") { |
| 733 | return undefined; |
| 734 | } |
| 735 | // Only loopback and LAN answer within 3 seconds reliably. A tailnet |
| 736 | // over DERP, a VPN, or the relay can spike past it on a live socket; |
| 737 | // a dead one is still caught early by the silent socket check. |
| 738 | return answersQuickly(routeKind) |
| 739 | ? QUICK_CONNECTION_PROBE_TIMEOUT |
| 740 | : CONNECTION_PROBE_TIMEOUT; |
| 741 | case "ConnectRequested": |
| 742 | case "DisconnectRequested": |
| 743 | case "BetterRouteAvailable": |
| 744 | return undefined; |
| 745 | } |
| 746 | }; |
| 747 | |
| 748 | const isForegroundProbe = (next: SupervisorSignal) => |
| 749 | next._tag === "Wakeup" && next.reason === "application-active"; |
| 750 | |
| 751 | // Ends "silent" when the host answers a fresh request while the socket |
| 752 | // delivers nothing, as when iOS kills a suspended socket without a close |
| 753 | // event, and "alive" once the socket delivers a frame. A slow or |
| 754 | // unreachable host never ends it. |
| 755 | const awaitSilentSocket = Effect.fn("EnvironmentSupervisor.awaitSilentSocket")(function* ( |
| 756 | lease: ConnectedLease, |
| 757 | routeKind: ConnectionRouteKind | undefined, |
| 758 | longResume: boolean, |
| 759 | ) { |
| 760 | const startedAt = yield* Clock.monotonicTimeNanos; |
| 761 | yield* Effect.sleep(SILENT_SOCKET_CHECK_AFTER); |
| 762 | const configLoaded = yield* lease.session.initialConfig.pipe( |
| 763 | Effect.as(true), |
| 764 | Effect.catchCause(() => Effect.succeed(false)), |
| 765 | ); |
| 766 | if (!configLoaded) return yield* Effect.never; |
| 767 | // Frames that queued while the app was suspended have drained by now, so |
| 768 | // a later frame shows the path still works. |
| 769 | const framesBefore = yield* lease.session.inboundFrames; |
| 770 | const socketSilent = lease.session.inboundFrames.pipe( |
| 771 | Effect.map((frames) => frames === framesBefore), |
| 772 | ); |
| 773 | // The session only counts frames, so a host check slower than the probe |
| 774 | // deadline would otherwise hide a frame until after the socket is replaced. |
| 775 | const frameArrives = Effect.sleep(FRAME_CHECK_EVERY).pipe( |
| 776 | Effect.andThen(socketSilent), |
| 777 | Effect.repeat({ while: (silent) => silent }), |
| 778 | Effect.as("alive" as const), |
| 779 | ); |
| 780 | const hostAnswersWhileSilent = Effect.gen(function* () { |
| 781 | // A radio still waking up or a stale pooled connection can fail one check. |
| 782 | while ((yield* driver.checkHost(lease.prepared)) !== "answered") { |
| 783 | yield* Effect.sleep(SILENT_SOCKET_RECHECK_EVERY); |
| 784 | } |
| 785 | if (!answersQuickly(routeKind) && !longResume) { |
| 786 | const silentFor = (yield* Clock.monotonicTimeNanos) - startedAt; |
| 787 | const remaining = Duration.toNanosUnsafe(REMOTE_SILENT_SOCKET_AFTER) - silentFor; |
| 788 | if (remaining > 0n) yield* Effect.sleep(Duration.nanos(remaining)); |
| 789 | } |
| 790 | return (yield* socketSilent) ? ("silent" as const) : ("alive" as const); |
| 791 | }); |
| 792 | return yield* Effect.race(hostAnswersWhileSilent, frameArrives); |
| 793 | }); |
| 794 | |
| 795 | // Returning to the app, an explicit retry, and the network reporting offline |
| 796 | // all probe the live session instead of replacing it, so a healthy socket is |
| 797 | // not torn down (the offline report is often wrong, for example for a |
| 798 | // loopback server). |
| 799 | const monitorConnectedLease = Effect.fnUntraced(function* (lease: ConnectedLease) { |
| 800 | // A probe answers an explicit retry here, so the retry must not also reset |
| 801 | // the backoff of a later, unrelated failure. |
| 802 | const takeSignal = Queue.take(signals).pipe( |
| 803 | Effect.tap((next) => |
| 804 | next._tag === "RetryRequested" ? Ref.set(resetRetryState, false) : Effect.void, |
| 805 | ), |
| 806 | ); |
| 807 | // Ticks always run: the check skips the preferred route itself, and the |
| 808 | // route list can grow while connected (learned routes). |
| 809 | yield* Stream.tick(Duration.millis(BETTER_ROUTE_CHECK_INTERVAL_MS)).pipe( |
| 810 | Stream.drop(1), |
| 811 | Stream.runForEach(() => requestBetterRouteCheck(lease)), |
| 812 | Effect.forkScoped, |
| 813 | ); |
| 814 | const routeKind = yield* leaseRouteKind(lease); |
| 815 | for (;;) { |
| 816 | const next = yield* takeSignal; |
| 817 | const end = yield* connectedLeaseEnd(next, lease); |
| 818 | if (end !== undefined) { |
| 819 | return end === "reset"; |
| 820 | } |
| 821 | if ( |
| 822 | (next._tag === "NetworkChanged" && next.network !== "offline") || |
| 823 | (next._tag === "Wakeup" && ConnectionWakeups.resetsRetryBackoff(next.reason)) |
| 824 | ) { |
| 825 | yield* requestBetterRouteCheck(lease); |
| 826 | } |
| 827 | const probeTimeout = probeTimeoutFor(next, routeKind); |
| 828 | if (probeTimeout === undefined) { |
| 829 | continue; |
| 830 | } |
| 831 | yield* Ref.set(probeUnanswered, true); |
| 832 | let probe = yield* Effect.forkChild(lease.session.probe); |
| 833 | // A busy server can miss one foreground probe while the socket is still |
| 834 | // healthy, so the first miss asks again and only the follow-up's miss |
| 835 | // replaces the session. |
| 836 | let toleratesMiss = isForegroundProbe(next); |
| 837 | let silentSocket: Fiber.Fiber<"silent" | "alive"> | undefined; |
| 838 | // A signal during a running check, such as a network change that kills |
| 839 | // the socket, needs a fresh check only if the running one sees a frame. |
| 840 | let recheck = false; |
| 841 | let longResume = isLongResume(next); |
| 842 | const startSilentSocketCheck = Effect.gen(function* () { |
| 843 | silentSocket = yield* Effect.forkChild(awaitSilentSocket(lease, routeKind, longResume)); |
| 844 | recheck = false; |
| 845 | }); |
| 846 | // The web and desktop foreground probe allows a busy host 15 seconds, |
| 847 | // and a busy host can answer a fresh request before a queued probe. |
| 848 | const armSilentSocketCheck = Effect.fnUntraced(function* (signal: SupervisorSignal) { |
| 849 | if (isForegroundProbe(signal)) return; |
| 850 | if (silentSocket !== undefined) { |
| 851 | recheck = true; |
| 852 | return; |
| 853 | } |
| 854 | yield* startSilentSocketCheck; |
| 855 | }); |
| 856 | yield* armSilentSocketCheck(next); |
| 857 | const stopProbe = Effect.suspend(() => |
| 858 | Effect.all( |
| 859 | [Fiber.interrupt(probe), silentSocket ? Fiber.interrupt(silentSocket) : Effect.void], |
| 860 | { discard: true }, |
| 861 | ), |
| 862 | ); |
| 863 | let longResumeStartedAt = yield* Clock.monotonicTimeNanos; |
| 864 | let deadline = longResumeStartedAt + Duration.toNanosUnsafe(probeTimeout); |
| 865 | for (;;) { |
| 866 | const remaining = deadline - (yield* Clock.monotonicTimeNanos); |
| 867 | const probeEvent = yield* Effect.raceAllFirst([ |
| 868 | Fiber.await(probe).pipe( |
| 869 | Effect.map((exit) => ({ _tag: "ProbeCompleted" as const, exit })), |
| 870 | ), |
| 871 | takeSignal.pipe(Effect.map((signal) => ({ _tag: "Signal" as const, signal }))), |
| 872 | Effect.sleep(Duration.nanos(remaining > 0n ? remaining : 0n)).pipe( |
| 873 | Effect.as({ _tag: "TimedOut" as const }), |
| 874 | ), |
| 875 | (silentSocket ? Fiber.join(silentSocket) : Effect.never).pipe( |
| 876 | Effect.map((check) => |
| 877 | check === "silent" |
| 878 | ? { _tag: "SocketSilent" as const } |
| 879 | : { _tag: "SocketAlive" as const }, |
| 880 | ), |
| 881 | ), |
| 882 | ]); |
| 883 | if (probeEvent._tag === "SocketAlive") { |
| 884 | silentSocket = undefined; |
| 885 | // A frame shows the socket still works, so its probe gets the usual resume time. |
| 886 | if (longResume) { |
| 887 | const aliveDeadline = |
| 888 | longResumeStartedAt + Duration.toNanosUnsafe(QUICK_CONNECTION_PROBE_TIMEOUT); |
| 889 | if (aliveDeadline > deadline) deadline = aliveDeadline; |
| 890 | } |
| 891 | if (recheck) yield* startSilentSocketCheck; |
| 892 | continue; |
| 893 | } |
| 894 | if (probeEvent._tag === "SocketSilent") { |
| 895 | // The host is reachable, so replace the socket at once, even while |
| 896 | // the network reports offline. |
| 897 | yield* stopProbe; |
| 898 | return true; |
| 899 | } |
| 900 | if (probeEvent._tag === "TimedOut") { |
| 901 | if (toleratesMiss) { |
| 902 | toleratesMiss = false; |
| 903 | yield* Fiber.interrupt(probe); |
| 904 | probe = yield* Effect.forkChild(lease.session.probe); |
| 905 | deadline = (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(probeTimeout); |
| 906 | continue; |
| 907 | } |
| 908 | yield* stopProbe; |
| 909 | // A long suspension leaves the socket suspect even when the network |
| 910 | // reports offline, so it is replaced like a silent one. |
| 911 | if (longResume) return true; |
| 912 | return yield* new ConnectionTransientError({ |
| 913 | reason: "timeout", |
| 914 | ...translatableDetail( |
| 915 | msg("{host} did not respond to a connection health check.", { |
| 916 | host: target.label, |
| 917 | }), |
| 918 | ), |
| 919 | }); |
| 920 | } |
| 921 | if (probeEvent._tag === "ProbeCompleted") { |
| 922 | if (silentSocket) yield* Fiber.interrupt(silentSocket); |
| 923 | if (Exit.isSuccess(probeEvent.exit)) { |
| 924 | yield* Ref.set(probeUnanswered, false); |
| 925 | } |
| 926 | yield* probeEvent.exit; |
| 927 | break; |
| 928 | } |
| 929 | const endDuringProbe = yield* connectedLeaseEnd(probeEvent.signal, lease); |
| 930 | if (endDuringProbe !== undefined) { |
| 931 | yield* stopProbe; |
| 932 | return endDuringProbe === "reset"; |
| 933 | } |
| 934 | const signalTimeout = probeTimeoutFor(probeEvent.signal, routeKind); |
| 935 | if (signalTimeout !== undefined) { |
| 936 | // A retry or network signal wants a definite answer, even when its |
| 937 | // own deadline lands after the foreground one. |
| 938 | if (!isForegroundProbe(probeEvent.signal)) toleratesMiss = false; |
| 939 | if (isLongResume(probeEvent.signal) && !longResume) { |
| 940 | longResume = true; |
| 941 | longResumeStartedAt = yield* Clock.monotonicTimeNanos; |
| 942 | } |
| 943 | yield* armSilentSocketCheck(probeEvent.signal); |
| 944 | const signalDeadline = |
| 945 | (yield* Clock.monotonicTimeNanos) + Duration.toNanosUnsafe(signalTimeout); |
| 946 | if (signalDeadline < deadline) deadline = signalDeadline; |
| 947 | } |
| 948 | } |
| 949 | } |
| 950 | }); |
| 951 | |
| 952 | // Advanced as a session goes live and kept across loop restarts, so no two sessions share one. |
| 953 | let generation = 0; |
| 954 | const runAttempt = Effect.fnUntraced(function* ( |
| 955 | attempt: number, |
| 956 | nextGeneration: number, |
| 957 | lastFailure: ConnectionAttemptError | null, |
| 958 | pendingRetry: Option.Option<PendingRetryTrace>, |
| 959 | ignoreOffline: boolean, |
| 960 | ) { |
| 961 | const switchingTo = yield* Ref.get(preferredRouteId); |
| 962 | yield* SubscriptionRef.set(prepared, Option.none()); |
| 963 | const synchronizedRoutes = yield* Ref.make(0); |
| 964 | const establishment = yield* Effect.raceAllFirst([ |
| 965 | Effect.exit( |
| 966 | establishTracedConnection( |
| 967 | attempt, |
| 968 | nextGeneration, |
| 969 | lastFailure, |
| 970 | pendingRetry, |
| 971 | synchronizedRoutes, |
| 972 | ), |
| 973 | ).pipe( |
| 974 | Effect.map((exit): EstablishmentEvent => ({ |
| 975 | _tag: "Completed", |
| 976 | exit, |
| 977 | })), |
| 978 | ), |
| 979 | waitForEstablishmentInterrupt(ignoreOffline).pipe( |
| 980 | Effect.map((resetRetry): EstablishmentEvent => ({ |
| 981 | _tag: "Interrupted", |
| 982 | resetRetry, |
| 983 | })), |
| 984 | ), |
| 985 | // Each route may use the full setup time before the next is tried, and |
| 986 | // each route that reaches synchronizing adds its own snapshot budget, so |
| 987 | // a route that fails late does not eat the next route's time. |
| 988 | Effect.gen(function* () { |
| 989 | const routes = connectionRoutes(yield* Ref.get(currentEntry)); |
| 990 | const setup = Duration.toNanosUnsafe(Duration.times(establishmentTimeout, routes.length)); |
| 991 | // The desktop bounds SSH preparation, which may download a server |
| 992 | // release, so for SSH-only entries the clock stops while preparing. |
| 993 | const pausesWhilePreparing = routes.every((route) => connectionRouteKind(route) === "ssh"); |
| 994 | const preparing = (expected: boolean) => |
| 995 | SubscriptionRef.changes(state).pipe( |
| 996 | Stream.filter((current) => (current.stage === "preparing") === expected), |
| 997 | Stream.runHead, |
| 998 | ); |
| 999 | let spent = 0n; |
| 1000 | while (true) { |
| 1001 | if (pausesWhilePreparing) yield* preparing(false); |
| 1002 | const synchronized = BigInt(yield* Ref.get(synchronizedRoutes)); |
| 1003 | const remaining = |
| 1004 | setup + synchronized * Duration.toNanosUnsafe(synchronizationTimeout) - spent; |
| 1005 | if (remaining <= 0n) { |
| 1006 | return { _tag: "TimedOut" } as const satisfies EstablishmentEvent; |
| 1007 | } |
| 1008 | const resumedAt = yield* Clock.monotonicTimeNanos; |
| 1009 | const sleep = Effect.sleep(Duration.nanos(remaining)); |
| 1010 | yield* pausesWhilePreparing ? Effect.raceFirst(sleep, preparing(true)) : sleep; |
| 1011 | spent += (yield* Clock.monotonicTimeNanos) - resumedAt; |
| 1012 | } |
| 1013 | }), |
| 1014 | ]); |
| 1015 | |
| 1016 | if (establishment._tag === "Interrupted") { |
| 1017 | return { |
| 1018 | _tag: "Interrupted", |
| 1019 | established: false, |
| 1020 | stable: false, |
| 1021 | resetRetry: establishment.resetRetry, |
| 1022 | } satisfies AttemptOutcome; |
| 1023 | } |
| 1024 | if (establishment._tag === "TimedOut") { |
| 1025 | return { |
| 1026 | _tag: "Failure", |
| 1027 | established: false, |
| 1028 | stable: false, |
| 1029 | failure: { |
| 1030 | error: new ConnectionTransientError({ |
| 1031 | reason: "timeout", |
| 1032 | ...setupTimeoutDetail, |
| 1033 | }), |
| 1034 | attemptSpan: Option.none(), |
| 1035 | }, |
| 1036 | } satisfies AttemptOutcome; |
| 1037 | } |
| 1038 | if (Exit.isFailure(establishment.exit)) { |
| 1039 | yield* logUnexpectedFailure("attempt", establishment.exit.cause); |
| 1040 | return failureFromCause(target, establishment.exit.cause, false, false); |
| 1041 | } |
| 1042 | |
| 1043 | const active = establishment.exit.value; |
| 1044 | if (Option.isSome(switchingTo)) { |
| 1045 | const landed = connectionRouteId(active.lease.prepared.target); |
| 1046 | if (landed !== switchingTo.value) { |
| 1047 | const until = |
| 1048 | (yield* Clock.monotonicTimeNanos) + BigInt(BETTER_ROUTE_COOLDOWN_MS) * 1_000_000n; |
| 1049 | yield* Ref.update(routeCooldowns, (current) => |
| 1050 | new Map(current).set(switchingTo.value, until), |
| 1051 | ); |
| 1052 | } |
| 1053 | } |
| 1054 | const currentIntent = yield* Ref.get(intent); |
| 1055 | if (!currentIntent.desired || (currentIntent.network === "offline" && !ignoreOffline)) { |
| 1056 | return { |
| 1057 | _tag: "Interrupted", |
| 1058 | established: false, |
| 1059 | stable: false, |
| 1060 | resetRetry: false, |
| 1061 | } satisfies AttemptOutcome; |
| 1062 | } |
| 1063 | |
| 1064 | const routes = yield* currentRoutes; |
| 1065 | const activeRouteId = connectionRouteId(active.lease.prepared.target); |
| 1066 | const connectedLease: ConnectedLease = { |
| 1067 | ...active.lease, |
| 1068 | activeRoute: routes.find((route) => connectionRouteId(route.target) === activeRouteId), |
| 1069 | generation: nextGeneration, |
| 1070 | }; |
| 1071 | const refusedRouteIds = yield* Ref.updateAndGet(refusedRoutes, (known) => |
| 1072 | mergeRefusedRoutes( |
| 1073 | known, |
| 1074 | active.lease.refusedRouteIds, |
| 1075 | (id) => |
| 1076 | id !== activeRouteId && routes.some((route) => connectionRouteId(route.target) === id), |
| 1077 | ), |
| 1078 | ); |
| 1079 | const connectedAt = yield* Clock.currentTimeMillis; |
| 1080 | generation = nextGeneration; |
| 1081 | yield* SubscriptionRef.set(prepared, Option.some(connectedLease.prepared)); |
| 1082 | yield* SubscriptionRef.set(session, Option.some(connectedLease.session)); |
| 1083 | if (options?.learnRoutes !== undefined) { |
| 1084 | yield* learnRoutesFrom(options.learnRoutes, connectedLease).pipe(Effect.forkScoped); |
| 1085 | } |
| 1086 | yield* setState({ |
| 1087 | desired: true, |
| 1088 | network: currentIntent.network, |
| 1089 | phase: "connected", |
| 1090 | stage: null, |
| 1091 | attempt, |
| 1092 | generation: nextGeneration, |
| 1093 | lastFailure: null, |
| 1094 | retryAt: null, |
| 1095 | refusedRouteIds, |
| 1096 | }); |
| 1097 | |
| 1098 | const connectedExit = yield* Effect.raceFirst( |
| 1099 | connectedLease.session.closed.pipe( |
| 1100 | Effect.mapError((error): TracedAttemptFailure => ({ |
| 1101 | error, |
| 1102 | attemptSpan: active.attemptSpan, |
| 1103 | })), |
| 1104 | ), |
| 1105 | monitorConnectedLease(connectedLease).pipe( |
| 1106 | Effect.mapError((error): TracedAttemptFailure => ({ |
| 1107 | error, |
| 1108 | attemptSpan: active.attemptSpan, |
| 1109 | })), |
| 1110 | ), |
| 1111 | ).pipe(Effect.exit); |
| 1112 | const connectedForMs = (yield* Clock.currentTimeMillis) - connectedAt; |
| 1113 | if (Exit.isSuccess(connectedExit)) { |
| 1114 | return { |
| 1115 | _tag: "Interrupted", |
| 1116 | established: true, |
| 1117 | stable: connectedForMs >= BACKOFF_RESET_AFTER_MS, |
| 1118 | resetRetry: connectedExit.value, |
| 1119 | } satisfies AttemptOutcome; |
| 1120 | } |
| 1121 | yield* logUnexpectedFailure("attempt", connectedExit.cause); |
| 1122 | const outcome = failureFromCause( |
| 1123 | target, |
| 1124 | connectedExit.cause, |
| 1125 | true, |
| 1126 | connectedForMs >= BACKOFF_RESET_AFTER_MS, |
| 1127 | ); |
| 1128 | yield* Effect.void.pipe( |
| 1129 | Effect.withSpan("EnvironmentSupervisor.connectionLost", { |
| 1130 | root: true, |
| 1131 | attributes: { |
| 1132 | "environment.id": target.environmentId, |
| 1133 | "environment.label": target.label, |
| 1134 | "environment.target.kind": target._tag, |
| 1135 | "connection.connected_ms": connectedForMs, |
| 1136 | "connection.failure.reason": outcome.failure.error.reason, |
| 1137 | "connection.failure.detail": outcome.failure.error.detail, |
| 1138 | }, |
| 1139 | }), |
| 1140 | ); |
| 1141 | return outcome; |
| 1142 | }, Effect.ensuring(clearLease)); |
| 1143 | |
| 1144 | const waitForRetrySignal = Effect.fnUntraced(function* (delayMs: number) { |
| 1145 | // @effect-diagnostics-next-line raceFirstWithSleepToTimeout:off - the sleep is the retry delay (false), not a timeout around the signal loop |
| 1146 | return yield* Effect.raceFirst( |
| 1147 | Effect.sleep(delayMs).pipe(Effect.as(false)), |
| 1148 | Effect.gen(function* () { |
| 1149 | for (;;) { |
| 1150 | const next = yield* Queue.take(signals); |
| 1151 | switch (next._tag) { |
| 1152 | case "Wakeup": |
| 1153 | return ConnectionWakeups.resetsRetryBackoff(next.reason); |
| 1154 | case "ConnectRequested": |
| 1155 | case "DisconnectRequested": |
| 1156 | case "RetryRequested": |
| 1157 | case "NetworkChanged": |
| 1158 | return false; |
| 1159 | case "BetterRouteAvailable": |
| 1160 | break; |
| 1161 | } |
| 1162 | } |
| 1163 | }), |
| 1164 | ); |
| 1165 | }); |
| 1166 | |
| 1167 | const waitForSignal = Queue.take(signals).pipe( |
| 1168 | Effect.repeat({ while: (next) => next._tag === "BetterRouteAvailable" }), |
| 1169 | Effect.map( |
| 1170 | (next) => next._tag === "Wakeup" && ConnectionWakeups.resetsRetryBackoff(next.reason), |
| 1171 | ), |
| 1172 | ); |
| 1173 | |
| 1174 | const run = Effect.fnUntraced(function* () { |
| 1175 | let failureCount = 0; |
| 1176 | // Approval waits count as attempts but not toward backoff, so a later outage starts low. |
| 1177 | let backoffFailures = 0; |
| 1178 | let approvalWaits = 0; |
| 1179 | let latestFailure: ConnectionAttemptError | null = null; |
| 1180 | let pendingRetry = Option.none<PendingRetryTrace>(); |
| 1181 | const resetRetryLadder = () => { |
| 1182 | failureCount = 0; |
| 1183 | backoffFailures = 0; |
| 1184 | approvalWaits = 0; |
| 1185 | pendingRetry = Option.none(); |
| 1186 | }; |
| 1187 | let replacing = false; |
| 1188 | |
| 1189 | for (;;) { |
| 1190 | const retryRequested = yield* Ref.getAndSet(resetRetryState, false); |
| 1191 | if (retryRequested) { |
| 1192 | failureCount = 0; |
| 1193 | backoffFailures = 0; |
| 1194 | approvalWaits = 0; |
| 1195 | latestFailure = null; |
| 1196 | pendingRetry = Option.none(); |
| 1197 | } |
| 1198 | const currentIntent = yield* Ref.get(intent); |
| 1199 | if (!currentIntent.desired) { |
| 1200 | resetRetryLadder(); |
| 1201 | latestFailure = null; |
| 1202 | yield* clearLease; |
| 1203 | yield* setState(availableState(currentIntent, generation)); |
| 1204 | yield* waitForSignal; |
| 1205 | continue; |
| 1206 | } |
| 1207 | // The platform's offline report can stay wrong after the network is back, |
| 1208 | // so an explicit retry still tries instead of waiting for it to change. |
| 1209 | const ignoreOffline = replacing || retryRequested; |
| 1210 | if (currentIntent.network === "offline" && !ignoreOffline) { |
| 1211 | yield* clearLease; |
| 1212 | yield* setState(offlineState(currentIntent, generation, failureCount + 1, latestFailure)); |
| 1213 | const applicationActivated = yield* waitForSignal; |
| 1214 | if (applicationActivated) { |
| 1215 | resetRetryLadder(); |
| 1216 | } |
| 1217 | continue; |
| 1218 | } |
| 1219 | |
| 1220 | const attempt = failureCount + 1; |
| 1221 | const nextGeneration = generation + 1; |
| 1222 | const outcome: AttemptOutcome = yield* Effect.acquireUseRelease( |
| 1223 | Scope.make(), |
| 1224 | (scope) => |
| 1225 | runAttempt(attempt, nextGeneration, latestFailure, pendingRetry, ignoreOffline).pipe( |
| 1226 | Scope.provide(scope), |
| 1227 | Effect.catchCause((cause) => |
| 1228 | logUnexpectedFailure("attempt", cause).pipe( |
| 1229 | Effect.as(failureFromCause(target, cause, generation === nextGeneration, false)), |
| 1230 | ), |
| 1231 | ), |
| 1232 | ), |
| 1233 | (scope, exit) => |
| 1234 | Scope.close(scope, exit).pipe( |
| 1235 | Effect.catchCause((cause) => logUnexpectedFailure("release", cause)), |
| 1236 | ), |
| 1237 | ); |
| 1238 | replacing = false; |
| 1239 | const failedProbe = yield* Ref.getAndSet(probeUnanswered, false); |
| 1240 | if (outcome.established) { |
| 1241 | // A session came up, so no earlier failure describes the connection. |
| 1242 | // Losing a live session is not a failed connection either, for example |
| 1243 | // a socket the OS killed while the app was suspended: its replacement |
| 1244 | // reports no failure until it fails itself. |
| 1245 | latestFailure = null; |
| 1246 | if (outcome.stable) { |
| 1247 | resetRetryLadder(); |
| 1248 | } |
| 1249 | } |
| 1250 | if (outcome._tag === "Interrupted") { |
| 1251 | if (outcome.resetRetry) { |
| 1252 | resetRetryLadder(); |
| 1253 | replacing = true; |
| 1254 | } |
| 1255 | continue; |
| 1256 | } |
| 1257 | |
| 1258 | const attemptSpan: Option.Option<Tracer.Span> = outcome.failure.attemptSpan; |
| 1259 | const error: ConnectionAttemptError = outcome.failure.error; |
| 1260 | if (!outcome.established) { |
| 1261 | latestFailure = error; |
| 1262 | } |
| 1263 | if (error._tag === "ConnectionBlockedError") { |
| 1264 | const blockedIntent = yield* Ref.get(intent); |
| 1265 | yield* setState({ |
| 1266 | desired: blockedIntent.desired, |
| 1267 | network: blockedIntent.network, |
| 1268 | phase: "blocked", |
| 1269 | stage: null, |
| 1270 | attempt, |
| 1271 | generation, |
| 1272 | lastFailure: error, |
| 1273 | retryAt: null, |
| 1274 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 1275 | }); |
| 1276 | const applicationActivated = yield* waitForSignal; |
| 1277 | if (applicationActivated) { |
| 1278 | resetRetryLadder(); |
| 1279 | } |
| 1280 | continue; |
| 1281 | } |
| 1282 | |
| 1283 | if (failedProbe) { |
| 1284 | resetRetryLadder(); |
| 1285 | yield* setState(connectingState(yield* Ref.get(intent), generation, 1, latestFailure)); |
| 1286 | continue; |
| 1287 | } |
| 1288 | |
| 1289 | // The first loss of a live session retries almost at once; the ladder |
| 1290 | // still counts it, so a host that keeps dropping sessions backs off. |
| 1291 | const firstLoss = outcome.established && failureCount === 0; |
| 1292 | failureCount += 1; |
| 1293 | if (error.reason === "approval-pending") approvalWaits += 1; |
| 1294 | const awaitingApproval = |
| 1295 | error.reason === "approval-pending" && approvalWaits <= APPROVAL_PENDING_FAST_RETRIES; |
| 1296 | if (!awaitingApproval) backoffFailures += 1; |
| 1297 | const delayMs = awaitingApproval |
| 1298 | ? APPROVAL_PENDING_RETRY_DELAY_MS |
| 1299 | : firstLoss |
| 1300 | ? Math.round(SESSION_LOSS_RETRY_DELAY_MS * (0.5 + (yield* Random.next) / 2)) |
| 1301 | : error.reason === "approval-pending" |
| 1302 | ? Math.max( |
| 1303 | APPROVAL_PENDING_RETRY_DELAY_MS, |
| 1304 | retryDelayMs(backoffFailures - 1, yield* Random.next), |
| 1305 | ) |
| 1306 | : retryDelayMs(backoffFailures - 1, yield* Random.next); |
| 1307 | pendingRetry = Option.map(attemptSpan, (previousAttempt) => ({ |
| 1308 | previousAttempt, |
| 1309 | failureCount, |
| 1310 | delayMs, |
| 1311 | reason: error.reason, |
| 1312 | })); |
| 1313 | const failedIntent = yield* Ref.get(intent); |
| 1314 | yield* setState({ |
| 1315 | desired: failedIntent.desired, |
| 1316 | network: failedIntent.network, |
| 1317 | phase: "backoff", |
| 1318 | stage: null, |
| 1319 | attempt, |
| 1320 | generation, |
| 1321 | lastFailure: latestFailure, |
| 1322 | retryAt: (yield* Clock.currentTimeMillis) + delayMs, |
| 1323 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 1324 | }); |
| 1325 | const applicationActivated = yield* waitForRetrySignal(delayMs); |
| 1326 | if (applicationActivated) { |
| 1327 | resetRetryLadder(); |
| 1328 | } |
| 1329 | } |
| 1330 | }); |
| 1331 | |
| 1332 | // Registered before the fibers below so closing the scope interrupts them first. |
| 1333 | // A loop still running when the queue shuts down would report that as a failure. |
| 1334 | yield* Effect.addFinalizer(() => Queue.shutdown(signals).pipe(Effect.andThen(clearLease))); |
| 1335 | // A defect in a background listener would otherwise end it for good, silently. |
| 1336 | // Untraced: a span here would stay open for the supervisor's whole lifetime. |
| 1337 | const keepRunning = Effect.fnUntraced( |
| 1338 | function* (task: string, effect: Effect.Effect<void>, onRestart?: Effect.Effect<void>) { |
| 1339 | let current = effect; |
| 1340 | while (true) { |
| 1341 | const exit = yield* Effect.exit(current); |
| 1342 | if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; |
| 1343 | yield* Effect.logError("Connection supervisor task failed; restarting it.").pipe( |
| 1344 | Effect.annotateLogs({ |
| 1345 | "environment.id": target.environmentId, |
| 1346 | "environment.label": target.label, |
| 1347 | task, |
| 1348 | ...safeErrorLogAttributes(exit.cause.reasons.find(Cause.isDieReason)?.defect), |
| 1349 | }), |
| 1350 | ); |
| 1351 | yield* Effect.sleep(RETRY_BASE_DELAY_MS); |
| 1352 | current = onRestart ? onRestart.pipe(Effect.andThen(effect)) : effect; |
| 1353 | } |
| 1354 | }, |
| 1355 | (self) => Effect.forkScoped(self), |
| 1356 | ); |
| 1357 | |
| 1358 | const applyNetwork = (network: NetworkStatus) => |
| 1359 | Ref.modify(intent, (current) => { |
| 1360 | const next = targetNetwork(network); |
| 1361 | return current.network === next |
| 1362 | ? [false, current] |
| 1363 | : ([true, { ...current, network: next }] as const); |
| 1364 | }).pipe( |
| 1365 | Effect.flatMap((changed) => |
| 1366 | changed |
| 1367 | ? (network === "offline" ? Effect.void : forgetSilentRoutes).pipe( |
| 1368 | Effect.andThen(signal({ _tag: "NetworkChanged", network })), |
| 1369 | ) |
| 1370 | : Effect.void, |
| 1371 | ), |
| 1372 | ); |
| 1373 | // A restarted listener may have missed changes while it was down. |
| 1374 | yield* keepRunning( |
| 1375 | "network-listener", |
| 1376 | connectivity.changes.pipe(Stream.runForEach(applyNetwork)), |
| 1377 | connectivity.status.pipe(Effect.flatMap(applyNetwork)), |
| 1378 | ); |
| 1379 | yield* keepRunning( |
| 1380 | "wakeup-listener", |
| 1381 | wakeups.changes.pipe( |
| 1382 | Stream.filter(ConnectionWakeups.reachesConnectionSupervisor), |
| 1383 | Stream.runForEach((reason) => |
| 1384 | // A long suspension can hide a network change the platform never reported. |
| 1385 | (reason === "network-changed" || reason === "application-active-long" |
| 1386 | ? forgetSilentRoutes |
| 1387 | : Effect.void |
| 1388 | ).pipe(Effect.andThen(signal({ _tag: "Wakeup", reason }))), |
| 1389 | ), |
| 1390 | ), |
| 1391 | ); |
| 1392 | yield* keepRunning( |
| 1393 | "better-route-checker", |
| 1394 | Queue.take(betterRouteChecks).pipe(Effect.flatMap(checkBetterRoutes), Effect.forever), |
| 1395 | ); |
| 1396 | // A restarted loop has forgotten its retry ladder, so it waits out the |
| 1397 | // ladder's ceiling unless a signal cuts the wait short. |
| 1398 | yield* run().pipe( |
| 1399 | Effect.catchCause((cause) => |
| 1400 | Effect.gen(function* () { |
| 1401 | yield* logUnexpectedFailure("loop", cause); |
| 1402 | const currentIntent = yield* Ref.get(intent); |
| 1403 | const lastFailure = unexpectedFailure(target).error; |
| 1404 | yield* setState( |
| 1405 | !currentIntent.desired |
| 1406 | ? availableState(currentIntent, generation) |
| 1407 | : currentIntent.network === "offline" |
| 1408 | ? offlineState(currentIntent, generation, 1, lastFailure) |
| 1409 | : { |
| 1410 | desired: true, |
| 1411 | network: currentIntent.network, |
| 1412 | phase: "backoff", |
| 1413 | stage: null, |
| 1414 | attempt: 1, |
| 1415 | generation, |
| 1416 | lastFailure, |
| 1417 | retryAt: (yield* Clock.currentTimeMillis) + RETRY_MAX_DELAY_MS, |
| 1418 | refusedRouteIds: NO_REFUSED_ROUTES, |
| 1419 | }, |
| 1420 | ); |
| 1421 | yield* waitForRetrySignal(RETRY_MAX_DELAY_MS); |
| 1422 | }), |
| 1423 | ), |
| 1424 | Effect.forever, |
| 1425 | Effect.forkScoped, |
| 1426 | ); |
| 1427 | |
| 1428 | const connect = Ref.update(intent, (current) => ({ |
| 1429 | ...current, |
| 1430 | desired: true, |
| 1431 | })).pipe( |
| 1432 | Effect.andThen(signal({ _tag: "ConnectRequested" })), |
| 1433 | Effect.withSpan("EnvironmentSupervisor.connect"), |
| 1434 | ); |
| 1435 | |
| 1436 | const disconnect = Ref.update(intent, (current) => ({ |
| 1437 | ...current, |
| 1438 | desired: false, |
| 1439 | })).pipe( |
| 1440 | Effect.andThen(signal({ _tag: "DisconnectRequested" })), |
| 1441 | Effect.withSpan("EnvironmentSupervisor.disconnect"), |
| 1442 | ); |
| 1443 | |
| 1444 | const retryNow = Ref.set(resetRetryState, true).pipe( |
| 1445 | Effect.andThen(signal({ _tag: "RetryRequested" })), |
| 1446 | Effect.withSpan("EnvironmentSupervisor.retryNow"), |
| 1447 | ); |
| 1448 | |
| 1449 | return EnvironmentSupervisor.of({ |
| 1450 | target, |
| 1451 | state, |
| 1452 | session, |
| 1453 | prepared, |
| 1454 | connect, |
| 1455 | disconnect, |
| 1456 | retryNow, |
| 1457 | }); |
| 1458 | }); |