The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | AuthStandardClientScopes, |
| 3 | EnvironmentId, |
| 4 | type ServerConfig, |
| 5 | type ServerConfigStreamEvent, |
| 6 | } from "@t3tools/contracts"; |
| 7 | import { describe, expect, it } from "@effect/vitest"; |
| 8 | import { RelayClientTracer } from "@t3tools/shared/relayTracing"; |
| 9 | import * as Cause from "effect/Cause"; |
| 10 | import * as Deferred from "effect/Deferred"; |
| 11 | import * as Effect from "effect/Effect"; |
| 12 | import * as Exit from "effect/Exit"; |
| 13 | import * as Fiber from "effect/Fiber"; |
| 14 | import * as Layer from "effect/Layer"; |
| 15 | import * as Logger from "effect/Logger"; |
| 16 | import * as Option from "effect/Option"; |
| 17 | import * as Queue from "effect/Queue"; |
| 18 | import * as Random from "effect/Random"; |
| 19 | import * as Ref from "effect/Ref"; |
| 20 | import * as References from "effect/References"; |
| 21 | import * as Scope from "effect/Scope"; |
| 22 | import * as Stream from "effect/Stream"; |
| 23 | import * as SubscriptionRef from "effect/SubscriptionRef"; |
| 24 | import * as Clock from "effect/Clock"; |
| 25 | import * as TestClock from "effect/testing/TestClock"; |
| 26 | import * as Tracer from "effect/Tracer"; |
| 27 | |
| 28 | import * as RemoteEnvironmentAuthorization from "../authorization/service.ts"; |
| 29 | import * as ConnectionCredentialStore from "./credentialStore.ts"; |
| 30 | import * as TokenStore from "../authorization/tokenStore.ts"; |
| 31 | import * as ClientCapabilities from "../platform/capabilities.ts"; |
| 32 | import * as ManagedRelay from "../relay/managedRelay.ts"; |
| 33 | import * as RpcHttp from "../rpc/http.ts"; |
| 34 | import type { WsRpcProtocolClient } from "../rpc/protocol.ts"; |
| 35 | import { fetchEnvironmentSessionState } from "../state/session.ts"; |
| 36 | import type { ConnectionCatalogEntry, ConnectionRoute } from "./catalog.ts"; |
| 37 | import * as Connectivity from "./connectivity.ts"; |
| 38 | import * as ConnectionDriver from "./driver.ts"; |
| 39 | import { BearerConnectionProfile } from "./catalog.ts"; |
| 40 | import { |
| 41 | BearerConnectionTarget, |
| 42 | ConnectionBlockedError, |
| 43 | ConnectionTransientError, |
| 44 | PrimaryConnectionTarget, |
| 45 | RelayConnectionTarget, |
| 46 | SshConnectionTarget, |
| 47 | type ConnectionAttemptError, |
| 48 | type ConnectionTarget, |
| 49 | type NetworkStatus, |
| 50 | type PreparedConnection, |
| 51 | type SupervisorConnectionState, |
| 52 | } from "./model.ts"; |
| 53 | import * as RpcSession from "../rpc/session.ts"; |
| 54 | import { presentConnectionState } from "./presentation.ts"; |
| 55 | import * as EnvironmentSupervisor from "./supervisor.ts"; |
| 56 | import * as ConnectionWakeups from "./wakeups.ts"; |
| 57 | import { connectionRouteId, entryWithRoutes, isLearned, mergeLearnedRoutes } from "./routes.ts"; |
| 58 | |
| 59 | const TARGET = new PrimaryConnectionTarget({ |
| 60 | environmentId: EnvironmentId.make("environment-1"), |
| 61 | label: "Test environment", |
| 62 | httpBaseUrl: "https://environment.example.test", |
| 63 | wsBaseUrl: "wss://environment.example.test", |
| 64 | }); |
| 65 | |
| 66 | const RELAY_TARGET = new RelayConnectionTarget({ |
| 67 | environmentId: TARGET.environmentId, |
| 68 | label: TARGET.label, |
| 69 | }); |
| 70 | |
| 71 | const TARGET_ENTRY: ConnectionCatalogEntry = { |
| 72 | target: TARGET, |
| 73 | profile: Option.none(), |
| 74 | enabled: true, |
| 75 | }; |
| 76 | |
| 77 | const RELAY_ENTRY: ConnectionCatalogEntry = { |
| 78 | target: RELAY_TARGET, |
| 79 | profile: Option.none(), |
| 80 | enabled: true, |
| 81 | }; |
| 82 | |
| 83 | const PREPARED_CONNECTION: PreparedConnection = { |
| 84 | environmentId: TARGET.environmentId, |
| 85 | label: TARGET.label, |
| 86 | httpBaseUrl: TARGET.httpBaseUrl, |
| 87 | socketUrl: "wss://environment.example.test/ws", |
| 88 | httpAuthorization: null, |
| 89 | target: TARGET, |
| 90 | }; |
| 91 | |
| 92 | const TEST_RPC_CLIENT = {} as WsRpcProtocolClient; |
| 93 | |
| 94 | const LAN_TARGET = new BearerConnectionTarget({ |
| 95 | environmentId: TARGET.environmentId, |
| 96 | label: TARGET.label, |
| 97 | connectionId: "bearer:lan", |
| 98 | }); |
| 99 | const LAN_ROUTE = { |
| 100 | target: LAN_TARGET, |
| 101 | profile: Option.some( |
| 102 | new BearerConnectionProfile({ |
| 103 | connectionId: LAN_TARGET.connectionId, |
| 104 | environmentId: TARGET.environmentId, |
| 105 | label: TARGET.label, |
| 106 | httpBaseUrl: "http://192.168.1.10:3773/", |
| 107 | wsBaseUrl: "ws://192.168.1.10:3773/", |
| 108 | }), |
| 109 | ), |
| 110 | } satisfies ConnectionRoute; |
| 111 | const LAN_THEN_RELAY_ENTRY: ConnectionCatalogEntry = { |
| 112 | target: LAN_ROUTE.target, |
| 113 | profile: LAN_ROUTE.profile, |
| 114 | alternateRoutes: [{ target: RELAY_TARGET, profile: Option.none() }], |
| 115 | enabled: true, |
| 116 | }; |
| 117 | |
| 118 | const SSH_ENTRY: ConnectionCatalogEntry = { |
| 119 | target: new SshConnectionTarget({ |
| 120 | environmentId: TARGET.environmentId, |
| 121 | label: TARGET.label, |
| 122 | connectionId: "ssh-connection", |
| 123 | }), |
| 124 | profile: Option.none(), |
| 125 | enabled: true, |
| 126 | }; |
| 127 | |
| 128 | const LAN_ENTRY: ConnectionCatalogEntry = { |
| 129 | target: LAN_ROUTE.target, |
| 130 | profile: LAN_ROUTE.profile, |
| 131 | enabled: true, |
| 132 | }; |
| 133 | |
| 134 | const PROBE_CAPABLE_CONFIG = { |
| 135 | environment: { capabilities: { connectionProbe: true } }, |
| 136 | } as ServerConfig; |
| 137 | |
| 138 | function preparedFor(target: ConnectionTarget): PreparedConnection { |
| 139 | return { ...PREPARED_CONNECTION, target }; |
| 140 | } |
| 141 | |
| 142 | function transient(message = "Connection failed.") { |
| 143 | return new ConnectionTransientError({ |
| 144 | reason: "transport", |
| 145 | detail: message, |
| 146 | }); |
| 147 | } |
| 148 | |
| 149 | function blocked(message = "Authentication required.") { |
| 150 | return new ConnectionBlockedError({ |
| 151 | reason: "authentication", |
| 152 | detail: message, |
| 153 | }); |
| 154 | } |
| 155 | |
| 156 | function hostRefusal() { |
| 157 | return new ConnectionBlockedError({ |
| 158 | reason: "authentication", |
| 159 | detail: "The environment credential is invalid.", |
| 160 | hop: "server", |
| 161 | }); |
| 162 | } |
| 163 | |
| 164 | function awaitState( |
| 165 | state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>, |
| 166 | predicate: (value: SupervisorConnectionState) => boolean, |
| 167 | ) { |
| 168 | return SubscriptionRef.changes(state).pipe( |
| 169 | Stream.filter(predicate), |
| 170 | Stream.runHead, |
| 171 | Effect.map(Option.getOrThrow), |
| 172 | ); |
| 173 | } |
| 174 | |
| 175 | function recordLogs(logs: Array<unknown>, annotations?: Array<unknown>) { |
| 176 | return Logger.layer( |
| 177 | [ |
| 178 | Logger.make(({ message, fiber }) => { |
| 179 | logs.push(message); |
| 180 | annotations?.push(fiber.getRef(References.CurrentLogAnnotations)); |
| 181 | }), |
| 182 | ], |
| 183 | { mergeWithExisting: false }, |
| 184 | ); |
| 185 | } |
| 186 | |
| 187 | const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: { |
| 188 | readonly networkStatus?: NetworkStatus; |
| 189 | readonly prepare?: ( |
| 190 | attempt: number, |
| 191 | target: ConnectionTarget, |
| 192 | ) => Effect.Effect<PreparedConnection, ConnectionAttemptError>; |
| 193 | readonly ready?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>; |
| 194 | readonly release?: (attempt: number) => Effect.Effect<void>; |
| 195 | readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>; |
| 196 | readonly configEvents?: Stream.Stream<ServerConfigStreamEvent> | undefined; |
| 197 | readonly initialConfig?: (attempt: number) => Effect.Effect<ServerConfig, ConnectionAttemptError>; |
| 198 | readonly checkRoute?: (route: ConnectionRoute) => Effect.Effect<ConnectionDriver.RouteCheck>; |
| 199 | readonly checkHost?: (prepared: PreparedConnection) => Effect.Effect<ConnectionDriver.RouteCheck>; |
| 200 | readonly preflight?: (route: ConnectionRoute) => Effect.Effect<ConnectionDriver.RoutePreflight>; |
| 201 | readonly inboundFrames?: Effect.Effect<number>; |
| 202 | }) { |
| 203 | const networkStatus = yield* SubscriptionRef.make<NetworkStatus>( |
| 204 | options?.networkStatus ?? "online", |
| 205 | ); |
| 206 | const prepareCount = yield* Ref.make(0); |
| 207 | const sessionCount = yield* Ref.make(0); |
| 208 | const releaseCount = yield* Ref.make(0); |
| 209 | const wakeups = yield* SubscriptionRef.make<{ |
| 210 | readonly sequence: number; |
| 211 | readonly reason: ConnectionWakeups.ConnectionWakeup; |
| 212 | }>({ |
| 213 | sequence: 0, |
| 214 | reason: "application-active", |
| 215 | }); |
| 216 | const closedSessions = yield* Ref.make< |
| 217 | ReadonlyArray<Deferred.Deferred<never, ConnectionTransientError>> |
| 218 | >([]); |
| 219 | |
| 220 | const connectivity = Connectivity.Connectivity.of({ |
| 221 | status: SubscriptionRef.get(networkStatus), |
| 222 | changes: SubscriptionRef.changes(networkStatus), |
| 223 | }); |
| 224 | |
| 225 | const prepare = Effect.fn("TestConnectionDriver.prepare")(function* (target: ConnectionTarget) { |
| 226 | const attempt = yield* Ref.updateAndGet(prepareCount, (count) => count + 1); |
| 227 | if (options?.prepare) { |
| 228 | return yield* options.prepare(attempt, target); |
| 229 | } |
| 230 | return PREPARED_CONNECTION; |
| 231 | }); |
| 232 | |
| 233 | const checkRoute = (route: ConnectionRoute) => |
| 234 | options?.checkRoute?.(route) ?? Effect.succeed<ConnectionDriver.RouteCheck>("unchecked"); |
| 235 | |
| 236 | const connectRoute = Effect.fn("TestConnectionDriver.connectRoute")(function* ( |
| 237 | target: ConnectionTarget, |
| 238 | reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>, |
| 239 | ) { |
| 240 | yield* reportProgress({ stage: "preparing" }); |
| 241 | const prepared = yield* prepare(target); |
| 242 | yield* reportProgress({ stage: "opening", prepared }); |
| 243 | |
| 244 | const attempt = yield* Ref.updateAndGet(sessionCount, (count) => count + 1); |
| 245 | const closed = yield* Deferred.make<never, ConnectionTransientError>(); |
| 246 | yield* Ref.update(closedSessions, (sessions) => [...sessions, closed]); |
| 247 | |
| 248 | const session = yield* Effect.acquireRelease( |
| 249 | Effect.succeed({ |
| 250 | client: TEST_RPC_CLIENT, |
| 251 | initialConfig: |
| 252 | options?.initialConfig?.(attempt) ?? |
| 253 | Effect.die(new Error("Initial config is not used by supervisor tests.")), |
| 254 | subscribeServerConfig: () => |
| 255 | options?.configEvents ?? |
| 256 | Stream.fromEffect( |
| 257 | (options?.initialConfig?.(attempt) ?? Effect.never).pipe(Effect.orDie), |
| 258 | ).pipe( |
| 259 | Stream.map((config) => ({ version: 1 as const, type: "snapshot" as const, config })), |
| 260 | ), |
| 261 | ready: options?.ready?.(attempt) ?? Effect.void, |
| 262 | probe: options?.probe?.(attempt) ?? Effect.void, |
| 263 | inboundFrames: options?.inboundFrames ?? Effect.succeed(0), |
| 264 | closed: Deferred.await(closed), |
| 265 | } satisfies RpcSession.RpcSession), |
| 266 | () => |
| 267 | Ref.update(releaseCount, (count) => count + 1).pipe( |
| 268 | Effect.andThen(options?.release?.(attempt) ?? Effect.void), |
| 269 | ), |
| 270 | ); |
| 271 | |
| 272 | yield* reportProgress({ stage: "synchronizing", prepared }); |
| 273 | yield* session.ready; |
| 274 | return { prepared, session } satisfies ConnectionDriver.RouteConnection; |
| 275 | }); |
| 276 | |
| 277 | const connect = Effect.fn("TestConnectionDriver.connect")(function* ( |
| 278 | entry: ConnectionCatalogEntry, |
| 279 | reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>, |
| 280 | ) { |
| 281 | yield* reportProgress({ stage: "preparing" }); |
| 282 | return yield* ConnectionDriver.connectOverRoutes(entry, checkRoute, (route) => |
| 283 | connectRoute(route.target, reportProgress), |
| 284 | ); |
| 285 | }); |
| 286 | |
| 287 | const layerDependencies = Layer.mergeAll( |
| 288 | Layer.succeed(Random.Random, { |
| 289 | nextDoubleUnsafe: () => 1 - Number.EPSILON, |
| 290 | nextIntUnsafe: () => 0, |
| 291 | }), |
| 292 | Layer.succeed(Connectivity.Connectivity, connectivity), |
| 293 | Layer.succeed( |
| 294 | ConnectionWakeups.ConnectionWakeups, |
| 295 | ConnectionWakeups.ConnectionWakeups.of({ |
| 296 | changes: SubscriptionRef.changes(wakeups).pipe( |
| 297 | Stream.drop(1), |
| 298 | Stream.map((event) => event.reason), |
| 299 | ), |
| 300 | }), |
| 301 | ), |
| 302 | Layer.succeed( |
| 303 | ConnectionDriver.ConnectionDriver, |
| 304 | ConnectionDriver.ConnectionDriver.of({ |
| 305 | connect, |
| 306 | preflight: (_entry, route) => |
| 307 | options?.preflight?.(route) ?? |
| 308 | checkRoute(route).pipe( |
| 309 | Effect.map((check): ConnectionDriver.RoutePreflight => |
| 310 | check === "answered" ? "ready" : check, |
| 311 | ), |
| 312 | ), |
| 313 | checkHost: (prepared) => |
| 314 | options?.checkHost?.(prepared) ?? |
| 315 | Effect.succeed<ConnectionDriver.RouteCheck>("unchecked"), |
| 316 | }), |
| 317 | ), |
| 318 | ); |
| 319 | |
| 320 | return { |
| 321 | dependencies: layerDependencies, |
| 322 | prepareCount, |
| 323 | sessionCount, |
| 324 | releaseCount, |
| 325 | setNetworkStatus: (status: NetworkStatus) => SubscriptionRef.set(networkStatus, status), |
| 326 | wake: (reason: ConnectionWakeups.ConnectionWakeup) => |
| 327 | SubscriptionRef.update(wakeups, (event) => ({ |
| 328 | sequence: event.sequence + 1, |
| 329 | reason, |
| 330 | })), |
| 331 | closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* ( |
| 332 | exit: Exit.Exit<never, ConnectionTransientError> = Exit.fail(transient("Session closed.")), |
| 333 | ) { |
| 334 | const sessions = yield* Ref.get(closedSessions); |
| 335 | const latest = sessions.at(-1); |
| 336 | if (latest) { |
| 337 | yield* Deferred.done(latest, exit); |
| 338 | } |
| 339 | }), |
| 340 | }; |
| 341 | }); |
| 342 | |
| 343 | describe("EnvironmentSupervisor", () => { |
| 344 | it.effect("does not attempt a connection until it is desired", () => |
| 345 | Effect.gen(function* () { |
| 346 | const harness = yield* makeHarness(); |
| 347 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe( |
| 348 | Effect.provide(harness.dependencies), |
| 349 | ); |
| 350 | |
| 351 | expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("available"); |
| 352 | expect(yield* Ref.get(harness.prepareCount)).toBe(0); |
| 353 | }), |
| 354 | ); |
| 355 | |
| 356 | it.effect("does not let the initial connect signal cancel the first attempt", () => |
| 357 | Effect.gen(function* () { |
| 358 | const harness = yield* makeHarness(); |
| 359 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe( |
| 360 | Effect.provide(harness.dependencies), |
| 361 | ); |
| 362 | |
| 363 | yield* supervisor.connect; |
| 364 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 365 | |
| 366 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 367 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 368 | }), |
| 369 | ); |
| 370 | |
| 371 | it.effect("waits while offline and connects immediately when the network returns", () => |
| 372 | Effect.gen(function* () { |
| 373 | const harness = yield* makeHarness({ networkStatus: "offline" }); |
| 374 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 375 | initiallyDesired: true, |
| 376 | }).pipe(Effect.provide(harness.dependencies)); |
| 377 | |
| 378 | yield* awaitState(supervisor.state, (state) => state.phase === "offline"); |
| 379 | expect(yield* Ref.get(harness.prepareCount)).toBe(0); |
| 380 | |
| 381 | yield* harness.setNetworkStatus("online"); |
| 382 | const ready = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 383 | |
| 384 | expect(ready).toMatchObject({ |
| 385 | desired: true, |
| 386 | network: "online", |
| 387 | phase: "connected", |
| 388 | attempt: 1, |
| 389 | generation: 1, |
| 390 | lastFailure: null, |
| 391 | }); |
| 392 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 393 | }), |
| 394 | ); |
| 395 | |
| 396 | it.effect("explicit retry connects while the network still reports offline", () => |
| 397 | Effect.gen(function* () { |
| 398 | const harness = yield* makeHarness({ |
| 399 | networkStatus: "offline", |
| 400 | prepare: () => Effect.sleep("1 second").pipe(Effect.as(PREPARED_CONNECTION)), |
| 401 | }); |
| 402 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 403 | initiallyDesired: true, |
| 404 | }).pipe(Effect.provide(harness.dependencies)); |
| 405 | |
| 406 | yield* awaitState(supervisor.state, (state) => state.phase === "offline"); |
| 407 | // The loop wakes on the earlier signal and takes the retry's flag while |
| 408 | // RetryRequested is still queued, so that signal must not cancel the attempt. |
| 409 | yield* supervisor.connect; |
| 410 | yield* supervisor.retryNow; |
| 411 | yield* TestClock.adjust("1 second"); |
| 412 | const settled = yield* awaitState( |
| 413 | supervisor.state, |
| 414 | (state) => state.phase === "connected" || state.phase === "offline", |
| 415 | ); |
| 416 | |
| 417 | expect(settled.phase).toBe("connected"); |
| 418 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 419 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 420 | }), |
| 421 | ); |
| 422 | |
| 423 | it.effect.each([ |
| 424 | { label: "cancels connection setup", retry: false, phase: "offline", sessions: 0 }, |
| 425 | { label: "does not cancel an explicit retry", retry: true, phase: "connected", sessions: 1 }, |
| 426 | ])("an offline report during setup $label", ({ retry, phase, sessions }) => |
| 427 | Effect.gen(function* () { |
| 428 | const firstAttemptStarted = yield* Deferred.make<void>(); |
| 429 | const harness = yield* makeHarness({ |
| 430 | prepare: () => |
| 431 | Deferred.succeed(firstAttemptStarted, undefined).pipe( |
| 432 | Effect.andThen(Effect.sleep("1 second")), |
| 433 | Effect.as(PREPARED_CONNECTION), |
| 434 | ), |
| 435 | }); |
| 436 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 437 | initiallyDesired: true, |
| 438 | }).pipe(Effect.provide(harness.dependencies)); |
| 439 | |
| 440 | yield* Deferred.await(firstAttemptStarted); |
| 441 | if (retry) { |
| 442 | yield* supervisor.retryNow; |
| 443 | } |
| 444 | yield* harness.setNetworkStatus("offline"); |
| 445 | yield* TestClock.adjust("1 second"); |
| 446 | const settled = yield* awaitState( |
| 447 | supervisor.state, |
| 448 | (state) => state.phase === "connected" || state.phase === "offline", |
| 449 | ); |
| 450 | |
| 451 | expect(settled.phase).toBe(phase); |
| 452 | expect(yield* Ref.get(harness.sessionCount)).toBe(sessions); |
| 453 | }), |
| 454 | ); |
| 455 | |
| 456 | it.effect("connects to the primary environment while the device reports offline", () => |
| 457 | Effect.gen(function* () { |
| 458 | const harness = yield* makeHarness({ networkStatus: "offline" }); |
| 459 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 460 | initiallyDesired: true, |
| 461 | }).pipe(Effect.provide(harness.dependencies)); |
| 462 | |
| 463 | const settled = yield* awaitState( |
| 464 | supervisor.state, |
| 465 | (state) => state.phase === "connected" || state.phase === "offline", |
| 466 | ); |
| 467 | |
| 468 | expect(settled).toMatchObject({ |
| 469 | phase: "connected", |
| 470 | network: "unknown", |
| 471 | attempt: 1, |
| 472 | lastFailure: null, |
| 473 | }); |
| 474 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 475 | }), |
| 476 | ); |
| 477 | |
| 478 | it.effect( |
| 479 | "keeps a primary attempt through an offline report and restarts it when the network returns", |
| 480 | () => |
| 481 | Effect.gen(function* () { |
| 482 | const started = yield* Deferred.make<void>(); |
| 483 | const harness = yield* makeHarness({ |
| 484 | prepare: (attempt) => |
| 485 | attempt === 1 |
| 486 | ? Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)) |
| 487 | : Effect.succeed(PREPARED_CONNECTION), |
| 488 | }); |
| 489 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 490 | initiallyDesired: true, |
| 491 | }).pipe(Effect.provide(harness.dependencies)); |
| 492 | |
| 493 | yield* Deferred.await(started); |
| 494 | yield* harness.setNetworkStatus("offline"); |
| 495 | yield* TestClock.adjust("1 second"); |
| 496 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 497 | |
| 498 | yield* harness.setNetworkStatus("online"); |
| 499 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 500 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 501 | }).pipe(Effect.provide(TestClock.layer())), |
| 502 | ); |
| 503 | |
| 504 | it.effect("keeps the latest failure visible throughout the next connection attempt", () => |
| 505 | Effect.gen(function* () { |
| 506 | const harness = yield* makeHarness({ |
| 507 | prepare: (attempt) => |
| 508 | attempt === 1 ? Effect.fail(transient("Relay connection timed out.")) : Effect.never, |
| 509 | }); |
| 510 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 511 | initiallyDesired: true, |
| 512 | }).pipe(Effect.provide(harness.dependencies)); |
| 513 | |
| 514 | yield* awaitState( |
| 515 | supervisor.state, |
| 516 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 517 | ); |
| 518 | yield* TestClock.adjust("3 seconds"); |
| 519 | |
| 520 | const retrying = yield* awaitState( |
| 521 | supervisor.state, |
| 522 | (state) => |
| 523 | state.phase === "connecting" && state.stage === "preparing" && state.attempt === 2, |
| 524 | ); |
| 525 | expect(retrying).toMatchObject({ |
| 526 | phase: "connecting", |
| 527 | stage: "preparing", |
| 528 | attempt: 2, |
| 529 | lastFailure: { |
| 530 | _tag: "ConnectionTransientError", |
| 531 | reason: "transport", |
| 532 | message: "Relay connection timed out.", |
| 533 | }, |
| 534 | }); |
| 535 | }).pipe(Effect.provide(TestClock.layer())), |
| 536 | ); |
| 537 | |
| 538 | it.effect("checks back on a pending approval every few seconds instead of backing off", () => |
| 539 | Effect.gen(function* () { |
| 540 | const pending = new ConnectionTransientError({ |
| 541 | reason: "approval-pending", |
| 542 | detail: "Approve this device on Mac mini. It shows code K7QM-2XFD.", |
| 543 | }); |
| 544 | const harness = yield* makeHarness({ |
| 545 | prepare: (attempt) => |
| 546 | Effect.fail(attempt <= 8 ? pending : transient("Relay connection timed out.")), |
| 547 | }); |
| 548 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 549 | initiallyDesired: true, |
| 550 | }).pipe(Effect.provide(harness.dependencies)); |
| 551 | |
| 552 | for (let attempt = 1; attempt <= 8; attempt++) { |
| 553 | const waiting = yield* awaitState( |
| 554 | supervisor.state, |
| 555 | (state) => state.phase === "backoff" && state.attempt === attempt, |
| 556 | ); |
| 557 | expect((waiting.retryAt ?? 0) - (yield* Clock.currentTimeMillis)).toBe(4_000); |
| 558 | yield* TestClock.adjust("4 seconds"); |
| 559 | } |
| 560 | // The waits do not count toward backoff, so an outage right after starts at the bottom. |
| 561 | const failed = yield* awaitState( |
| 562 | supervisor.state, |
| 563 | (state) => state.phase === "backoff" && state.attempt === 9, |
| 564 | ); |
| 565 | expect((failed.retryAt ?? 0) - (yield* Clock.currentTimeMillis)).toBeLessThanOrEqual(2_000); |
| 566 | }).pipe(Effect.provide(TestClock.layer())), |
| 567 | ); |
| 568 | |
| 569 | it.effect("backs off a device nobody approved once its request has expired", () => |
| 570 | Effect.gen(function* () { |
| 571 | const pending = new ConnectionTransientError({ |
| 572 | reason: "approval-pending", |
| 573 | detail: "Approve this device on Mac mini. It shows code K7QM-2XFD.", |
| 574 | }); |
| 575 | const harness = yield* makeHarness({ prepare: () => Effect.fail(pending) }); |
| 576 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 577 | initiallyDesired: true, |
| 578 | }).pipe(Effect.provide(harness.dependencies)); |
| 579 | |
| 580 | const delays: number[] = []; |
| 581 | for (let attempt = 1; attempt <= 156; attempt++) { |
| 582 | const waiting = yield* awaitState( |
| 583 | supervisor.state, |
| 584 | (state) => state.phase === "backoff" && state.attempt === attempt, |
| 585 | ); |
| 586 | const delay = (waiting.retryAt ?? 0) - (yield* Clock.currentTimeMillis); |
| 587 | delays.push(delay); |
| 588 | yield* TestClock.adjust(delay); |
| 589 | } |
| 590 | // Ten minutes of 4 s checks, then backoff that never polls faster than that. |
| 591 | expect(delays.slice(0, 150).every((delay) => delay === 4_000)).toBe(true); |
| 592 | expect(delays.slice(150).every((delay) => delay >= 4_000)).toBe(true); |
| 593 | expect(delays.at(-1)).toBeGreaterThanOrEqual(32_000); |
| 594 | }).pipe(Effect.provide(TestClock.layer())), |
| 595 | ); |
| 596 | |
| 597 | it.effect("converts unexpected driver defects into retryable failures", () => |
| 598 | Effect.gen(function* () { |
| 599 | const harness = yield* makeHarness({ |
| 600 | prepare: (attempt) => |
| 601 | attempt === 1 |
| 602 | ? Effect.die(new Error("Native transport defect.")) |
| 603 | : Effect.succeed(PREPARED_CONNECTION), |
| 604 | }); |
| 605 | const logs: unknown[] = []; |
| 606 | const annotations: unknown[] = []; |
| 607 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 608 | initiallyDesired: true, |
| 609 | }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations)))); |
| 610 | |
| 611 | const failed = yield* awaitState( |
| 612 | supervisor.state, |
| 613 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 614 | ); |
| 615 | expect(failed).toMatchObject({ |
| 616 | lastFailure: { |
| 617 | _tag: "ConnectionTransientError", |
| 618 | reason: "transport", |
| 619 | message: "Test environment connection failed unexpectedly.", |
| 620 | }, |
| 621 | }); |
| 622 | expect(logs).toEqual([["Connection attempt failed with an unexpected defect."]]); |
| 623 | expect(annotations).toMatchObject([ |
| 624 | { "environment.id": RELAY_TARGET.environmentId, errorName: "Error" }, |
| 625 | ]); |
| 626 | |
| 627 | yield* TestClock.adjust("3 seconds"); |
| 628 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 629 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 630 | }).pipe(Effect.provide(TestClock.layer())), |
| 631 | ); |
| 632 | |
| 633 | it.effect.each([ |
| 634 | { |
| 635 | label: "the attempt's network-status read dies", |
| 636 | firstPrepare: Effect.never, |
| 637 | statusDefect: true, |
| 638 | log: "Connection attempt failed with an unexpected defect.", |
| 639 | }, |
| 640 | { |
| 641 | label: "an attempt interrupts itself", |
| 642 | firstPrepare: Effect.interrupt, |
| 643 | statusDefect: false, |
| 644 | log: "Connection attempt failed with an unexpected interruption.", |
| 645 | }, |
| 646 | ])("retries after $label", ({ firstPrepare, statusDefect, log }) => |
| 647 | Effect.gen(function* () { |
| 648 | const harness = yield* makeHarness({ |
| 649 | prepare: (attempt) => (attempt === 1 ? firstPrepare : Effect.succeed(PREPARED_CONNECTION)), |
| 650 | }); |
| 651 | const statusReads = yield* Ref.make(0); |
| 652 | const logs: unknown[] = []; |
| 653 | const annotations: Array<Record<string, unknown>> = []; |
| 654 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 655 | initiallyDesired: true, |
| 656 | }).pipe( |
| 657 | Effect.provideService( |
| 658 | Connectivity.Connectivity, |
| 659 | Connectivity.Connectivity.of({ |
| 660 | // The supervisor reads once at startup; the second read belongs to the first attempt. |
| 661 | status: Ref.updateAndGet(statusReads, (count) => count + 1).pipe( |
| 662 | Effect.flatMap((count) => |
| 663 | statusDefect && count === 2 |
| 664 | ? Effect.die(new Error("Platform connectivity defect.")) |
| 665 | : Effect.succeed("online" as const), |
| 666 | ), |
| 667 | ), |
| 668 | changes: Stream.never, |
| 669 | }), |
| 670 | ), |
| 671 | Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))), |
| 672 | ); |
| 673 | |
| 674 | const failed = yield* awaitState( |
| 675 | supervisor.state, |
| 676 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 677 | ); |
| 678 | expect(failed).toMatchObject({ |
| 679 | generation: 0, |
| 680 | lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, |
| 681 | }); |
| 682 | expect(logs).toEqual([[log]]); |
| 683 | expect(Object.hasOwn(annotations[0]!, "errorType")).toBe(statusDefect); |
| 684 | |
| 685 | yield* TestClock.adjust("999 millis"); |
| 686 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 687 | yield* TestClock.adjust("2 seconds"); |
| 688 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 689 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 690 | }).pipe(Effect.provide(TestClock.layer())), |
| 691 | ); |
| 692 | |
| 693 | it.effect.each([ |
| 694 | { |
| 695 | label: "is interrupted", |
| 696 | closed: Exit.interrupt(), |
| 697 | releaseDefect: false, |
| 698 | log: "Connection attempt failed with an unexpected interruption.", |
| 699 | }, |
| 700 | { |
| 701 | label: "fails while interrupted", |
| 702 | closed: Exit.failCause( |
| 703 | Cause.combine(Cause.fail(transient("Session closed.")), Cause.interrupt()), |
| 704 | ), |
| 705 | releaseDefect: false, |
| 706 | log: null, |
| 707 | }, |
| 708 | { |
| 709 | label: "closes while its release defects", |
| 710 | closed: Exit.fail(transient("Session closed.")), |
| 711 | releaseDefect: true, |
| 712 | log: "Connection release failed with an unexpected defect.", |
| 713 | }, |
| 714 | ])("reconnects soon after a stable session $label", ({ closed, releaseDefect, log }) => |
| 715 | Effect.gen(function* () { |
| 716 | const harness = yield* makeHarness({ |
| 717 | prepare: (attempt) => |
| 718 | attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), |
| 719 | release: (session) => |
| 720 | releaseDefect && session === 1 |
| 721 | ? Effect.die(new Error("Session release defect.")) |
| 722 | : Effect.void, |
| 723 | }); |
| 724 | const logs: unknown[] = []; |
| 725 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 726 | initiallyDesired: true, |
| 727 | }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs)))); |
| 728 | |
| 729 | yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 730 | yield* TestClock.adjust("2 seconds"); |
| 731 | yield* awaitState( |
| 732 | supervisor.state, |
| 733 | (state) => state.phase === "connected" && state.generation === 1, |
| 734 | ); |
| 735 | yield* TestClock.adjust("1 minute"); |
| 736 | yield* harness.closeLatestSession(closed); |
| 737 | const lost = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); |
| 738 | expect(logs).toEqual(log === null ? [] : [[log]]); |
| 739 | expect(lost).toMatchObject({ phase: "backoff", generation: 1 }); |
| 740 | expect(lost.retryAt).toBeLessThanOrEqual((yield* Clock.currentTimeMillis) + 250); |
| 741 | |
| 742 | yield* TestClock.adjust("250 millis"); |
| 743 | const replaced = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 744 | expect(replaced.generation).toBe(2); |
| 745 | }).pipe(Effect.provide(TestClock.layer())), |
| 746 | ); |
| 747 | |
| 748 | it.effect( |
| 749 | "does not reuse a live session's generation when its attempt defects after going live", |
| 750 | () => |
| 751 | Effect.gen(function* () { |
| 752 | const harness = yield* makeHarness({ |
| 753 | prepare: (attempt) => |
| 754 | attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), |
| 755 | }); |
| 756 | let logCalls = 0; |
| 757 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 758 | initiallyDesired: true, |
| 759 | }).pipe( |
| 760 | Effect.provide( |
| 761 | Layer.merge( |
| 762 | harness.dependencies, |
| 763 | Logger.layer( |
| 764 | [ |
| 765 | // Logging the lost session's interruption throws, so the attempt itself defects. |
| 766 | Logger.make(() => { |
| 767 | logCalls += 1; |
| 768 | if (logCalls === 1) throw new Error("Logger defect."); |
| 769 | }), |
| 770 | ], |
| 771 | { mergeWithExisting: false }, |
| 772 | ), |
| 773 | ), |
| 774 | ), |
| 775 | ); |
| 776 | |
| 777 | yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 778 | yield* TestClock.adjust("2 seconds"); |
| 779 | yield* awaitState( |
| 780 | supervisor.state, |
| 781 | (state) => state.phase === "connected" && state.generation === 1, |
| 782 | ); |
| 783 | yield* harness.closeLatestSession(Exit.interrupt()); |
| 784 | const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 785 | expect(backoff).toMatchObject({ generation: 1, lastFailure: null }); |
| 786 | |
| 787 | yield* TestClock.adjust("4 seconds"); |
| 788 | const replaced = yield* awaitState( |
| 789 | supervisor.state, |
| 790 | (state) => state.phase === "connected", |
| 791 | ); |
| 792 | expect(replaced.generation).toBe(2); |
| 793 | }).pipe(Effect.provide(TestClock.layer())), |
| 794 | ); |
| 795 | |
| 796 | it.effect("restarts the loop after a defect outside an attempt", () => |
| 797 | Effect.gen(function* () { |
| 798 | const harness = yield* makeHarness(); |
| 799 | const logs: unknown[] = []; |
| 800 | let randomReads = 0; |
| 801 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 802 | initiallyDesired: true, |
| 803 | }).pipe( |
| 804 | Effect.provideService(Random.Random, { |
| 805 | // Both session losses below die here, so the second reaches the restart handler again. |
| 806 | nextDoubleUnsafe: () => { |
| 807 | randomReads += 1; |
| 808 | if (randomReads <= 2) throw new Error("Random defect."); |
| 809 | return 1 - Number.EPSILON; |
| 810 | }, |
| 811 | nextIntUnsafe: () => 0, |
| 812 | }), |
| 813 | Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))), |
| 814 | ); |
| 815 | |
| 816 | yield* awaitState( |
| 817 | supervisor.state, |
| 818 | (state) => state.phase === "connected" && state.generation === 1, |
| 819 | ); |
| 820 | yield* harness.closeLatestSession(); |
| 821 | const restarted = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 822 | expect(logs).toEqual([["Connection loop failed with an unexpected defect."]]); |
| 823 | expect(restarted).toMatchObject({ |
| 824 | attempt: 1, |
| 825 | generation: 1, |
| 826 | lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, |
| 827 | retryAt: (yield* Clock.currentTimeMillis) + 300_000, |
| 828 | }); |
| 829 | |
| 830 | yield* TestClock.adjust("299999 millis"); |
| 831 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 832 | yield* TestClock.adjust("1 millis"); |
| 833 | const reconnected = yield* awaitState( |
| 834 | supervisor.state, |
| 835 | (state) => state.phase === "connected", |
| 836 | ); |
| 837 | expect(reconnected.generation).toBe(2); |
| 838 | |
| 839 | yield* harness.setNetworkStatus("offline"); |
| 840 | yield* harness.closeLatestSession(); |
| 841 | const offline = yield* awaitState(supervisor.state, (state) => state.phase !== "connected"); |
| 842 | expect(offline).toMatchObject({ |
| 843 | phase: "offline", |
| 844 | attempt: 1, |
| 845 | lastFailure: { _tag: "ConnectionTransientError", reason: "transport" }, |
| 846 | }); |
| 847 | |
| 848 | yield* supervisor.retryNow; |
| 849 | const retried = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 850 | expect(retried.generation).toBe(3); |
| 851 | }).pipe(Effect.provide(TestClock.layer())), |
| 852 | ); |
| 853 | |
| 854 | it.effect.each(["connecting", "connected"] as const)( |
| 855 | "stops attempting when its scope closes while %s", |
| 856 | (phase) => |
| 857 | Effect.gen(function* () { |
| 858 | const preparing = yield* Deferred.make<void>(); |
| 859 | const harness = yield* makeHarness({ |
| 860 | prepare: () => |
| 861 | Deferred.succeed(preparing, undefined).pipe( |
| 862 | Effect.andThen( |
| 863 | phase === "connected" ? Effect.succeed(PREPARED_CONNECTION) : Effect.never, |
| 864 | ), |
| 865 | ), |
| 866 | }); |
| 867 | const logs: unknown[] = []; |
| 868 | const scope = yield* Scope.make(); |
| 869 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 870 | initiallyDesired: true, |
| 871 | }).pipe( |
| 872 | Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))), |
| 873 | Scope.provide(scope), |
| 874 | ); |
| 875 | |
| 876 | yield* Deferred.await(preparing); |
| 877 | yield* awaitState(supervisor.state, (state) => state.phase === phase); |
| 878 | // Yielding between finalizers lets the loop run if the queue shuts down before its interrupt. |
| 879 | yield* Scope.close(scope, Exit.void).pipe( |
| 880 | Effect.provideService(References.MaxOpsBeforeYield, 1), |
| 881 | ); |
| 882 | yield* TestClock.adjust("10 minutes"); |
| 883 | expect(logs).toEqual([]); |
| 884 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 885 | }).pipe(Effect.provide(TestClock.layer())), |
| 886 | ); |
| 887 | |
| 888 | it.effect("explicit retry interrupts the current backoff", () => |
| 889 | Effect.gen(function* () { |
| 890 | const harness = yield* makeHarness({ |
| 891 | prepare: (attempt) => |
| 892 | attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION), |
| 893 | }); |
| 894 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 895 | initiallyDesired: true, |
| 896 | }).pipe(Effect.provide(harness.dependencies)); |
| 897 | |
| 898 | yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 899 | yield* supervisor.retryNow; |
| 900 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 901 | |
| 902 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 903 | }), |
| 904 | ); |
| 905 | |
| 906 | it.effect.each([ |
| 907 | { label: "fails", cause: Cause.fail(blocked()) }, |
| 908 | { |
| 909 | label: "fails while interrupted", |
| 910 | cause: Cause.combine(Cause.fail(blocked()), Cause.interrupt()), |
| 911 | }, |
| 912 | ])( |
| 913 | "keeps a blocked host that $label idle until an external signal requests another attempt", |
| 914 | ({ cause }) => |
| 915 | Effect.gen(function* () { |
| 916 | const harness = yield* makeHarness(); |
| 917 | const driver = yield* ConnectionDriver.ConnectionDriver.pipe( |
| 918 | Effect.provide(harness.dependencies), |
| 919 | ); |
| 920 | const connects = yield* Ref.make(0); |
| 921 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 922 | initiallyDesired: true, |
| 923 | }).pipe( |
| 924 | // The driver keeps only the typed error of a failed route, so the cause is mixed here. |
| 925 | Effect.provideService(ConnectionDriver.ConnectionDriver, { |
| 926 | ...driver, |
| 927 | connect: (entry, reportProgress) => |
| 928 | Ref.updateAndGet(connects, (count) => count + 1).pipe( |
| 929 | Effect.flatMap((count) => |
| 930 | count === 1 ? Effect.failCause(cause) : driver.connect(entry, reportProgress), |
| 931 | ), |
| 932 | ), |
| 933 | }), |
| 934 | Effect.provide(harness.dependencies), |
| 935 | ); |
| 936 | |
| 937 | const failed = yield* awaitState(supervisor.state, (state) => state.lastFailure !== null); |
| 938 | expect(failed).toMatchObject({ |
| 939 | phase: "blocked", |
| 940 | lastFailure: { _tag: "ConnectionBlockedError", detail: "Authentication required." }, |
| 941 | }); |
| 942 | yield* TestClock.adjust("1 hour"); |
| 943 | expect(yield* Ref.get(connects)).toBe(1); |
| 944 | |
| 945 | yield* supervisor.retryNow; |
| 946 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 947 | expect(yield* Ref.get(connects)).toBe(2); |
| 948 | }).pipe(Effect.provide(TestClock.layer())), |
| 949 | ); |
| 950 | |
| 951 | it.effect("retries a blocked connection when platform credentials change", () => |
| 952 | Effect.gen(function* () { |
| 953 | const harness = yield* makeHarness({ |
| 954 | prepare: (attempt) => |
| 955 | attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION), |
| 956 | }); |
| 957 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 958 | initiallyDesired: true, |
| 959 | }).pipe(Effect.provide(harness.dependencies)); |
| 960 | |
| 961 | yield* awaitState(supervisor.state, (state) => state.phase === "blocked"); |
| 962 | yield* harness.wake("credentials-changed"); |
| 963 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 964 | |
| 965 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 966 | }), |
| 967 | ); |
| 968 | |
| 969 | it.effect("does not let platform wakeups reset an in-flight attempt", () => |
| 970 | Effect.gen(function* () { |
| 971 | const firstAttemptStarted = yield* Deferred.make<void>(); |
| 972 | const harness = yield* makeHarness({ |
| 973 | prepare: () => |
| 974 | Deferred.succeed(firstAttemptStarted, undefined).pipe(Effect.andThen(Effect.never)), |
| 975 | }); |
| 976 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 977 | initiallyDesired: true, |
| 978 | }).pipe(Effect.provide(harness.dependencies)); |
| 979 | |
| 980 | yield* Deferred.await(firstAttemptStarted); |
| 981 | yield* Effect.all( |
| 982 | [ |
| 983 | harness.wake("credentials-changed"), |
| 984 | harness.wake("application-active"), |
| 985 | harness.wake("credentials-changed"), |
| 986 | ], |
| 987 | { concurrency: "unbounded" }, |
| 988 | ); |
| 989 | yield* Effect.yieldNow; |
| 990 | |
| 991 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 992 | |
| 993 | yield* TestClock.adjust("15 seconds"); |
| 994 | const retrying = yield* awaitState( |
| 995 | supervisor.state, |
| 996 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 997 | ); |
| 998 | |
| 999 | expect(retrying).toMatchObject({ |
| 1000 | lastFailure: { |
| 1001 | _tag: "ConnectionTransientError", |
| 1002 | reason: "timeout", |
| 1003 | message: "Test environment did not respond during connection setup.", |
| 1004 | }, |
| 1005 | }); |
| 1006 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 1007 | expect(yield* Ref.get(harness.sessionCount)).toBe(0); |
| 1008 | }).pipe(Effect.provide(TestClock.layer())), |
| 1009 | ); |
| 1010 | |
| 1011 | it.effect("reports a slow SSH preparation failure instead of the setup timeout", () => |
| 1012 | Effect.gen(function* () { |
| 1013 | const preparationFailure = new ConnectionTransientError({ |
| 1014 | reason: "remote-unavailable", |
| 1015 | detail: |
| 1016 | "Could not prepare the SSH environment: curl: (28) Operation timed out after 240003 milliseconds", |
| 1017 | }); |
| 1018 | const harness = yield* makeHarness({ |
| 1019 | prepare: () => |
| 1020 | Effect.sleep("4 minutes").pipe(Effect.andThen(Effect.fail(preparationFailure))), |
| 1021 | }); |
| 1022 | const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, { |
| 1023 | initiallyDesired: true, |
| 1024 | }).pipe(Effect.provide(harness.dependencies)); |
| 1025 | |
| 1026 | yield* awaitState( |
| 1027 | supervisor.state, |
| 1028 | (state) => state.phase === "connecting" && state.stage === "preparing", |
| 1029 | ); |
| 1030 | const firstFailure = yield* awaitState( |
| 1031 | supervisor.state, |
| 1032 | (state) => state.phase === "backoff", |
| 1033 | ).pipe(Effect.forkChild({ startImmediately: true })); |
| 1034 | yield* TestClock.adjust("4 minutes"); |
| 1035 | |
| 1036 | expect(yield* Fiber.join(firstFailure)).toMatchObject({ |
| 1037 | attempt: 1, |
| 1038 | lastFailure: { |
| 1039 | _tag: "ConnectionTransientError", |
| 1040 | reason: "remote-unavailable", |
| 1041 | message: preparationFailure.message, |
| 1042 | }, |
| 1043 | }); |
| 1044 | }).pipe(Effect.provide(TestClock.layer())), |
| 1045 | ); |
| 1046 | |
| 1047 | it.effect("times out an SSH route that stops answering after a slow preparation", () => |
| 1048 | Effect.gen(function* () { |
| 1049 | const harness = yield* makeHarness({ |
| 1050 | prepare: () => Effect.sleep("4 minutes").pipe(Effect.as(PREPARED_CONNECTION)), |
| 1051 | ready: () => Effect.never, |
| 1052 | }); |
| 1053 | const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, { |
| 1054 | initiallyDesired: true, |
| 1055 | }).pipe(Effect.provide(harness.dependencies)); |
| 1056 | |
| 1057 | yield* awaitState( |
| 1058 | supervisor.state, |
| 1059 | (state) => state.phase === "connecting" && state.stage === "preparing", |
| 1060 | ); |
| 1061 | yield* TestClock.adjust("4 minutes"); |
| 1062 | yield* awaitState( |
| 1063 | supervisor.state, |
| 1064 | (state) => state.phase === "connecting" && state.stage === "synchronizing", |
| 1065 | ); |
| 1066 | const firstFailure = yield* awaitState( |
| 1067 | supervisor.state, |
| 1068 | (state) => state.phase === "backoff", |
| 1069 | ).pipe(Effect.forkChild({ startImmediately: true })); |
| 1070 | yield* TestClock.adjust("90 seconds"); |
| 1071 | |
| 1072 | expect(yield* Fiber.join(firstFailure)).toMatchObject({ |
| 1073 | attempt: 1, |
| 1074 | lastFailure: { |
| 1075 | _tag: "ConnectionTransientError", |
| 1076 | reason: "timeout", |
| 1077 | message: "Test environment did not respond during connection setup.", |
| 1078 | }, |
| 1079 | }); |
| 1080 | }).pipe(Effect.provide(TestClock.layer())), |
| 1081 | ); |
| 1082 | |
| 1083 | it.effect("does not time a second SSH route's slow preparation", () => |
| 1084 | Effect.gen(function* () { |
| 1085 | const secondSshTarget = new SshConnectionTarget({ |
| 1086 | environmentId: TARGET.environmentId, |
| 1087 | label: TARGET.label, |
| 1088 | connectionId: "ssh-connection-2", |
| 1089 | }); |
| 1090 | const firstRouteFailure = transient("The first SSH route dropped while synchronizing."); |
| 1091 | const harness = yield* makeHarness({ |
| 1092 | prepare: (attempt, target) => |
| 1093 | attempt === 1 |
| 1094 | ? Effect.succeed(preparedFor(target)) |
| 1095 | : Effect.sleep("4 minutes").pipe( |
| 1096 | Effect.andThen( |
| 1097 | Effect.fail( |
| 1098 | new ConnectionTransientError({ |
| 1099 | reason: "remote-unavailable", |
| 1100 | detail: "Could not prepare the SSH environment.", |
| 1101 | }), |
| 1102 | ), |
| 1103 | ), |
| 1104 | ), |
| 1105 | ready: () => Effect.fail(firstRouteFailure), |
| 1106 | }); |
| 1107 | const supervisor = yield* EnvironmentSupervisor.make( |
| 1108 | { ...SSH_ENTRY, alternateRoutes: [{ target: secondSshTarget, profile: Option.none() }] }, |
| 1109 | { initiallyDesired: true }, |
| 1110 | ).pipe(Effect.provide(harness.dependencies)); |
| 1111 | |
| 1112 | yield* awaitState( |
| 1113 | supervisor.state, |
| 1114 | (state) => state.phase === "connecting" && state.stage === "preparing", |
| 1115 | ); |
| 1116 | const firstFailure = yield* awaitState( |
| 1117 | supervisor.state, |
| 1118 | (state) => state.phase === "backoff", |
| 1119 | ).pipe(Effect.forkChild({ startImmediately: true })); |
| 1120 | yield* TestClock.adjust("4 minutes"); |
| 1121 | |
| 1122 | // The driver reports the first transient failure across its routes. |
| 1123 | expect(yield* Fiber.join(firstFailure)).toMatchObject({ |
| 1124 | attempt: 1, |
| 1125 | lastFailure: { reason: "transport", message: firstRouteFailure.message }, |
| 1126 | }); |
| 1127 | expect(yield* Ref.get(harness.prepareCount)).toBe(2); |
| 1128 | }).pipe(Effect.provide(TestClock.layer())), |
| 1129 | ); |
| 1130 | |
| 1131 | it.effect("treats an involuntary session close as transient and reconnects", () => |
| 1132 | Effect.gen(function* () { |
| 1133 | const harness = yield* makeHarness(); |
| 1134 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1135 | initiallyDesired: true, |
| 1136 | }).pipe(Effect.provide(harness.dependencies)); |
| 1137 | |
| 1138 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1139 | yield* harness.closeLatestSession(); |
| 1140 | yield* awaitState( |
| 1141 | supervisor.state, |
| 1142 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 1143 | ); |
| 1144 | expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true); |
| 1145 | |
| 1146 | yield* TestClock.adjust("250 millis"); |
| 1147 | yield* awaitState( |
| 1148 | supervisor.state, |
| 1149 | (state) => state.phase === "connected" && state.generation === 2, |
| 1150 | ); |
| 1151 | |
| 1152 | expect(yield* Ref.get(harness.sessionCount)).toBe(2); |
| 1153 | expect(Option.isSome(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true); |
| 1154 | }).pipe(Effect.provide(TestClock.layer())), |
| 1155 | ); |
| 1156 | |
| 1157 | it.effect("does not report a session lost while suspended as a failed connection", () => |
| 1158 | Effect.gen(function* () { |
| 1159 | const allowReconnect = yield* Deferred.make<void>(); |
| 1160 | const harness = yield* makeHarness({ |
| 1161 | prepare: (attempt) => |
| 1162 | attempt === 2 |
| 1163 | ? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION)) |
| 1164 | : Effect.succeed(PREPARED_CONNECTION), |
| 1165 | }); |
| 1166 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1167 | initiallyDesired: true, |
| 1168 | }).pipe(Effect.provide(harness.dependencies)); |
| 1169 | |
| 1170 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1171 | // The OS killed the socket while the app was suspended; its close arrives |
| 1172 | // before the app reports it is active again. |
| 1173 | yield* harness.closeLatestSession(); |
| 1174 | const waiting = yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 1175 | expect(presentConnectionState(waiting)).toEqual({ |
| 1176 | phase: "reconnecting", |
| 1177 | error: null, |
| 1178 | traceId: null, |
| 1179 | }); |
| 1180 | |
| 1181 | yield* harness.wake("application-active"); |
| 1182 | const replacing = yield* awaitState( |
| 1183 | supervisor.state, |
| 1184 | (state) => state.phase === "connecting" && state.stage === "preparing", |
| 1185 | ); |
| 1186 | expect(presentConnectionState(replacing).error).toBeNull(); |
| 1187 | |
| 1188 | yield* Deferred.succeed(allowReconnect, undefined); |
| 1189 | yield* awaitState( |
| 1190 | supervisor.state, |
| 1191 | (state) => state.phase === "connected" && state.generation === 2, |
| 1192 | ); |
| 1193 | }).pipe(Effect.provide(TestClock.layer())), |
| 1194 | ); |
| 1195 | |
| 1196 | it.effect("restarts the retry ladder when a resume replaces a silent socket", () => |
| 1197 | Effect.gen(function* () { |
| 1198 | const harness = yield* makeHarness({ |
| 1199 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1200 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1201 | probe: (attempt) => (attempt === 2 ? Effect.never : Effect.void), |
| 1202 | checkHost: () => Effect.succeed("answered"), |
| 1203 | }); |
| 1204 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1205 | initiallyDesired: true, |
| 1206 | }).pipe(Effect.provide(harness.dependencies)); |
| 1207 | |
| 1208 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1209 | yield* harness.closeLatestSession(); |
| 1210 | yield* awaitState( |
| 1211 | supervisor.state, |
| 1212 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 1213 | ); |
| 1214 | yield* TestClock.adjust("2 seconds"); |
| 1215 | yield* awaitState( |
| 1216 | supervisor.state, |
| 1217 | (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 2, |
| 1218 | ); |
| 1219 | |
| 1220 | yield* harness.wake("application-active-long"); |
| 1221 | yield* TestClock.adjust("500 millis"); |
| 1222 | yield* awaitState( |
| 1223 | supervisor.state, |
| 1224 | (state) => state.phase === "connected" && state.generation === 3 && state.attempt === 1, |
| 1225 | ); |
| 1226 | }).pipe(Effect.provide(TestClock.layer())), |
| 1227 | ); |
| 1228 | |
| 1229 | it.effect("restarts the retry ladder when a long resume interrupts connection setup", () => |
| 1230 | Effect.gen(function* () { |
| 1231 | const harness = yield* makeHarness({ |
| 1232 | prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)), |
| 1233 | }); |
| 1234 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1235 | initiallyDesired: true, |
| 1236 | }).pipe(Effect.provide(harness.dependencies)); |
| 1237 | |
| 1238 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1239 | yield* harness.closeLatestSession(); |
| 1240 | yield* awaitState( |
| 1241 | supervisor.state, |
| 1242 | (state) => state.phase === "backoff" && state.attempt === 1, |
| 1243 | ); |
| 1244 | yield* TestClock.adjust("3 seconds"); |
| 1245 | yield* awaitState( |
| 1246 | supervisor.state, |
| 1247 | (state) => state.phase === "connecting" && state.attempt === 2, |
| 1248 | ); |
| 1249 | |
| 1250 | yield* harness.wake("application-active-long"); |
| 1251 | yield* awaitState( |
| 1252 | supervisor.state, |
| 1253 | (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1, |
| 1254 | ); |
| 1255 | }).pipe(Effect.provide(TestClock.layer())), |
| 1256 | ); |
| 1257 | |
| 1258 | it.effect("probes the active session without reconnecting on application activation", () => |
| 1259 | Effect.gen(function* () { |
| 1260 | const probeCount = yield* Ref.make(0); |
| 1261 | const probeCalled = yield* Deferred.make<void>(); |
| 1262 | const harness = yield* makeHarness({ |
| 1263 | probe: () => |
| 1264 | Ref.update(probeCount, (count) => count + 1).pipe( |
| 1265 | Effect.andThen(Deferred.succeed(probeCalled, undefined)), |
| 1266 | ), |
| 1267 | }); |
| 1268 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1269 | initiallyDesired: true, |
| 1270 | }).pipe(Effect.provide(harness.dependencies)); |
| 1271 | |
| 1272 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1273 | yield* harness.wake("application-active"); |
| 1274 | yield* Deferred.await(probeCalled); |
| 1275 | |
| 1276 | expect(yield* Ref.get(probeCount)).toBe(1); |
| 1277 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1278 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 1279 | expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); |
| 1280 | }), |
| 1281 | ); |
| 1282 | |
| 1283 | it.effect("keeps a live mobile session after a long background resume", () => |
| 1284 | Effect.gen(function* () { |
| 1285 | const probeCount = yield* Ref.make(0); |
| 1286 | const harness = yield* makeHarness({ |
| 1287 | probe: () => Ref.update(probeCount, (count) => count + 1), |
| 1288 | }); |
| 1289 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1290 | initiallyDesired: true, |
| 1291 | }).pipe(Effect.provide(harness.dependencies)); |
| 1292 | |
| 1293 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1294 | yield* harness.wake("application-active-long"); |
| 1295 | yield* TestClock.adjust("5 seconds"); |
| 1296 | |
| 1297 | expect(yield* Ref.get(probeCount)).toBe(1); |
| 1298 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1299 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 1300 | phase: "connected", |
| 1301 | generation: 1, |
| 1302 | }); |
| 1303 | }).pipe(Effect.provide(TestClock.layer())), |
| 1304 | ); |
| 1305 | |
| 1306 | const makeSilentSocketHarness = (options?: { |
| 1307 | readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>; |
| 1308 | readonly checkHost?: (check: number) => ConnectionDriver.RouteCheck; |
| 1309 | }) => |
| 1310 | Effect.gen(function* () { |
| 1311 | const hostChecks = yield* Ref.make<ReadonlyArray<string>>([]); |
| 1312 | const harness = yield* makeHarness({ |
| 1313 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1314 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1315 | probe: options?.probe ?? ((attempt) => (attempt === 1 ? Effect.never : Effect.void)), |
| 1316 | checkHost: (prepared) => |
| 1317 | Ref.updateAndGet(hostChecks, (checks) => [...checks, prepared.target._tag]).pipe( |
| 1318 | Effect.map((checks) => options?.checkHost?.(checks.length) ?? "answered"), |
| 1319 | ), |
| 1320 | }); |
| 1321 | return { ...harness, hostChecks }; |
| 1322 | }); |
| 1323 | |
| 1324 | it.effect.each(["application-active-probe", "application-active-long"] as const)( |
| 1325 | "replaces a LAN socket that stays silent while its host answers afresh (%s)", |
| 1326 | (wakeup) => |
| 1327 | Effect.gen(function* () { |
| 1328 | const harness = yield* makeSilentSocketHarness(); |
| 1329 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1330 | initiallyDesired: true, |
| 1331 | }).pipe(Effect.provide(harness.dependencies)); |
| 1332 | |
| 1333 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1334 | yield* harness.wake(wakeup); |
| 1335 | yield* TestClock.adjust("499 millis"); |
| 1336 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1337 | |
| 1338 | yield* TestClock.adjust("1 millis"); |
| 1339 | yield* awaitState( |
| 1340 | supervisor.state, |
| 1341 | (state) => state.phase === "connected" && state.generation === 2, |
| 1342 | ); |
| 1343 | expect(yield* Ref.get(harness.hostChecks)).toEqual(["BearerConnectionTarget"]); |
| 1344 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1345 | }).pipe(Effect.provide(TestClock.layer())), |
| 1346 | ); |
| 1347 | |
| 1348 | it.effect("replaces a remote socket only after 2.5 seconds of silence", () => |
| 1349 | Effect.gen(function* () { |
| 1350 | const harness = yield* makeSilentSocketHarness(); |
| 1351 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1352 | initiallyDesired: true, |
| 1353 | }).pipe(Effect.provide(harness.dependencies)); |
| 1354 | |
| 1355 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1356 | yield* harness.wake("application-active-probe"); |
| 1357 | yield* TestClock.adjust("2499 millis"); |
| 1358 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1359 | |
| 1360 | yield* TestClock.adjust("1 millis"); |
| 1361 | yield* awaitState( |
| 1362 | supervisor.state, |
| 1363 | (state) => state.phase === "connected" && state.generation === 2, |
| 1364 | ); |
| 1365 | expect(yield* Ref.get(harness.hostChecks)).toEqual(["RelayConnectionTarget"]); |
| 1366 | }).pipe(Effect.provide(TestClock.layer())), |
| 1367 | ); |
| 1368 | |
| 1369 | it.effect("replaces a silent remote socket without the wait after a long resume", () => |
| 1370 | Effect.gen(function* () { |
| 1371 | const harness = yield* makeSilentSocketHarness(); |
| 1372 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1373 | initiallyDesired: true, |
| 1374 | }).pipe(Effect.provide(harness.dependencies)); |
| 1375 | |
| 1376 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1377 | yield* harness.wake("application-active-long"); |
| 1378 | yield* TestClock.adjust("499 millis"); |
| 1379 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1380 | |
| 1381 | yield* TestClock.adjust("1 millis"); |
| 1382 | yield* awaitState( |
| 1383 | supervisor.state, |
| 1384 | (state) => state.phase === "connected" && state.generation === 2, |
| 1385 | ); |
| 1386 | }).pipe(Effect.provide(TestClock.layer())), |
| 1387 | ); |
| 1388 | |
| 1389 | it.effect("replaces a silent socket even while the network reports offline", () => |
| 1390 | Effect.gen(function* () { |
| 1391 | const harness = yield* makeSilentSocketHarness(); |
| 1392 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1393 | initiallyDesired: true, |
| 1394 | }).pipe(Effect.provide(harness.dependencies)); |
| 1395 | |
| 1396 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1397 | yield* harness.setNetworkStatus("offline"); |
| 1398 | yield* TestClock.adjust("500 millis"); |
| 1399 | yield* awaitState( |
| 1400 | supervisor.state, |
| 1401 | (state) => state.phase === "connected" && state.generation === 2, |
| 1402 | ); |
| 1403 | }).pipe(Effect.provide(TestClock.layer())), |
| 1404 | ); |
| 1405 | |
| 1406 | it.effect("checks the primary environment's socket when the device reports offline", () => |
| 1407 | Effect.gen(function* () { |
| 1408 | const harness = yield* makeSilentSocketHarness(); |
| 1409 | const target = new PrimaryConnectionTarget({ |
| 1410 | ...TARGET, |
| 1411 | httpBaseUrl: "http://127.0.0.1:3773", |
| 1412 | wsBaseUrl: "ws://127.0.0.1:3773", |
| 1413 | }); |
| 1414 | const supervisor = yield* EnvironmentSupervisor.make( |
| 1415 | { ...TARGET_ENTRY, target }, |
| 1416 | { initiallyDesired: true }, |
| 1417 | ).pipe(Effect.provide(harness.dependencies)); |
| 1418 | |
| 1419 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1420 | yield* harness.setNetworkStatus("offline"); |
| 1421 | yield* TestClock.adjust("500 millis"); |
| 1422 | yield* awaitState( |
| 1423 | supervisor.state, |
| 1424 | (state) => state.phase === "connected" && state.generation === 2, |
| 1425 | ); |
| 1426 | }).pipe(Effect.provide(TestClock.layer())), |
| 1427 | ); |
| 1428 | |
| 1429 | it.effect("replaces a remote socket after a long resume while the network reports offline", () => |
| 1430 | Effect.gen(function* () { |
| 1431 | const harness = yield* makeSilentSocketHarness(); |
| 1432 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1433 | initiallyDesired: true, |
| 1434 | }).pipe(Effect.provide(harness.dependencies)); |
| 1435 | |
| 1436 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1437 | yield* harness.setNetworkStatus("offline"); |
| 1438 | yield* harness.wake("application-active-long"); |
| 1439 | yield* TestClock.adjust("2500 millis"); |
| 1440 | yield* awaitState( |
| 1441 | supervisor.state, |
| 1442 | (state) => state.phase === "connected" && state.generation === 2, |
| 1443 | ); |
| 1444 | }).pipe(Effect.provide(TestClock.layer())), |
| 1445 | ); |
| 1446 | |
| 1447 | it.effect("checks the host afresh when the network changes after a frame", () => |
| 1448 | Effect.gen(function* () { |
| 1449 | const frames = yield* Ref.make(0); |
| 1450 | const harness = yield* makeHarness({ |
| 1451 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1452 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1453 | probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void), |
| 1454 | inboundFrames: Ref.get(frames), |
| 1455 | checkHost: () => Effect.succeed("answered"), |
| 1456 | }); |
| 1457 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1458 | initiallyDesired: true, |
| 1459 | }).pipe(Effect.provide(harness.dependencies)); |
| 1460 | |
| 1461 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1462 | yield* harness.wake("application-active-probe"); |
| 1463 | yield* TestClock.adjust("1 second"); |
| 1464 | yield* harness.wake("network-changed"); |
| 1465 | yield* TestClock.adjust("500 millis"); |
| 1466 | yield* Ref.update(frames, (count) => count + 1); |
| 1467 | // The running check sees the frame at 1.6 s, so a fresh one starts then. |
| 1468 | yield* TestClock.adjust("2599 millis"); |
| 1469 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1470 | |
| 1471 | yield* TestClock.adjust("1 millis"); |
| 1472 | yield* awaitState( |
| 1473 | supervisor.state, |
| 1474 | (state) => state.phase === "connected" && state.generation === 2, |
| 1475 | ); |
| 1476 | }).pipe(Effect.provide(TestClock.layer())), |
| 1477 | ); |
| 1478 | |
| 1479 | it.effect("keeps a running check's window when the network changes during a long resume", () => |
| 1480 | Effect.gen(function* () { |
| 1481 | const harness = yield* makeSilentSocketHarness(); |
| 1482 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1483 | initiallyDesired: true, |
| 1484 | }).pipe(Effect.provide(harness.dependencies)); |
| 1485 | |
| 1486 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1487 | yield* harness.wake("application-active-long"); |
| 1488 | yield* TestClock.adjust("1 second"); |
| 1489 | yield* harness.wake("network-changed"); |
| 1490 | yield* harness.setNetworkStatus("offline"); |
| 1491 | yield* TestClock.adjust("1500 millis"); |
| 1492 | yield* awaitState( |
| 1493 | supervisor.state, |
| 1494 | (state) => state.phase === "connected" && state.generation === 2, |
| 1495 | ); |
| 1496 | }).pipe(Effect.provide(TestClock.layer())), |
| 1497 | ); |
| 1498 | |
| 1499 | it.effect("checks the host again when one check goes unanswered", () => |
| 1500 | Effect.gen(function* () { |
| 1501 | const harness = yield* makeSilentSocketHarness({ |
| 1502 | checkHost: (check) => (check === 1 ? "silent" : "answered"), |
| 1503 | }); |
| 1504 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1505 | initiallyDesired: true, |
| 1506 | }).pipe(Effect.provide(harness.dependencies)); |
| 1507 | |
| 1508 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1509 | yield* harness.wake("application-active-probe"); |
| 1510 | yield* TestClock.adjust("1499 millis"); |
| 1511 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1512 | |
| 1513 | yield* TestClock.adjust("1 millis"); |
| 1514 | yield* awaitState( |
| 1515 | supervisor.state, |
| 1516 | (state) => state.phase === "connected" && state.generation === 2, |
| 1517 | ); |
| 1518 | expect(yield* Ref.get(harness.hostChecks)).toHaveLength(2); |
| 1519 | }).pipe(Effect.provide(TestClock.layer())), |
| 1520 | ); |
| 1521 | |
| 1522 | it.effect("leaves a web or desktop foreground probe to its own deadline", () => |
| 1523 | Effect.gen(function* () { |
| 1524 | const harness = yield* makeSilentSocketHarness({ |
| 1525 | probe: (attempt) => (attempt === 1 ? Effect.sleep("4 seconds") : Effect.void), |
| 1526 | }); |
| 1527 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1528 | initiallyDesired: true, |
| 1529 | }).pipe(Effect.provide(harness.dependencies)); |
| 1530 | |
| 1531 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1532 | yield* harness.wake("application-active"); |
| 1533 | yield* TestClock.adjust("20 seconds"); |
| 1534 | |
| 1535 | expect(yield* Ref.get(harness.hostChecks)).toEqual([]); |
| 1536 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1537 | }).pipe(Effect.provide(TestClock.layer())), |
| 1538 | ); |
| 1539 | |
| 1540 | it.effect("checks the host when a network change arrives during a foreground probe", () => |
| 1541 | Effect.gen(function* () { |
| 1542 | const harness = yield* makeSilentSocketHarness(); |
| 1543 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1544 | initiallyDesired: true, |
| 1545 | }).pipe(Effect.provide(harness.dependencies)); |
| 1546 | |
| 1547 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1548 | yield* harness.wake("application-active"); |
| 1549 | yield* TestClock.adjust("100 millis"); |
| 1550 | yield* harness.wake("network-changed"); |
| 1551 | yield* TestClock.adjust("499 millis"); |
| 1552 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1553 | |
| 1554 | yield* TestClock.adjust("1 millis"); |
| 1555 | yield* awaitState( |
| 1556 | supervisor.state, |
| 1557 | (state) => state.phase === "connected" && state.generation === 2, |
| 1558 | ); |
| 1559 | }).pipe(Effect.provide(TestClock.layer())), |
| 1560 | ); |
| 1561 | |
| 1562 | it.effect.each(["application-active-probe", "application-active-long"] as const)( |
| 1563 | "keeps waiting for the probe when a frame arrives during the host check (%s)", |
| 1564 | (wakeup) => |
| 1565 | Effect.gen(function* () { |
| 1566 | const frames = yield* Ref.make(0); |
| 1567 | const harness = yield* makeHarness({ |
| 1568 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1569 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1570 | probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void), |
| 1571 | inboundFrames: Ref.get(frames), |
| 1572 | checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")), |
| 1573 | }); |
| 1574 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1575 | initiallyDesired: true, |
| 1576 | }).pipe(Effect.provide(harness.dependencies)); |
| 1577 | |
| 1578 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1579 | yield* harness.wake(wakeup); |
| 1580 | yield* TestClock.adjust("5 seconds"); |
| 1581 | |
| 1582 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1583 | expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); |
| 1584 | }).pipe(Effect.provide(TestClock.layer())), |
| 1585 | ); |
| 1586 | |
| 1587 | it.effect("keeps a long-resume socket that delivers a frame during a slow host check", () => |
| 1588 | Effect.gen(function* () { |
| 1589 | const frames = yield* Ref.make(0); |
| 1590 | const harness = yield* makeHarness({ |
| 1591 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1592 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1593 | probe: (attempt) => (attempt === 1 ? Effect.sleep("2500 millis") : Effect.void), |
| 1594 | inboundFrames: Ref.get(frames), |
| 1595 | checkHost: () => Effect.sleep("2 seconds").pipe(Effect.as("answered")), |
| 1596 | }); |
| 1597 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1598 | initiallyDesired: true, |
| 1599 | }).pipe(Effect.provide(harness.dependencies)); |
| 1600 | |
| 1601 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1602 | yield* harness.wake("application-active-long"); |
| 1603 | yield* TestClock.adjust("700 millis"); |
| 1604 | yield* Ref.update(frames, (count) => count + 1); |
| 1605 | yield* TestClock.adjust("5 seconds"); |
| 1606 | |
| 1607 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1608 | expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); |
| 1609 | }).pipe(Effect.provide(TestClock.layer())), |
| 1610 | ); |
| 1611 | |
| 1612 | it.effect("waits out a LAN resume probe when the host does not answer a fresh request", () => |
| 1613 | Effect.gen(function* () { |
| 1614 | const harness = yield* makeHarness({ |
| 1615 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 1616 | initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG), |
| 1617 | probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void), |
| 1618 | checkHost: () => Effect.succeed("silent"), |
| 1619 | }); |
| 1620 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1621 | initiallyDesired: true, |
| 1622 | }).pipe(Effect.provide(harness.dependencies)); |
| 1623 | |
| 1624 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1625 | yield* harness.wake("application-active-probe"); |
| 1626 | yield* TestClock.adjust("2999 millis"); |
| 1627 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1628 | |
| 1629 | yield* TestClock.adjust("1 millis"); |
| 1630 | yield* awaitState( |
| 1631 | supervisor.state, |
| 1632 | (state) => state.phase === "connected" && state.generation === 2, |
| 1633 | ); |
| 1634 | }).pipe(Effect.provide(TestClock.layer())), |
| 1635 | ); |
| 1636 | |
| 1637 | it.effect.each(["application-active-probe", "network-changed"] as const)( |
| 1638 | "gives a live remote socket 15 seconds to answer a %s probe", |
| 1639 | (wakeup) => |
| 1640 | Effect.gen(function* () { |
| 1641 | // A slow path delays the fresh request as much as the probe. |
| 1642 | const harness = yield* makeSilentSocketHarness({ |
| 1643 | probe: (attempt) => (attempt === 1 ? Effect.sleep("4 seconds") : Effect.void), |
| 1644 | checkHost: () => "silent", |
| 1645 | }); |
| 1646 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1647 | initiallyDesired: true, |
| 1648 | }).pipe(Effect.provide(harness.dependencies)); |
| 1649 | |
| 1650 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1651 | yield* harness.wake(wakeup); |
| 1652 | yield* TestClock.adjust("20 seconds"); |
| 1653 | |
| 1654 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1655 | expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); |
| 1656 | }).pipe(Effect.provide(TestClock.layer())), |
| 1657 | ); |
| 1658 | |
| 1659 | it.effect("replaces an unanswered long resume probe even while the network reports offline", () => |
| 1660 | Effect.gen(function* () { |
| 1661 | const harness = yield* makeHarness({ |
| 1662 | probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void), |
| 1663 | }); |
| 1664 | const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, { |
| 1665 | initiallyDesired: true, |
| 1666 | }).pipe(Effect.provide(harness.dependencies)); |
| 1667 | |
| 1668 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1669 | yield* harness.wake("application-active-long"); |
| 1670 | yield* harness.setNetworkStatus("offline"); |
| 1671 | yield* TestClock.adjust("1499 millis"); |
| 1672 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1673 | |
| 1674 | yield* TestClock.adjust("1 millis"); |
| 1675 | const reconnected = yield* awaitState( |
| 1676 | supervisor.state, |
| 1677 | (state) => state.phase === "connected" && state.generation === 2, |
| 1678 | ); |
| 1679 | expect(reconnected.lastFailure).toBeNull(); |
| 1680 | }).pipe(Effect.provide(TestClock.layer())), |
| 1681 | ); |
| 1682 | |
| 1683 | it.effect("reconnects immediately when the session closes during a resume probe", () => |
| 1684 | Effect.gen(function* () { |
| 1685 | const probeStarted = yield* Deferred.make<void>(); |
| 1686 | const harness = yield* makeHarness({ |
| 1687 | probe: (attempt) => |
| 1688 | attempt === 1 |
| 1689 | ? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)) |
| 1690 | : Effect.void, |
| 1691 | }); |
| 1692 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1693 | initiallyDesired: true, |
| 1694 | }).pipe(Effect.provide(harness.dependencies)); |
| 1695 | |
| 1696 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1697 | yield* harness.wake("application-active-probe"); |
| 1698 | yield* Deferred.await(probeStarted); |
| 1699 | yield* harness.closeLatestSession(); |
| 1700 | |
| 1701 | const reconnected = yield* awaitState( |
| 1702 | supervisor.state, |
| 1703 | (state) => state.phase === "connected" && state.generation === 2, |
| 1704 | ); |
| 1705 | expect(reconnected.attempt).toBe(1); |
| 1706 | expect(yield* Ref.get(harness.sessionCount)).toBe(2); |
| 1707 | }).pipe(Effect.provide(TestClock.layer())), |
| 1708 | ); |
| 1709 | |
| 1710 | it.effect("reconnects immediately when the foreground liveness probe fails", () => |
| 1711 | Effect.gen(function* () { |
| 1712 | const allowReconnect = yield* Deferred.make<void>(); |
| 1713 | const harness = yield* makeHarness({ |
| 1714 | prepare: (attempt) => |
| 1715 | attempt === 2 |
| 1716 | ? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION)) |
| 1717 | : Effect.succeed(PREPARED_CONNECTION), |
| 1718 | probe: (attempt) => |
| 1719 | attempt === 1 ? Effect.fail(transient("The live session is stale.")) : Effect.void, |
| 1720 | }); |
| 1721 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1722 | initiallyDesired: true, |
| 1723 | }).pipe(Effect.provide(harness.dependencies)); |
| 1724 | |
| 1725 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1726 | yield* harness.wake("application-active"); |
| 1727 | const reconnecting = yield* awaitState( |
| 1728 | supervisor.state, |
| 1729 | (state) => state.phase === "connecting", |
| 1730 | ); |
| 1731 | expect(reconnecting.attempt).toBe(1); |
| 1732 | expect(reconnecting.lastFailure).toBeNull(); |
| 1733 | expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true); |
| 1734 | |
| 1735 | yield* Deferred.succeed(allowReconnect, undefined); |
| 1736 | yield* awaitState( |
| 1737 | supervisor.state, |
| 1738 | (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1, |
| 1739 | ); |
| 1740 | |
| 1741 | expect(yield* Ref.get(harness.sessionCount)).toBe(2); |
| 1742 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1743 | }).pipe(Effect.provide(TestClock.layer())), |
| 1744 | ); |
| 1745 | |
| 1746 | it.effect("honors an explicit disconnect while a foreground probe is stalled", () => |
| 1747 | Effect.gen(function* () { |
| 1748 | const probeStarted = yield* Deferred.make<void>(); |
| 1749 | const harness = yield* makeHarness({ |
| 1750 | probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), |
| 1751 | }); |
| 1752 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1753 | initiallyDesired: true, |
| 1754 | }).pipe(Effect.provide(harness.dependencies)); |
| 1755 | |
| 1756 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1757 | yield* harness.wake("application-active"); |
| 1758 | yield* Deferred.await(probeStarted); |
| 1759 | yield* supervisor.disconnect; |
| 1760 | yield* awaitState(supervisor.state, (state) => state.phase === "available"); |
| 1761 | |
| 1762 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1763 | }), |
| 1764 | ); |
| 1765 | |
| 1766 | it.effect( |
| 1767 | "keeps a live session when the automatic follow-up to a slow foreground probe answers", |
| 1768 | () => |
| 1769 | Effect.gen(function* () { |
| 1770 | const probeCount = yield* Ref.make(0); |
| 1771 | const probeStarted = yield* Queue.unbounded<number>(); |
| 1772 | const harness = yield* makeHarness({ |
| 1773 | probe: () => |
| 1774 | Ref.updateAndGet(probeCount, (count) => count + 1).pipe( |
| 1775 | Effect.tap((call) => Queue.offer(probeStarted, call)), |
| 1776 | Effect.flatMap((call) => (call === 1 ? Effect.never : Effect.void)), |
| 1777 | ), |
| 1778 | }); |
| 1779 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1780 | initiallyDesired: true, |
| 1781 | }).pipe(Effect.provide(harness.dependencies)); |
| 1782 | |
| 1783 | yield* awaitState( |
| 1784 | supervisor.state, |
| 1785 | (state) => state.phase === "connected" && state.generation === 1, |
| 1786 | ); |
| 1787 | yield* harness.wake("application-active"); |
| 1788 | expect(yield* Queue.take(probeStarted)).toBe(1); |
| 1789 | // A busy server misses the foreground probe deadline once while the |
| 1790 | // socket stays open; the supervisor asks again on its own. |
| 1791 | yield* TestClock.adjust("15 seconds"); |
| 1792 | expect(yield* Queue.take(probeStarted)).toBe(2); |
| 1793 | yield* TestClock.adjust("30 seconds"); |
| 1794 | |
| 1795 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 1796 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1797 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 1798 | phase: "connected", |
| 1799 | generation: 1, |
| 1800 | }); |
| 1801 | }).pipe(Effect.provide(TestClock.layer())), |
| 1802 | ); |
| 1803 | |
| 1804 | it.effect("replaces a session when the automatic follow-up probe also goes unanswered", () => |
| 1805 | Effect.gen(function* () { |
| 1806 | const probeCount = yield* Ref.make(0); |
| 1807 | const harness = yield* makeHarness({ |
| 1808 | probe: () => |
| 1809 | Ref.update(probeCount, (count) => count + 1).pipe(Effect.andThen(Effect.never)), |
| 1810 | }); |
| 1811 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1812 | initiallyDesired: true, |
| 1813 | }).pipe(Effect.provide(harness.dependencies)); |
| 1814 | |
| 1815 | yield* awaitState( |
| 1816 | supervisor.state, |
| 1817 | (state) => state.phase === "connected" && state.generation === 1, |
| 1818 | ); |
| 1819 | // One foreground event and no other signal: the follow-up's miss alone |
| 1820 | // replaces the session. |
| 1821 | yield* harness.wake("application-active"); |
| 1822 | yield* TestClock.adjust("15 seconds"); |
| 1823 | yield* TestClock.adjust("15 seconds"); |
| 1824 | yield* awaitState( |
| 1825 | supervisor.state, |
| 1826 | (state) => state.phase === "connected" && state.generation === 2, |
| 1827 | ); |
| 1828 | |
| 1829 | expect(yield* Ref.get(probeCount)).toBe(2); |
| 1830 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1831 | }).pipe(Effect.provide(TestClock.layer())), |
| 1832 | ); |
| 1833 | |
| 1834 | it.effect("replaces a session when a retry lands late in a slow foreground probe", () => |
| 1835 | Effect.gen(function* () { |
| 1836 | const probeStarted = yield* Deferred.make<void>(); |
| 1837 | const harness = yield* makeHarness({ |
| 1838 | probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)), |
| 1839 | }); |
| 1840 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1841 | initiallyDesired: true, |
| 1842 | }).pipe(Effect.provide(harness.dependencies)); |
| 1843 | |
| 1844 | yield* awaitState( |
| 1845 | supervisor.state, |
| 1846 | (state) => state.phase === "connected" && state.generation === 1, |
| 1847 | ); |
| 1848 | yield* harness.wake("application-active"); |
| 1849 | yield* Deferred.await(probeStarted); |
| 1850 | yield* TestClock.adjust("13 seconds"); |
| 1851 | // The retry's own deadline falls after the foreground one, but it still |
| 1852 | // asks for a definite answer, so the first miss is not forgiven. |
| 1853 | yield* supervisor.retryNow; |
| 1854 | yield* TestClock.adjust("5 seconds"); |
| 1855 | yield* awaitState( |
| 1856 | supervisor.state, |
| 1857 | (state) => state.phase === "connected" && state.generation === 2, |
| 1858 | ); |
| 1859 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1860 | }).pipe(Effect.provide(TestClock.layer())), |
| 1861 | ); |
| 1862 | |
| 1863 | it.effect("does not churn a healthy session when credentials change", () => |
| 1864 | Effect.gen(function* () { |
| 1865 | const harness = yield* makeHarness(); |
| 1866 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 1867 | initiallyDesired: true, |
| 1868 | }).pipe(Effect.provide(harness.dependencies)); |
| 1869 | |
| 1870 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1871 | yield* harness.wake("credentials-changed"); |
| 1872 | yield* Effect.yieldNow; |
| 1873 | |
| 1874 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 1875 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 1876 | expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); |
| 1877 | }), |
| 1878 | ); |
| 1879 | |
| 1880 | it.effect("releases and reconnects a relay session when credentials change", () => |
| 1881 | Effect.gen(function* () { |
| 1882 | const relaySpans: Array<Tracer.NativeSpan> = []; |
| 1883 | const relayTracer = Tracer.make({ |
| 1884 | span(options) { |
| 1885 | const span = new Tracer.NativeSpan(options); |
| 1886 | relaySpans.push(span); |
| 1887 | return span; |
| 1888 | }, |
| 1889 | }); |
| 1890 | const harness = yield* makeHarness({ |
| 1891 | prepare: () => Effect.succeed(preparedFor(RELAY_TARGET)), |
| 1892 | }); |
| 1893 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 1894 | initiallyDesired: true, |
| 1895 | }).pipe( |
| 1896 | Effect.provide(harness.dependencies), |
| 1897 | Effect.provideService(RelayClientTracer, Option.some(relayTracer)), |
| 1898 | ); |
| 1899 | |
| 1900 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 1901 | yield* harness.wake("credentials-changed"); |
| 1902 | yield* awaitState( |
| 1903 | supervisor.state, |
| 1904 | (state) => state.phase === "connected" && state.generation === 2, |
| 1905 | ); |
| 1906 | |
| 1907 | expect(yield* Ref.get(harness.sessionCount)).toBe(2); |
| 1908 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 1909 | const attempts = relaySpans.filter((span) => span.name === "relay.connection.attempt"); |
| 1910 | expect(attempts).toHaveLength(2); |
| 1911 | for (const span of attempts) { |
| 1912 | expect(span.attributes.get("environment.id")).toBe(RELAY_TARGET.environmentId); |
| 1913 | expect(span.attributes.get("environment.target.kind")).toBe(RELAY_TARGET._tag); |
| 1914 | expect(span.attributes.has("environment.label")).toBe(false); |
| 1915 | } |
| 1916 | }), |
| 1917 | ); |
| 1918 | |
| 1919 | it.effect("refreshes HTTP authorization without replacing the active relay session", () => |
| 1920 | Effect.gen(function* () { |
| 1921 | const endpoint = { |
| 1922 | httpBaseUrl: TARGET.httpBaseUrl, |
| 1923 | wsBaseUrl: TARGET.wsBaseUrl, |
| 1924 | providerKind: "cloudflare_tunnel" as const, |
| 1925 | }; |
| 1926 | const token = yield* Ref.make( |
| 1927 | Option.some( |
| 1928 | new TokenStore.RemoteDpopAccessToken({ |
| 1929 | environmentId: TARGET.environmentId, |
| 1930 | accountId: "test-account", |
| 1931 | label: TARGET.label, |
| 1932 | endpoint, |
| 1933 | accessToken: "access-token-1", |
| 1934 | expiresAtEpochMs: 3_600_000, |
| 1935 | dpopThumbprint: "test-thumbprint", |
| 1936 | }), |
| 1937 | ), |
| 1938 | ); |
| 1939 | const bootstrapFails = yield* Ref.make(false); |
| 1940 | const bootstrapCalls = yield* Ref.make(0); |
| 1941 | const httpPaths: Array<string> = []; |
| 1942 | const sessionAuthorizations: Array<string | null> = []; |
| 1943 | const fetchFn = ((input, init) => { |
| 1944 | const request = new Request(input, init); |
| 1945 | const pathname = new URL(request.url).pathname; |
| 1946 | httpPaths.push(pathname); |
| 1947 | switch (pathname) { |
| 1948 | case "/.well-known/t3/environment": |
| 1949 | return Promise.resolve( |
| 1950 | Response.json({ |
| 1951 | environmentId: TARGET.environmentId, |
| 1952 | label: TARGET.label, |
| 1953 | platform: { os: "linux", arch: "x64" }, |
| 1954 | serverVersion: "0.0.0-test", |
| 1955 | capabilities: { repositoryIdentity: true }, |
| 1956 | }), |
| 1957 | ); |
| 1958 | case "/oauth/token": |
| 1959 | return Promise.resolve( |
| 1960 | Response.json({ |
| 1961 | access_token: "access-token-2", |
| 1962 | issued_token_type: "urn:ietf:params:oauth:token-type:access_token", |
| 1963 | token_type: "DPoP", |
| 1964 | expires_in: 3_600, |
| 1965 | scope: AuthStandardClientScopes.join(" "), |
| 1966 | }), |
| 1967 | ); |
| 1968 | case "/api/auth/websocket-ticket/rpc": |
| 1969 | return Promise.resolve( |
| 1970 | Response.json({ |
| 1971 | ticket: "ws-ticket", |
| 1972 | expiresAt: "2026-09-04T01:00:00.000Z", |
| 1973 | }), |
| 1974 | ); |
| 1975 | case "/api/auth/session": { |
| 1976 | const authorization = request.headers.get("authorization"); |
| 1977 | sessionAuthorizations.push(authorization); |
| 1978 | return Promise.resolve( |
| 1979 | Response.json({ |
| 1980 | authenticated: authorization === "DPoP access-token-2", |
| 1981 | auth: { |
| 1982 | policy: "loopback-browser", |
| 1983 | bootstrapMethods: ["one-time-token"], |
| 1984 | sessionMethods: ["dpop-access-token"], |
| 1985 | sessionCookieName: "t3_session_test", |
| 1986 | }, |
| 1987 | scopes: AuthStandardClientScopes, |
| 1988 | }), |
| 1989 | ); |
| 1990 | } |
| 1991 | default: |
| 1992 | return Promise.reject(new Error(`Unexpected HTTP request to ${request.url}`)); |
| 1993 | } |
| 1994 | }) satisfies typeof fetch; |
| 1995 | const signer = ManagedRelay.ManagedRelayDpopSigner.of({ |
| 1996 | thumbprint: Effect.succeed("test-thumbprint"), |
| 1997 | createProof: () => Effect.succeed("test-proof"), |
| 1998 | }); |
| 1999 | const unused = () => Effect.die("Unexpected relay operation."); |
| 2000 | const relay = ManagedRelay.ManagedRelayClient.of({ |
| 2001 | relayUrl: "https://relay.example.test", |
| 2002 | listEnvironments: unused, |
| 2003 | listDevices: unused, |
| 2004 | createEnvironmentLinkChallenge: unused, |
| 2005 | linkEnvironment: unused, |
| 2006 | unlinkEnvironment: unused, |
| 2007 | getT3ChatLink: unused, |
| 2008 | createT3ChatLinkRequest: unused, |
| 2009 | unlinkT3Chat: unused, |
| 2010 | getEnvironmentStatus: unused, |
| 2011 | connectEnvironment: Effect.fn("TestConnectionHttp.connectEnvironment")(function* () { |
| 2012 | yield* Ref.update(bootstrapCalls, (count) => count + 1); |
| 2013 | if (yield* Ref.get(bootstrapFails)) { |
| 2014 | return yield* new ManagedRelay.ManagedRelayRequestTimeoutError({ |
| 2015 | activity: "Relay environment connection", |
| 2016 | timeoutMs: 6_000, |
| 2017 | traceId: null, |
| 2018 | }); |
| 2019 | } |
| 2020 | return { |
| 2021 | environmentId: TARGET.environmentId, |
| 2022 | endpoint, |
| 2023 | credential: "relay-bootstrap", |
| 2024 | expiresAt: "2026-09-04T01:00:00.000Z", |
| 2025 | }; |
| 2026 | }), |
| 2027 | registerDevice: unused, |
| 2028 | unregisterDevice: unused, |
| 2029 | registerLiveActivity: unused, |
| 2030 | getAgentActivitySnapshot: unused, |
| 2031 | resetTokenCache: Effect.void, |
| 2032 | }); |
| 2033 | const layerHttp = RpcHttp.layerRemoteHttpClient(fetchFn); |
| 2034 | const remoteAuthorization = yield* RemoteEnvironmentAuthorization.make.pipe( |
| 2035 | Effect.provide( |
| 2036 | Layer.mergeAll( |
| 2037 | layerHttp, |
| 2038 | Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, signer), |
| 2039 | Layer.succeed(ManagedRelay.ManagedRelayClient, relay), |
| 2040 | Layer.succeed(ClientCapabilities.CloudSession, { |
| 2041 | identity: Effect.succeedSome({ accountId: "test-account" }), |
| 2042 | clerkToken: Effect.succeed("clerk-token"), |
| 2043 | }), |
| 2044 | Layer.succeed(ClientCapabilities.RelayDeviceIdentity, { |
| 2045 | deviceId: Effect.succeedNone, |
| 2046 | }), |
| 2047 | Layer.succeed(TokenStore.RemoteDpopAccessTokenStore, { |
| 2048 | get: () => Ref.get(token), |
| 2049 | put: (value) => Ref.set(token, Option.some(value)), |
| 2050 | remove: () => Ref.set(token, Option.none()), |
| 2051 | }), |
| 2052 | Layer.succeed(ClientCapabilities.ClientPresentation, { |
| 2053 | metadata: { label: "Test client", deviceType: "desktop" }, |
| 2054 | }), |
| 2055 | ConnectionCredentialStore.layerMemory(), |
| 2056 | ), |
| 2057 | ), |
| 2058 | ); |
| 2059 | const harness = yield* makeHarness({ |
| 2060 | prepare: () => |
| 2061 | remoteAuthorization |
| 2062 | .authorizeDpop({ expectedEnvironmentId: TARGET.environmentId }) |
| 2063 | .pipe(Effect.map((prepared) => ({ ...prepared, target: RELAY_TARGET }))), |
| 2064 | }); |
| 2065 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 2066 | initiallyDesired: true, |
| 2067 | }).pipe(Effect.provide(harness.dependencies)); |
| 2068 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2069 | const session = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session)); |
| 2070 | const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)); |
| 2071 | const readSession = fetchEnvironmentSessionState({ |
| 2072 | prepared, |
| 2073 | signer: Option.some(signer), |
| 2074 | remoteAuthorization: Option.some(remoteAuthorization), |
| 2075 | }).pipe(Effect.provide(layerHttp)); |
| 2076 | |
| 2077 | yield* TestClock.adjust("2 hours"); |
| 2078 | expect((yield* readSession).authenticated).toBe(true); |
| 2079 | expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]); |
| 2080 | expect(yield* Ref.get(bootstrapCalls)).toBe(1); |
| 2081 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session); |
| 2082 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 2083 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 2084 | phase: "connected", |
| 2085 | generation: 1, |
| 2086 | }); |
| 2087 | |
| 2088 | yield* TestClock.adjust("2 hours"); |
| 2089 | yield* Ref.set(bootstrapFails, true); |
| 2090 | const failure = yield* readSession.pipe(Effect.flip); |
| 2091 | expect(failure._tag).toBe("RemoteEnvironmentAuthFetchError"); |
| 2092 | expect(yield* Ref.get(bootstrapCalls)).toBe(2); |
| 2093 | expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]); |
| 2094 | expect(httpPaths.filter((path) => path === "/api/auth/websocket-ticket/rpc")).toHaveLength(1); |
| 2095 | expect(httpPaths.filter((path) => path === "/oauth/token")).toHaveLength(1); |
| 2096 | expect(yield* Ref.get(harness.sessionCount)).toBe(1); |
| 2097 | expect(yield* Ref.get(harness.releaseCount)).toBe(0); |
| 2098 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session); |
| 2099 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 2100 | phase: "connected", |
| 2101 | generation: 1, |
| 2102 | }); |
| 2103 | }).pipe(Effect.provide(TestClock.layer())), |
| 2104 | ); |
| 2105 | |
| 2106 | it.effect("does not lose an explicit disconnect among concurrent wakeup signals", () => |
| 2107 | Effect.gen(function* () { |
| 2108 | const harness = yield* makeHarness(); |
| 2109 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 2110 | initiallyDesired: true, |
| 2111 | }).pipe(Effect.provide(harness.dependencies)); |
| 2112 | |
| 2113 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2114 | yield* Effect.all( |
| 2115 | [ |
| 2116 | supervisor.disconnect, |
| 2117 | harness.wake("credentials-changed"), |
| 2118 | harness.wake("application-active"), |
| 2119 | harness.wake("credentials-changed"), |
| 2120 | ], |
| 2121 | { concurrency: "unbounded" }, |
| 2122 | ); |
| 2123 | yield* awaitState(supervisor.state, (state) => state.phase === "available"); |
| 2124 | |
| 2125 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 2126 | expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true); |
| 2127 | }), |
| 2128 | ); |
| 2129 | }); |
| 2130 | |
| 2131 | describe("EnvironmentSupervisor routes", () => { |
| 2132 | it.effect("connects over relay before a slow LAN reachability check times out", () => |
| 2133 | Effect.gen(function* () { |
| 2134 | const checkingLan = yield* Deferred.make<void>(); |
| 2135 | const harness = yield* makeHarness({ |
| 2136 | checkRoute: (route) => |
| 2137 | route.target._tag === "BearerConnectionTarget" |
| 2138 | ? Deferred.succeed(checkingLan, undefined).pipe( |
| 2139 | Effect.andThen(Effect.sleep("2500 millis")), |
| 2140 | Effect.as("silent" as const), |
| 2141 | ) |
| 2142 | : Effect.succeed("unchecked"), |
| 2143 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2144 | }); |
| 2145 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2146 | initiallyDesired: true, |
| 2147 | }).pipe(Effect.provide(harness.dependencies)); |
| 2148 | |
| 2149 | yield* Deferred.await(checkingLan); |
| 2150 | yield* TestClock.adjust("500 millis"); |
| 2151 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2152 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2153 | "RelayConnectionTarget", |
| 2154 | ); |
| 2155 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 2156 | }).pipe(Effect.provide(TestClock.layer())), |
| 2157 | ); |
| 2158 | |
| 2159 | it.effect("shares one head start across several slow LAN routes", () => |
| 2160 | Effect.gen(function* () { |
| 2161 | const secondLanTarget = new BearerConnectionTarget({ |
| 2162 | environmentId: TARGET.environmentId, |
| 2163 | label: TARGET.label, |
| 2164 | connectionId: "bearer:second-lan", |
| 2165 | }); |
| 2166 | const entry: ConnectionCatalogEntry = { |
| 2167 | ...LAN_THEN_RELAY_ENTRY, |
| 2168 | alternateRoutes: [ |
| 2169 | { |
| 2170 | target: secondLanTarget, |
| 2171 | profile: Option.some( |
| 2172 | new BearerConnectionProfile({ |
| 2173 | connectionId: secondLanTarget.connectionId, |
| 2174 | environmentId: TARGET.environmentId, |
| 2175 | label: TARGET.label, |
| 2176 | httpBaseUrl: "http://192.168.56.1:3773/", |
| 2177 | wsBaseUrl: "ws://192.168.56.1:3773/", |
| 2178 | }), |
| 2179 | ), |
| 2180 | }, |
| 2181 | { target: RELAY_TARGET, profile: Option.none() }, |
| 2182 | ], |
| 2183 | }; |
| 2184 | const checking = yield* Ref.make(0); |
| 2185 | const checkingLans = yield* Deferred.make<void>(); |
| 2186 | const harness = yield* makeHarness({ |
| 2187 | checkRoute: (route) => |
| 2188 | route.target._tag === "BearerConnectionTarget" |
| 2189 | ? Ref.updateAndGet(checking, (count) => count + 1).pipe( |
| 2190 | Effect.flatMap((count) => |
| 2191 | count === 2 ? Deferred.succeed(checkingLans, undefined) : Effect.void, |
| 2192 | ), |
| 2193 | Effect.andThen(Effect.sleep("2500 millis")), |
| 2194 | Effect.as("silent" as const), |
| 2195 | ) |
| 2196 | : Effect.succeed("unchecked"), |
| 2197 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2198 | }); |
| 2199 | const supervisor = yield* EnvironmentSupervisor.make(entry, { |
| 2200 | initiallyDesired: true, |
| 2201 | }).pipe(Effect.provide(harness.dependencies)); |
| 2202 | |
| 2203 | yield* Deferred.await(checkingLans); |
| 2204 | yield* TestClock.adjust("500 millis"); |
| 2205 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2206 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2207 | "RelayConnectionTarget", |
| 2208 | ); |
| 2209 | }).pipe(Effect.provide(TestClock.layer())), |
| 2210 | ); |
| 2211 | |
| 2212 | it.effect("still prefers a LAN route that responds within its head start", () => |
| 2213 | Effect.gen(function* () { |
| 2214 | const checkingLan = yield* Deferred.make<void>(); |
| 2215 | const harness = yield* makeHarness({ |
| 2216 | checkRoute: (route) => |
| 2217 | route.target._tag === "BearerConnectionTarget" |
| 2218 | ? Deferred.succeed(checkingLan, undefined).pipe( |
| 2219 | Effect.andThen(Effect.sleep("100 millis")), |
| 2220 | Effect.as("answered" as const), |
| 2221 | ) |
| 2222 | : Effect.succeed("unchecked"), |
| 2223 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2224 | }); |
| 2225 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2226 | initiallyDesired: true, |
| 2227 | }).pipe(Effect.provide(harness.dependencies)); |
| 2228 | |
| 2229 | yield* Deferred.await(checkingLan); |
| 2230 | yield* TestClock.adjust("100 millis"); |
| 2231 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2232 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2233 | "BearerConnectionTarget", |
| 2234 | ); |
| 2235 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 2236 | }).pipe(Effect.provide(TestClock.layer())), |
| 2237 | ); |
| 2238 | |
| 2239 | it.effect("tries a slow tailnet route that answered before a dead LAN route", () => |
| 2240 | Effect.gen(function* () { |
| 2241 | const tailnetTarget = new BearerConnectionTarget({ |
| 2242 | environmentId: TARGET.environmentId, |
| 2243 | label: TARGET.label, |
| 2244 | connectionId: "bearer:tailnet", |
| 2245 | }); |
| 2246 | // No relay: a Tailscale-only client away from the LAN. |
| 2247 | const entry: ConnectionCatalogEntry = { |
| 2248 | ...LAN_THEN_RELAY_ENTRY, |
| 2249 | alternateRoutes: [ |
| 2250 | { |
| 2251 | target: tailnetTarget, |
| 2252 | profile: Option.some( |
| 2253 | new BearerConnectionProfile({ |
| 2254 | connectionId: tailnetTarget.connectionId, |
| 2255 | environmentId: TARGET.environmentId, |
| 2256 | label: TARGET.label, |
| 2257 | httpBaseUrl: "http://100.100.1.2:3773/", |
| 2258 | wsBaseUrl: "ws://100.100.1.2:3773/", |
| 2259 | network: "tailscale", |
| 2260 | }), |
| 2261 | ), |
| 2262 | }, |
| 2263 | ], |
| 2264 | }; |
| 2265 | const checking = yield* Ref.make(0); |
| 2266 | const checkingBoth = yield* Deferred.make<void>(); |
| 2267 | const harness = yield* makeHarness({ |
| 2268 | checkRoute: (route) => |
| 2269 | Ref.updateAndGet(checking, (count) => count + 1).pipe( |
| 2270 | Effect.flatMap((count) => |
| 2271 | count === 2 ? Deferred.succeed(checkingBoth, undefined) : Effect.void, |
| 2272 | ), |
| 2273 | Effect.andThen( |
| 2274 | connectionRouteId(route.target) === LAN_TARGET.connectionId |
| 2275 | ? Effect.sleep("2500 millis").pipe(Effect.as("silent" as const)) |
| 2276 | : Effect.sleep("700 millis").pipe(Effect.as("answered" as const)), |
| 2277 | ), |
| 2278 | ), |
| 2279 | // The dead LAN address never answers the connection itself either. |
| 2280 | prepare: (_attempt, target) => |
| 2281 | target._tag === "BearerConnectionTarget" && |
| 2282 | target.connectionId === LAN_TARGET.connectionId |
| 2283 | ? Effect.never |
| 2284 | : Effect.succeed(preparedFor(target)), |
| 2285 | }); |
| 2286 | const supervisor = yield* EnvironmentSupervisor.make(entry, { |
| 2287 | initiallyDesired: true, |
| 2288 | }).pipe(Effect.provide(harness.dependencies)); |
| 2289 | |
| 2290 | yield* Deferred.await(checkingBoth); |
| 2291 | yield* TestClock.adjust("2500 millis"); |
| 2292 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2293 | expect( |
| 2294 | connectionRouteId( |
| 2295 | Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target, |
| 2296 | ), |
| 2297 | ).toBe("bearer:tailnet"); |
| 2298 | }).pipe(Effect.provide(TestClock.layer())), |
| 2299 | ); |
| 2300 | |
| 2301 | it.effect("still tries a slow LAN route when relay fails", () => |
| 2302 | Effect.gen(function* () { |
| 2303 | const checkingLan = yield* Deferred.make<void>(); |
| 2304 | const attempted: string[] = []; |
| 2305 | const harness = yield* makeHarness({ |
| 2306 | checkRoute: (route) => |
| 2307 | route.target._tag === "BearerConnectionTarget" |
| 2308 | ? Deferred.succeed(checkingLan, undefined).pipe( |
| 2309 | Effect.andThen(Effect.sleep("2500 millis")), |
| 2310 | Effect.as("answered" as const), |
| 2311 | ) |
| 2312 | : Effect.succeed("unchecked"), |
| 2313 | prepare: (_attempt, target) => { |
| 2314 | attempted.push(target._tag); |
| 2315 | return target._tag === "RelayConnectionTarget" |
| 2316 | ? Effect.fail(blocked("Sign in to T3 Connect.")) |
| 2317 | : Effect.succeed(preparedFor(target)); |
| 2318 | }, |
| 2319 | }); |
| 2320 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2321 | initiallyDesired: true, |
| 2322 | }).pipe(Effect.provide(harness.dependencies)); |
| 2323 | |
| 2324 | yield* Deferred.await(checkingLan); |
| 2325 | yield* TestClock.adjust("2500 millis"); |
| 2326 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2327 | expect(attempted).toEqual(["RelayConnectionTarget", "BearerConnectionTarget"]); |
| 2328 | }).pipe(Effect.provide(TestClock.layer())), |
| 2329 | ); |
| 2330 | |
| 2331 | it.effect("retries instead of waiting on a LAN route that stayed silent after relay failed", () => |
| 2332 | Effect.gen(function* () { |
| 2333 | const checkingLan = yield* Deferred.make<void>(); |
| 2334 | const attempted: string[] = []; |
| 2335 | const harness = yield* makeHarness({ |
| 2336 | checkRoute: (route) => |
| 2337 | route.target._tag === "BearerConnectionTarget" |
| 2338 | ? Deferred.succeed(checkingLan, undefined).pipe( |
| 2339 | Effect.andThen(Effect.sleep("2500 millis")), |
| 2340 | Effect.as("silent" as const), |
| 2341 | ) |
| 2342 | : Effect.succeed("unchecked"), |
| 2343 | prepare: (_attempt, target) => { |
| 2344 | attempted.push(target._tag); |
| 2345 | return target._tag === "RelayConnectionTarget" |
| 2346 | ? Effect.fail(transient("Relay socket closed.")) |
| 2347 | : Effect.never; |
| 2348 | }, |
| 2349 | }); |
| 2350 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2351 | initiallyDesired: true, |
| 2352 | }).pipe(Effect.provide(harness.dependencies)); |
| 2353 | |
| 2354 | yield* Deferred.await(checkingLan); |
| 2355 | yield* TestClock.adjust("2500 millis"); |
| 2356 | yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); |
| 2357 | expect(attempted).toEqual(["RelayConnectionTarget"]); |
| 2358 | }).pipe(Effect.provide(TestClock.layer())), |
| 2359 | ); |
| 2360 | |
| 2361 | it.effect("skips a silent LAN route and connects over T3 Connect", () => |
| 2362 | Effect.gen(function* () { |
| 2363 | const harness = yield* makeHarness({ |
| 2364 | checkRoute: (route) => |
| 2365 | Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"), |
| 2366 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2367 | }); |
| 2368 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2369 | initiallyDesired: true, |
| 2370 | }).pipe(Effect.provide(harness.dependencies)); |
| 2371 | |
| 2372 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2373 | const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)); |
| 2374 | expect(prepared.target._tag).toBe("RelayConnectionTarget"); |
| 2375 | expect(yield* Ref.get(harness.prepareCount)).toBe(1); |
| 2376 | }), |
| 2377 | ); |
| 2378 | |
| 2379 | it.effect("moves to a blocked route's fallback instead of stopping", () => |
| 2380 | Effect.gen(function* () { |
| 2381 | const harness = yield* makeHarness({ |
| 2382 | checkRoute: () => Effect.succeed("answered"), |
| 2383 | prepare: (_attempt, target) => |
| 2384 | target._tag === "BearerConnectionTarget" |
| 2385 | ? Effect.fail(blocked("The environment credential is invalid.")) |
| 2386 | : Effect.succeed(preparedFor(target)), |
| 2387 | }); |
| 2388 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2389 | initiallyDesired: true, |
| 2390 | }).pipe(Effect.provide(harness.dependencies)); |
| 2391 | |
| 2392 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2393 | const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)); |
| 2394 | expect(prepared.target._tag).toBe("RelayConnectionTarget"); |
| 2395 | }), |
| 2396 | ); |
| 2397 | |
| 2398 | it.effect("does not report a route that failed before reaching its host's sign-in", () => |
| 2399 | Effect.gen(function* () { |
| 2400 | const failures = [ |
| 2401 | new ConnectionBlockedError({ |
| 2402 | reason: "permission", |
| 2403 | detail: "T3 Connect is not enabled on this machine.", |
| 2404 | hop: "tunnel", |
| 2405 | failureCause: "tunnel_not_enabled", |
| 2406 | }), |
| 2407 | new ConnectionBlockedError({ |
| 2408 | reason: "authentication", |
| 2409 | detail: "This device's clock is wrong.", |
| 2410 | hop: "server", |
| 2411 | failureCause: "clock_skew", |
| 2412 | }), |
| 2413 | ]; |
| 2414 | for (const failure of failures) { |
| 2415 | const harness = yield* makeHarness({ |
| 2416 | checkRoute: () => Effect.succeed("answered"), |
| 2417 | prepare: (_attempt, target) => |
| 2418 | target._tag === "BearerConnectionTarget" |
| 2419 | ? Effect.fail(failure) |
| 2420 | : Effect.succeed(preparedFor(target)), |
| 2421 | }); |
| 2422 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2423 | initiallyDesired: true, |
| 2424 | }).pipe(Effect.provide(harness.dependencies)); |
| 2425 | |
| 2426 | const state = yield* awaitState(supervisor.state, (value) => value.phase === "connected"); |
| 2427 | expect(state.refusedRouteIds).toEqual([]); |
| 2428 | } |
| 2429 | }), |
| 2430 | ); |
| 2431 | |
| 2432 | it.effect("reports a preferred route whose host refuses sign-in once it answers again", () => |
| 2433 | Effect.gen(function* () { |
| 2434 | const lanReachable = yield* Ref.make(false); |
| 2435 | const lanRefuses = yield* Ref.make(true); |
| 2436 | const checkRoute = (route: ConnectionRoute) => |
| 2437 | route.target._tag === "BearerConnectionTarget" |
| 2438 | ? Ref.get(lanReachable).pipe( |
| 2439 | Effect.map((reachable): ConnectionDriver.RouteCheck => |
| 2440 | reachable ? "answered" : "silent", |
| 2441 | ), |
| 2442 | ) |
| 2443 | : Effect.succeed<ConnectionDriver.RouteCheck>("unchecked"); |
| 2444 | const harness = yield* makeHarness({ |
| 2445 | checkRoute, |
| 2446 | preflight: (route) => |
| 2447 | ConnectionDriver.preflightRoute( |
| 2448 | checkRoute(route), |
| 2449 | Ref.get(lanRefuses).pipe( |
| 2450 | Effect.flatMap((refuses) => |
| 2451 | refuses && route.target._tag === "BearerConnectionTarget" |
| 2452 | ? Effect.fail(hostRefusal()) |
| 2453 | : Effect.succeed(preparedFor(route.target)), |
| 2454 | ), |
| 2455 | ), |
| 2456 | ), |
| 2457 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2458 | }); |
| 2459 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2460 | initiallyDesired: true, |
| 2461 | }).pipe(Effect.provide(harness.dependencies)); |
| 2462 | |
| 2463 | const connected = yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2464 | expect(connected.refusedRouteIds).toEqual([]); |
| 2465 | |
| 2466 | yield* Ref.set(lanReachable, true); |
| 2467 | yield* harness.wake("network-changed"); |
| 2468 | const state = yield* awaitState( |
| 2469 | supervisor.state, |
| 2470 | (value) => value.refusedRouteIds.length > 0, |
| 2471 | ); |
| 2472 | expect(state).toMatchObject({ |
| 2473 | phase: "connected", |
| 2474 | generation: connected.generation, |
| 2475 | refusedRouteIds: [LAN_TARGET.connectionId], |
| 2476 | }); |
| 2477 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2478 | "RelayConnectionTarget", |
| 2479 | ); |
| 2480 | |
| 2481 | yield* Ref.set(lanRefuses, false); |
| 2482 | yield* harness.wake("network-changed"); |
| 2483 | const overLan = yield* awaitState( |
| 2484 | supervisor.state, |
| 2485 | (value) => value.phase === "connected" && value.generation > connected.generation, |
| 2486 | ); |
| 2487 | expect( |
| 2488 | connectionRouteId( |
| 2489 | Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target, |
| 2490 | ), |
| 2491 | ).toBe(LAN_TARGET.connectionId); |
| 2492 | expect(overLan.refusedRouteIds).toEqual([]); |
| 2493 | }), |
| 2494 | ); |
| 2495 | |
| 2496 | it.effect("keeps reporting a refused route after moving to a better route", () => |
| 2497 | Effect.gen(function* () { |
| 2498 | const loopbackTarget = new BearerConnectionTarget({ |
| 2499 | environmentId: TARGET.environmentId, |
| 2500 | label: TARGET.label, |
| 2501 | connectionId: "bearer:loopback", |
| 2502 | }); |
| 2503 | const entry: ConnectionCatalogEntry = { |
| 2504 | target: loopbackTarget, |
| 2505 | profile: Option.some( |
| 2506 | new BearerConnectionProfile({ |
| 2507 | connectionId: loopbackTarget.connectionId, |
| 2508 | environmentId: TARGET.environmentId, |
| 2509 | label: TARGET.label, |
| 2510 | httpBaseUrl: "http://127.0.0.1:3773/", |
| 2511 | wsBaseUrl: "ws://127.0.0.1:3773/", |
| 2512 | }), |
| 2513 | ), |
| 2514 | alternateRoutes: [LAN_ROUTE, { target: RELAY_TARGET, profile: Option.none() }], |
| 2515 | enabled: true, |
| 2516 | }; |
| 2517 | const lanReachable = yield* Ref.make(false); |
| 2518 | const checkRoute = (route: ConnectionRoute) => |
| 2519 | connectionRouteId(route.target) === LAN_TARGET.connectionId |
| 2520 | ? Ref.get(lanReachable).pipe( |
| 2521 | Effect.map((reachable): ConnectionDriver.RouteCheck => |
| 2522 | reachable ? "answered" : "silent", |
| 2523 | ), |
| 2524 | ) |
| 2525 | : Effect.succeed<ConnectionDriver.RouteCheck>( |
| 2526 | route.target._tag === "BearerConnectionTarget" ? "answered" : "unchecked", |
| 2527 | ); |
| 2528 | const prepare = (target: ConnectionTarget) => |
| 2529 | connectionRouteId(target) === loopbackTarget.connectionId |
| 2530 | ? Effect.fail(hostRefusal()) |
| 2531 | : Effect.succeed(preparedFor(target)); |
| 2532 | const harness = yield* makeHarness({ |
| 2533 | checkRoute, |
| 2534 | preflight: (route) => |
| 2535 | ConnectionDriver.preflightRoute(checkRoute(route), prepare(route.target)), |
| 2536 | prepare: (_attempt, target) => prepare(target), |
| 2537 | }); |
| 2538 | const supervisor = yield* EnvironmentSupervisor.make(entry, { |
| 2539 | initiallyDesired: true, |
| 2540 | }).pipe(Effect.provide(harness.dependencies)); |
| 2541 | |
| 2542 | const overRelay = yield* awaitState( |
| 2543 | supervisor.state, |
| 2544 | (state) => state.phase === "connected" && state.generation === 1, |
| 2545 | ); |
| 2546 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2547 | "RelayConnectionTarget", |
| 2548 | ); |
| 2549 | expect(overRelay.refusedRouteIds).toEqual([loopbackTarget.connectionId]); |
| 2550 | |
| 2551 | yield* Ref.set(lanReachable, true); |
| 2552 | yield* harness.wake("network-changed"); |
| 2553 | const overLan = yield* awaitState( |
| 2554 | supervisor.state, |
| 2555 | (state) => state.phase === "connected" && state.generation === 2, |
| 2556 | ); |
| 2557 | expect( |
| 2558 | connectionRouteId( |
| 2559 | Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target, |
| 2560 | ), |
| 2561 | ).toBe(LAN_TARGET.connectionId); |
| 2562 | expect(overLan.refusedRouteIds).toEqual([loopbackTarget.connectionId]); |
| 2563 | }), |
| 2564 | ); |
| 2565 | |
| 2566 | it.effect("tries every route when none answers the reachability check", () => |
| 2567 | Effect.gen(function* () { |
| 2568 | const harness = yield* makeHarness({ |
| 2569 | checkRoute: () => Effect.succeed("silent"), |
| 2570 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2571 | }); |
| 2572 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2573 | initiallyDesired: true, |
| 2574 | }).pipe(Effect.provide(harness.dependencies)); |
| 2575 | |
| 2576 | yield* awaitState(supervisor.state, (state) => state.phase === "connected"); |
| 2577 | const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)); |
| 2578 | expect(prepared.target._tag).toBe("BearerConnectionTarget"); |
| 2579 | }), |
| 2580 | ); |
| 2581 | |
| 2582 | it.effect("keeps retrying when one route is blocked and another failed transiently", () => |
| 2583 | Effect.gen(function* () { |
| 2584 | const harness = yield* makeHarness({ |
| 2585 | checkRoute: () => Effect.succeed("answered"), |
| 2586 | prepare: (_attempt, target) => |
| 2587 | target._tag === "RelayConnectionTarget" |
| 2588 | ? Effect.fail(blocked("Sign in to T3 Connect.")) |
| 2589 | : Effect.fail(transient("LAN socket refused.")), |
| 2590 | }); |
| 2591 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2592 | initiallyDesired: true, |
| 2593 | }).pipe(Effect.provide(harness.dependencies)); |
| 2594 | |
| 2595 | const state = yield* awaitState(supervisor.state, (value) => value.phase === "backoff"); |
| 2596 | expect(state.lastFailure?._tag).toBe("ConnectionTransientError"); |
| 2597 | }), |
| 2598 | ); |
| 2599 | |
| 2600 | it.effect("moves back to the LAN route when the network changes and it answers again", () => |
| 2601 | Effect.gen(function* () { |
| 2602 | const lanReachable = yield* Ref.make(false); |
| 2603 | const harness = yield* makeHarness({ |
| 2604 | checkRoute: (route) => |
| 2605 | route.target._tag === "BearerConnectionTarget" |
| 2606 | ? Ref.get(lanReachable).pipe( |
| 2607 | Effect.map((reachable) => (reachable ? "answered" : "silent")), |
| 2608 | ) |
| 2609 | : Effect.succeed("unchecked"), |
| 2610 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2611 | }); |
| 2612 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2613 | initiallyDesired: true, |
| 2614 | }).pipe(Effect.provide(harness.dependencies)); |
| 2615 | |
| 2616 | yield* awaitState( |
| 2617 | supervisor.state, |
| 2618 | (state) => state.phase === "connected" && state.generation === 1, |
| 2619 | ); |
| 2620 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2621 | "RelayConnectionTarget", |
| 2622 | ); |
| 2623 | |
| 2624 | yield* Ref.set(lanReachable, true); |
| 2625 | yield* harness.setNetworkStatus("unknown"); |
| 2626 | yield* harness.setNetworkStatus("online"); |
| 2627 | |
| 2628 | yield* awaitState( |
| 2629 | supervisor.state, |
| 2630 | (state) => state.phase === "connected" && state.generation === 2, |
| 2631 | ); |
| 2632 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2633 | "BearerConnectionTarget", |
| 2634 | ); |
| 2635 | expect(yield* Ref.get(harness.releaseCount)).toBe(1); |
| 2636 | }), |
| 2637 | ); |
| 2638 | |
| 2639 | it.effect("checks a dead LAN route less often until the network changes", () => |
| 2640 | Effect.gen(function* () { |
| 2641 | const lanReachable = yield* Ref.make(false); |
| 2642 | const lanChecks = yield* Ref.make(0); |
| 2643 | const harness = yield* makeHarness({ |
| 2644 | checkRoute: (route) => |
| 2645 | route.target._tag === "BearerConnectionTarget" |
| 2646 | ? Ref.update(lanChecks, (count) => count + 1).pipe( |
| 2647 | Effect.andThen(Ref.get(lanReachable)), |
| 2648 | Effect.map((reachable) => (reachable ? "answered" : "silent")), |
| 2649 | ) |
| 2650 | : Effect.succeed("unchecked"), |
| 2651 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2652 | }); |
| 2653 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2654 | initiallyDesired: true, |
| 2655 | }).pipe(Effect.provide(harness.dependencies)); |
| 2656 | |
| 2657 | yield* awaitState( |
| 2658 | supervisor.state, |
| 2659 | (state) => state.phase === "connected" && state.generation === 1, |
| 2660 | ); |
| 2661 | yield* Ref.set(lanChecks, 0); |
| 2662 | |
| 2663 | // Checked at 1, 3 and 7 minutes instead of every minute. |
| 2664 | for (let minute = 0; minute < 10; minute += 1) { |
| 2665 | yield* TestClock.adjust("60 seconds"); |
| 2666 | } |
| 2667 | expect(yield* Ref.get(lanChecks)).toBe(3); |
| 2668 | |
| 2669 | yield* Ref.set(lanReachable, true); |
| 2670 | yield* harness.wake("network-changed"); |
| 2671 | yield* awaitState( |
| 2672 | supervisor.state, |
| 2673 | (state) => state.phase === "connected" && state.generation === 2, |
| 2674 | ); |
| 2675 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2676 | "BearerConnectionTarget", |
| 2677 | ); |
| 2678 | }), |
| 2679 | ); |
| 2680 | |
| 2681 | it.effect("keeps checking a LAN route every minute when it answers but cannot be prepared", () => |
| 2682 | Effect.gen(function* () { |
| 2683 | const lanPreflights = yield* Ref.make(0); |
| 2684 | const harness = yield* makeHarness({ |
| 2685 | checkRoute: (route) => |
| 2686 | Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"), |
| 2687 | preflight: (route) => |
| 2688 | route.target._tag === "BearerConnectionTarget" |
| 2689 | ? Ref.update(lanPreflights, (count) => count + 1).pipe(Effect.as("unprepared" as const)) |
| 2690 | : Effect.succeed("unchecked"), |
| 2691 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2692 | }); |
| 2693 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2694 | initiallyDesired: true, |
| 2695 | }).pipe(Effect.provide(harness.dependencies)); |
| 2696 | |
| 2697 | yield* awaitState( |
| 2698 | supervisor.state, |
| 2699 | (state) => state.phase === "connected" && state.generation === 1, |
| 2700 | ); |
| 2701 | for (let minute = 0; minute < 10; minute += 1) { |
| 2702 | yield* TestClock.adjust("60 seconds"); |
| 2703 | } |
| 2704 | expect(yield* Ref.get(lanPreflights)).toBe(10); |
| 2705 | }), |
| 2706 | ); |
| 2707 | |
| 2708 | it.effect("ignores a dead LAN check that was running when the network changed", () => |
| 2709 | Effect.gen(function* () { |
| 2710 | const lanChecks = yield* Ref.make(0); |
| 2711 | const checkStarted = yield* Deferred.make<void>(); |
| 2712 | const finishCheck = yield* Deferred.make<void>(); |
| 2713 | const woke = yield* Ref.make(false); |
| 2714 | const wakeupHandled = yield* Deferred.make<void>(); |
| 2715 | const harness = yield* makeHarness({ |
| 2716 | checkRoute: (route) => |
| 2717 | route.target._tag === "BearerConnectionTarget" |
| 2718 | ? Ref.updateAndGet(lanChecks, (count) => count + 1).pipe( |
| 2719 | Effect.flatMap((check) => |
| 2720 | check === 1 |
| 2721 | ? Effect.succeed("silent" as const) |
| 2722 | : check === 2 |
| 2723 | ? Deferred.succeed(checkStarted, undefined).pipe( |
| 2724 | Effect.andThen(Deferred.await(finishCheck)), |
| 2725 | Effect.as("silent" as const), |
| 2726 | ) |
| 2727 | : Effect.succeed("answered" as const), |
| 2728 | ), |
| 2729 | ) |
| 2730 | : Effect.succeed("unchecked"), |
| 2731 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2732 | // The wakeup probes the session only after it forgets the silent routes. |
| 2733 | probe: () => |
| 2734 | Ref.get(woke).pipe( |
| 2735 | Effect.flatMap((afterWake) => |
| 2736 | afterWake ? Deferred.succeed(wakeupHandled, undefined) : Effect.void, |
| 2737 | ), |
| 2738 | Effect.asVoid, |
| 2739 | ), |
| 2740 | }); |
| 2741 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2742 | initiallyDesired: true, |
| 2743 | }).pipe(Effect.provide(harness.dependencies)); |
| 2744 | |
| 2745 | yield* awaitState( |
| 2746 | supervisor.state, |
| 2747 | (state) => state.phase === "connected" && state.generation === 1, |
| 2748 | ); |
| 2749 | yield* TestClock.adjust("60 seconds"); |
| 2750 | yield* Deferred.await(checkStarted); |
| 2751 | yield* Ref.set(woke, true); |
| 2752 | yield* harness.wake("network-changed"); |
| 2753 | yield* Deferred.await(wakeupHandled); |
| 2754 | yield* Deferred.succeed(finishCheck, undefined); |
| 2755 | |
| 2756 | yield* awaitState( |
| 2757 | supervisor.state, |
| 2758 | (state) => state.phase === "connected" && state.generation === 2, |
| 2759 | ); |
| 2760 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2761 | "BearerConnectionTarget", |
| 2762 | ); |
| 2763 | }), |
| 2764 | ); |
| 2765 | |
| 2766 | it.effect("moves off a dead LAN socket when the phone switches to cellular", () => |
| 2767 | Effect.gen(function* () { |
| 2768 | const lanReachable = yield* Ref.make(true); |
| 2769 | const harness = yield* makeHarness({ |
| 2770 | checkRoute: (route) => |
| 2771 | route.target._tag === "BearerConnectionTarget" |
| 2772 | ? Ref.get(lanReachable).pipe( |
| 2773 | Effect.map((reachable) => (reachable ? "answered" : "silent")), |
| 2774 | ) |
| 2775 | : Effect.succeed("unchecked"), |
| 2776 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2777 | probe: () => |
| 2778 | Ref.get(lanReachable).pipe( |
| 2779 | Effect.flatMap((reachable) => (reachable ? Effect.void : Effect.never)), |
| 2780 | ), |
| 2781 | }); |
| 2782 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 2783 | initiallyDesired: true, |
| 2784 | }).pipe(Effect.provide(harness.dependencies)); |
| 2785 | |
| 2786 | yield* awaitState( |
| 2787 | supervisor.state, |
| 2788 | (state) => state.phase === "connected" && state.generation === 1, |
| 2789 | ); |
| 2790 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2791 | "BearerConnectionTarget", |
| 2792 | ); |
| 2793 | |
| 2794 | yield* Ref.set(lanReachable, false); |
| 2795 | yield* harness.wake("network-changed"); |
| 2796 | yield* TestClock.adjust("3 seconds"); |
| 2797 | |
| 2798 | yield* awaitState( |
| 2799 | supervisor.state, |
| 2800 | (state) => state.phase === "connected" && state.generation === 2, |
| 2801 | ); |
| 2802 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2803 | "RelayConnectionTarget", |
| 2804 | ); |
| 2805 | }), |
| 2806 | ); |
| 2807 | |
| 2808 | it.effect.each(["initial", "late", "withdrawn", "withdrawn-upgrade", "omitted"] as const)( |
| 2809 | "learns the %s direct endpoint over T3 Connect", |
| 2810 | (discovery) => |
| 2811 | Effect.gen(function* () { |
| 2812 | const relayEntry: ConnectionCatalogEntry = |
| 2813 | discovery.startsWith("withdrawn") || discovery === "omitted" |
| 2814 | ? { |
| 2815 | target: LAN_ROUTE.target, |
| 2816 | profile: Option.some( |
| 2817 | new BearerConnectionProfile({ |
| 2818 | ...Option.getOrThrow(LAN_ROUTE.profile), |
| 2819 | ...(discovery === "withdrawn-upgrade" |
| 2820 | ? { |
| 2821 | httpBaseUrl: "https://old.example.test/", |
| 2822 | wsBaseUrl: "wss://old.example.test/", |
| 2823 | } |
| 2824 | : {}), |
| 2825 | learned: true, |
| 2826 | authorization: "t3-connect", |
| 2827 | }), |
| 2828 | ), |
| 2829 | alternateRoutes: [{ target: RELAY_TARGET, profile: Option.none() }], |
| 2830 | enabled: true, |
| 2831 | } |
| 2832 | : { |
| 2833 | target: RELAY_TARGET, |
| 2834 | profile: Option.none(), |
| 2835 | enabled: true, |
| 2836 | }; |
| 2837 | const initialProcessed = yield* Deferred.make<void>(); |
| 2838 | const discoveryProcessed = yield* Deferred.make<void>(); |
| 2839 | const discoveryStarted = yield* Deferred.make<void>(); |
| 2840 | const finishDiscovery = yield* Deferred.make<void>(); |
| 2841 | const lateAddress = "https://fixture.tail1234.ts.net/"; |
| 2842 | const learned = yield* Ref.make<ReadonlyArray<string>>([]); |
| 2843 | const harness = yield* makeHarness({ |
| 2844 | configEvents: |
| 2845 | discovery !== "initial" |
| 2846 | ? Stream.concat( |
| 2847 | Stream.make({ |
| 2848 | version: 1 as const, |
| 2849 | type: "snapshot" as const, |
| 2850 | config: (discovery === "omitted" |
| 2851 | ? {} |
| 2852 | : { directEndpoints: [] }) as unknown as ServerConfig, |
| 2853 | }).pipe(Stream.ensuring(Deferred.succeed(initialProcessed, undefined))), |
| 2854 | Stream.fromEffect( |
| 2855 | Deferred.await(finishDiscovery).pipe( |
| 2856 | Effect.as({ |
| 2857 | version: 1 as const, |
| 2858 | type: "snapshot" as const, |
| 2859 | config: { |
| 2860 | directEndpoints: [{ kind: "tailnet", httpBaseUrl: lateAddress }], |
| 2861 | } as unknown as ServerConfig, |
| 2862 | }), |
| 2863 | ), |
| 2864 | ), |
| 2865 | ).pipe(Stream.ensuring(Deferred.succeed(discoveryProcessed, undefined))) |
| 2866 | : undefined, |
| 2867 | checkRoute: (route) => |
| 2868 | route.target._tag === "BearerConnectionTarget" |
| 2869 | ? Effect.succeed("answered") |
| 2870 | : Effect.succeed("unchecked"), |
| 2871 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 2872 | initialConfig: () => |
| 2873 | Effect.succeed({ |
| 2874 | directEndpoints: |
| 2875 | discovery !== "initial" |
| 2876 | ? [] |
| 2877 | : [{ kind: "lan", httpBaseUrl: "http://192.168.1.10:3773/" }], |
| 2878 | } as unknown as ServerConfig), |
| 2879 | }); |
| 2880 | const current = yield* Ref.make(relayEntry); |
| 2881 | const supervisor = yield* EnvironmentSupervisor.make(relayEntry, { |
| 2882 | initiallyDesired: true, |
| 2883 | learnRoutes: ({ activeRoute, reported }) => |
| 2884 | Effect.gen(function* () { |
| 2885 | yield* Deferred.succeed(discoveryStarted, undefined); |
| 2886 | const entry = yield* Ref.get(current); |
| 2887 | const routes = mergeLearnedRoutes({ |
| 2888 | entry, |
| 2889 | activeRoute, |
| 2890 | reported, |
| 2891 | allowInsecure: true, |
| 2892 | }); |
| 2893 | if (routes === null) return Option.none(); |
| 2894 | const next = entryWithRoutes(entry, routes); |
| 2895 | yield* Ref.set(current, next); |
| 2896 | yield* Ref.update(learned, (all) => [ |
| 2897 | ...all, |
| 2898 | ...routes.filter(isLearned).map((route) => connectionRouteId(route.target)), |
| 2899 | ]); |
| 2900 | return Option.some(next); |
| 2901 | }), |
| 2902 | }).pipe(Effect.provide(harness.dependencies)); |
| 2903 | |
| 2904 | if (discovery === "omitted") { |
| 2905 | yield* Deferred.await(initialProcessed); |
| 2906 | expect(yield* Ref.get(current)).toEqual(relayEntry); |
| 2907 | expect(yield* Ref.get(learned)).toEqual([]); |
| 2908 | } else { |
| 2909 | yield* Deferred.await(discoveryStarted); |
| 2910 | } |
| 2911 | yield* Deferred.succeed(finishDiscovery, undefined); |
| 2912 | if (discovery === "withdrawn-upgrade") { |
| 2913 | yield* Deferred.await(discoveryProcessed); |
| 2914 | yield* TestClock.adjust("60 seconds"); |
| 2915 | expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(2); |
| 2916 | return; |
| 2917 | } |
| 2918 | if (discovery === "withdrawn" || discovery === "omitted") { |
| 2919 | yield* Deferred.await(discoveryProcessed); |
| 2920 | expect(yield* Ref.get(learned)).toContain( |
| 2921 | `learned:${TARGET.environmentId}:https://fixture.tail1234.ts.net`, |
| 2922 | ); |
| 2923 | expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1); |
| 2924 | return; |
| 2925 | } |
| 2926 | yield* awaitState( |
| 2927 | supervisor.state, |
| 2928 | (state) => state.phase === "connected" && state.generation === 2, |
| 2929 | ); |
| 2930 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 2931 | "BearerConnectionTarget", |
| 2932 | ); |
| 2933 | expect(yield* Ref.get(learned)).toEqual([ |
| 2934 | `learned:${TARGET.environmentId}:${discovery !== "initial" ? "https://fixture.tail1234.ts.net" : "http://192.168.1.10:3773"}`, |
| 2935 | ]); |
| 2936 | }), |
| 2937 | ); |
| 2938 | }); |
| 2939 | |
| 2940 | describe("EnvironmentSupervisor background fibers", () => { |
| 2941 | it.effect("hears the network return after its connectivity stream died", () => |
| 2942 | Effect.gen(function* () { |
| 2943 | const harness = yield* makeHarness(); |
| 2944 | const status = yield* SubscriptionRef.make<NetworkStatus>("offline"); |
| 2945 | const subscriptions = yield* Ref.make(0); |
| 2946 | const dying = Connectivity.Connectivity.of({ |
| 2947 | status: SubscriptionRef.get(status), |
| 2948 | changes: Stream.unwrap( |
| 2949 | Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( |
| 2950 | Effect.map((count) => |
| 2951 | count === 1 |
| 2952 | ? Stream.die(new Error("platform connectivity stream died")) |
| 2953 | : SubscriptionRef.changes(status), |
| 2954 | ), |
| 2955 | ), |
| 2956 | ), |
| 2957 | }); |
| 2958 | const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, { |
| 2959 | initiallyDesired: true, |
| 2960 | }).pipe( |
| 2961 | Effect.provideService(Connectivity.Connectivity, dying), |
| 2962 | Effect.provide(harness.dependencies), |
| 2963 | ); |
| 2964 | yield* TestClock.adjust("10 minutes"); |
| 2965 | expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("offline"); |
| 2966 | |
| 2967 | yield* SubscriptionRef.set(status, "online"); |
| 2968 | yield* TestClock.adjust("10 minutes"); |
| 2969 | expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); |
| 2970 | }).pipe(Effect.provide(TestClock.layer())), |
| 2971 | ); |
| 2972 | |
| 2973 | it.effect("replaces a stalled session on return to the app after its wakeup stream died", () => |
| 2974 | Effect.gen(function* () { |
| 2975 | const harness = yield* makeHarness({ probe: () => Effect.never }); |
| 2976 | const returns = yield* SubscriptionRef.make(0); |
| 2977 | const subscriptions = yield* Ref.make(0); |
| 2978 | const dying = ConnectionWakeups.ConnectionWakeups.of({ |
| 2979 | changes: Stream.unwrap( |
| 2980 | Ref.updateAndGet(subscriptions, (count) => count + 1).pipe( |
| 2981 | Effect.map((count) => |
| 2982 | count === 1 |
| 2983 | ? Stream.die(new Error("platform wakeup stream died")) |
| 2984 | : SubscriptionRef.changes(returns).pipe( |
| 2985 | Stream.drop(1), |
| 2986 | Stream.map(() => "application-active" as const), |
| 2987 | ), |
| 2988 | ), |
| 2989 | ), |
| 2990 | ), |
| 2991 | }); |
| 2992 | const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { |
| 2993 | initiallyDesired: true, |
| 2994 | }).pipe( |
| 2995 | Effect.provideService(ConnectionWakeups.ConnectionWakeups, dying), |
| 2996 | Effect.provide(harness.dependencies), |
| 2997 | ); |
| 2998 | yield* TestClock.adjust("10 minutes"); |
| 2999 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 3000 | phase: "connected", |
| 3001 | generation: 1, |
| 3002 | }); |
| 3003 | |
| 3004 | // The user returns to the app; the session's probe never answers. |
| 3005 | yield* SubscriptionRef.update(returns, (count) => count + 1); |
| 3006 | yield* TestClock.adjust("15 seconds"); |
| 3007 | yield* TestClock.adjust("15 seconds"); |
| 3008 | yield* TestClock.adjust("10 minutes"); |
| 3009 | expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({ |
| 3010 | phase: "connected", |
| 3011 | generation: 2, |
| 3012 | }); |
| 3013 | }).pipe(Effect.provide(TestClock.layer())), |
| 3014 | ); |
| 3015 | |
| 3016 | it.effect("still moves to a better route after one route check crashed", () => |
| 3017 | Effect.gen(function* () { |
| 3018 | const lanReachable = yield* Ref.make(false); |
| 3019 | const preflights = yield* Ref.make(0); |
| 3020 | const harness = yield* makeHarness({ |
| 3021 | checkRoute: (route) => |
| 3022 | route.target._tag === "BearerConnectionTarget" |
| 3023 | ? Ref.get(lanReachable).pipe( |
| 3024 | Effect.map((reachable) => (reachable ? "answered" : "silent")), |
| 3025 | ) |
| 3026 | : Effect.succeed("unchecked"), |
| 3027 | preflight: () => |
| 3028 | Ref.updateAndGet(preflights, (count) => count + 1).pipe( |
| 3029 | Effect.flatMap((count) => |
| 3030 | count === 1 |
| 3031 | ? Effect.die(new Error("route check threw")) |
| 3032 | : Ref.get(lanReachable).pipe( |
| 3033 | Effect.map((reachable) => (reachable ? "ready" : "silent")), |
| 3034 | ), |
| 3035 | ), |
| 3036 | ), |
| 3037 | prepare: (_attempt, target) => Effect.succeed(preparedFor(target)), |
| 3038 | }); |
| 3039 | const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, { |
| 3040 | initiallyDesired: true, |
| 3041 | }).pipe(Effect.provide(harness.dependencies)); |
| 3042 | yield* awaitState( |
| 3043 | supervisor.state, |
| 3044 | (state) => state.phase === "connected" && state.generation === 1, |
| 3045 | ); |
| 3046 | yield* TestClock.adjust("60 seconds"); |
| 3047 | expect(yield* Ref.get(preflights)).toBeGreaterThanOrEqual(1); |
| 3048 | |
| 3049 | yield* Ref.set(lanReachable, true); |
| 3050 | for (let minute = 0; minute < 60; minute += 1) { |
| 3051 | yield* TestClock.adjust("60 seconds"); |
| 3052 | } |
| 3053 | expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe( |
| 3054 | "BearerConnectionTarget", |
| 3055 | ); |
| 3056 | }).pipe(Effect.provide(TestClock.layer())), |
| 3057 | ); |
| 3058 | }); |