The best way to control Claude Code, Codex and more from any computer or phone

main
ts 3,058 lines 120 KB
Raw
1 import {
2 AuthStandardClientScopes,
3 EnvironmentId,
4 type ServerConfig,
5 type ServerConfigStreamEvent,
6 } from "@t3tools/contracts";
7 import { describe, expect, it } from "@effect/vitest";
8 import { RelayClientTracer } from "@t3tools/shared/relayTracing";
9 import * as Cause from "effect/Cause";
10 import * as Deferred from "effect/Deferred";
11 import * as Effect from "effect/Effect";
12 import * as Exit from "effect/Exit";
13 import * as Fiber from "effect/Fiber";
14 import * as Layer from "effect/Layer";
15 import * as Logger from "effect/Logger";
16 import * as Option from "effect/Option";
17 import * as Queue from "effect/Queue";
18 import * as Random from "effect/Random";
19 import * as Ref from "effect/Ref";
20 import * as References from "effect/References";
21 import * as Scope from "effect/Scope";
22 import * as Stream from "effect/Stream";
23 import * as SubscriptionRef from "effect/SubscriptionRef";
24 import * as Clock from "effect/Clock";
25 import * as TestClock from "effect/testing/TestClock";
26 import * as Tracer from "effect/Tracer";
27
28 import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
29 import * as ConnectionCredentialStore from "./credentialStore.ts";
30 import * as TokenStore from "../authorization/tokenStore.ts";
31 import * as ClientCapabilities from "../platform/capabilities.ts";
32 import * as ManagedRelay from "../relay/managedRelay.ts";
33 import * as RpcHttp from "../rpc/http.ts";
34 import type { WsRpcProtocolClient } from "../rpc/protocol.ts";
35 import { fetchEnvironmentSessionState } from "../state/session.ts";
36 import type { ConnectionCatalogEntry, ConnectionRoute } from "./catalog.ts";
37 import * as Connectivity from "./connectivity.ts";
38 import * as ConnectionDriver from "./driver.ts";
39 import { BearerConnectionProfile } from "./catalog.ts";
40 import {
41 BearerConnectionTarget,
42 ConnectionBlockedError,
43 ConnectionTransientError,
44 PrimaryConnectionTarget,
45 RelayConnectionTarget,
46 SshConnectionTarget,
47 type ConnectionAttemptError,
48 type ConnectionTarget,
49 type NetworkStatus,
50 type PreparedConnection,
51 type SupervisorConnectionState,
52 } from "./model.ts";
53 import * as RpcSession from "../rpc/session.ts";
54 import { presentConnectionState } from "./presentation.ts";
55 import * as EnvironmentSupervisor from "./supervisor.ts";
56 import * as ConnectionWakeups from "./wakeups.ts";
57 import { connectionRouteId, entryWithRoutes, isLearned, mergeLearnedRoutes } from "./routes.ts";
58
59 const TARGET = new PrimaryConnectionTarget({
60 environmentId: EnvironmentId.make("environment-1"),
61 label: "Test environment",
62 httpBaseUrl: "https://environment.example.test",
63 wsBaseUrl: "wss://environment.example.test",
64 });
65
66 const RELAY_TARGET = new RelayConnectionTarget({
67 environmentId: TARGET.environmentId,
68 label: TARGET.label,
69 });
70
71 const TARGET_ENTRY: ConnectionCatalogEntry = {
72 target: TARGET,
73 profile: Option.none(),
74 enabled: true,
75 };
76
77 const RELAY_ENTRY: ConnectionCatalogEntry = {
78 target: RELAY_TARGET,
79 profile: Option.none(),
80 enabled: true,
81 };
82
83 const PREPARED_CONNECTION: PreparedConnection = {
84 environmentId: TARGET.environmentId,
85 label: TARGET.label,
86 httpBaseUrl: TARGET.httpBaseUrl,
87 socketUrl: "wss://environment.example.test/ws",
88 httpAuthorization: null,
89 target: TARGET,
90 };
91
92 const TEST_RPC_CLIENT = {} as WsRpcProtocolClient;
93
94 const LAN_TARGET = new BearerConnectionTarget({
95 environmentId: TARGET.environmentId,
96 label: TARGET.label,
97 connectionId: "bearer:lan",
98 });
99 const LAN_ROUTE = {
100 target: LAN_TARGET,
101 profile: Option.some(
102 new BearerConnectionProfile({
103 connectionId: LAN_TARGET.connectionId,
104 environmentId: TARGET.environmentId,
105 label: TARGET.label,
106 httpBaseUrl: "http://192.168.1.10:3773/",
107 wsBaseUrl: "ws://192.168.1.10:3773/",
108 }),
109 ),
110 } satisfies ConnectionRoute;
111 const LAN_THEN_RELAY_ENTRY: ConnectionCatalogEntry = {
112 target: LAN_ROUTE.target,
113 profile: LAN_ROUTE.profile,
114 alternateRoutes: [{ target: RELAY_TARGET, profile: Option.none() }],
115 enabled: true,
116 };
117
118 const SSH_ENTRY: ConnectionCatalogEntry = {
119 target: new SshConnectionTarget({
120 environmentId: TARGET.environmentId,
121 label: TARGET.label,
122 connectionId: "ssh-connection",
123 }),
124 profile: Option.none(),
125 enabled: true,
126 };
127
128 const LAN_ENTRY: ConnectionCatalogEntry = {
129 target: LAN_ROUTE.target,
130 profile: LAN_ROUTE.profile,
131 enabled: true,
132 };
133
134 const PROBE_CAPABLE_CONFIG = {
135 environment: { capabilities: { connectionProbe: true } },
136 } as ServerConfig;
137
138 function preparedFor(target: ConnectionTarget): PreparedConnection {
139 return { ...PREPARED_CONNECTION, target };
140 }
141
142 function transient(message = "Connection failed.") {
143 return new ConnectionTransientError({
144 reason: "transport",
145 detail: message,
146 });
147 }
148
149 function blocked(message = "Authentication required.") {
150 return new ConnectionBlockedError({
151 reason: "authentication",
152 detail: message,
153 });
154 }
155
156 function hostRefusal() {
157 return new ConnectionBlockedError({
158 reason: "authentication",
159 detail: "The environment credential is invalid.",
160 hop: "server",
161 });
162 }
163
164 function awaitState(
165 state: SubscriptionRef.SubscriptionRef<SupervisorConnectionState>,
166 predicate: (value: SupervisorConnectionState) => boolean,
167 ) {
168 return SubscriptionRef.changes(state).pipe(
169 Stream.filter(predicate),
170 Stream.runHead,
171 Effect.map(Option.getOrThrow),
172 );
173 }
174
175 function recordLogs(logs: Array<unknown>, annotations?: Array<unknown>) {
176 return Logger.layer(
177 [
178 Logger.make(({ message, fiber }) => {
179 logs.push(message);
180 annotations?.push(fiber.getRef(References.CurrentLogAnnotations));
181 }),
182 ],
183 { mergeWithExisting: false },
184 );
185 }
186
187 const makeHarness = Effect.fn("TestConnectionHarness.make")(function* (options?: {
188 readonly networkStatus?: NetworkStatus;
189 readonly prepare?: (
190 attempt: number,
191 target: ConnectionTarget,
192 ) => Effect.Effect<PreparedConnection, ConnectionAttemptError>;
193 readonly ready?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
194 readonly release?: (attempt: number) => Effect.Effect<void>;
195 readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
196 readonly configEvents?: Stream.Stream<ServerConfigStreamEvent> | undefined;
197 readonly initialConfig?: (attempt: number) => Effect.Effect<ServerConfig, ConnectionAttemptError>;
198 readonly checkRoute?: (route: ConnectionRoute) => Effect.Effect<ConnectionDriver.RouteCheck>;
199 readonly checkHost?: (prepared: PreparedConnection) => Effect.Effect<ConnectionDriver.RouteCheck>;
200 readonly preflight?: (route: ConnectionRoute) => Effect.Effect<ConnectionDriver.RoutePreflight>;
201 readonly inboundFrames?: Effect.Effect<number>;
202 }) {
203 const networkStatus = yield* SubscriptionRef.make<NetworkStatus>(
204 options?.networkStatus ?? "online",
205 );
206 const prepareCount = yield* Ref.make(0);
207 const sessionCount = yield* Ref.make(0);
208 const releaseCount = yield* Ref.make(0);
209 const wakeups = yield* SubscriptionRef.make<{
210 readonly sequence: number;
211 readonly reason: ConnectionWakeups.ConnectionWakeup;
212 }>({
213 sequence: 0,
214 reason: "application-active",
215 });
216 const closedSessions = yield* Ref.make<
217 ReadonlyArray<Deferred.Deferred<never, ConnectionTransientError>>
218 >([]);
219
220 const connectivity = Connectivity.Connectivity.of({
221 status: SubscriptionRef.get(networkStatus),
222 changes: SubscriptionRef.changes(networkStatus),
223 });
224
225 const prepare = Effect.fn("TestConnectionDriver.prepare")(function* (target: ConnectionTarget) {
226 const attempt = yield* Ref.updateAndGet(prepareCount, (count) => count + 1);
227 if (options?.prepare) {
228 return yield* options.prepare(attempt, target);
229 }
230 return PREPARED_CONNECTION;
231 });
232
233 const checkRoute = (route: ConnectionRoute) =>
234 options?.checkRoute?.(route) ?? Effect.succeed<ConnectionDriver.RouteCheck>("unchecked");
235
236 const connectRoute = Effect.fn("TestConnectionDriver.connectRoute")(function* (
237 target: ConnectionTarget,
238 reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>,
239 ) {
240 yield* reportProgress({ stage: "preparing" });
241 const prepared = yield* prepare(target);
242 yield* reportProgress({ stage: "opening", prepared });
243
244 const attempt = yield* Ref.updateAndGet(sessionCount, (count) => count + 1);
245 const closed = yield* Deferred.make<never, ConnectionTransientError>();
246 yield* Ref.update(closedSessions, (sessions) => [...sessions, closed]);
247
248 const session = yield* Effect.acquireRelease(
249 Effect.succeed({
250 client: TEST_RPC_CLIENT,
251 initialConfig:
252 options?.initialConfig?.(attempt) ??
253 Effect.die(new Error("Initial config is not used by supervisor tests.")),
254 subscribeServerConfig: () =>
255 options?.configEvents ??
256 Stream.fromEffect(
257 (options?.initialConfig?.(attempt) ?? Effect.never).pipe(Effect.orDie),
258 ).pipe(
259 Stream.map((config) => ({ version: 1 as const, type: "snapshot" as const, config })),
260 ),
261 ready: options?.ready?.(attempt) ?? Effect.void,
262 probe: options?.probe?.(attempt) ?? Effect.void,
263 inboundFrames: options?.inboundFrames ?? Effect.succeed(0),
264 closed: Deferred.await(closed),
265 } satisfies RpcSession.RpcSession),
266 () =>
267 Ref.update(releaseCount, (count) => count + 1).pipe(
268 Effect.andThen(options?.release?.(attempt) ?? Effect.void),
269 ),
270 );
271
272 yield* reportProgress({ stage: "synchronizing", prepared });
273 yield* session.ready;
274 return { prepared, session } satisfies ConnectionDriver.RouteConnection;
275 });
276
277 const connect = Effect.fn("TestConnectionDriver.connect")(function* (
278 entry: ConnectionCatalogEntry,
279 reportProgress: (progress: ConnectionDriver.ConnectionDriverProgress) => Effect.Effect<void>,
280 ) {
281 yield* reportProgress({ stage: "preparing" });
282 return yield* ConnectionDriver.connectOverRoutes(entry, checkRoute, (route) =>
283 connectRoute(route.target, reportProgress),
284 );
285 });
286
287 const layerDependencies = Layer.mergeAll(
288 Layer.succeed(Random.Random, {
289 nextDoubleUnsafe: () => 1 - Number.EPSILON,
290 nextIntUnsafe: () => 0,
291 }),
292 Layer.succeed(Connectivity.Connectivity, connectivity),
293 Layer.succeed(
294 ConnectionWakeups.ConnectionWakeups,
295 ConnectionWakeups.ConnectionWakeups.of({
296 changes: SubscriptionRef.changes(wakeups).pipe(
297 Stream.drop(1),
298 Stream.map((event) => event.reason),
299 ),
300 }),
301 ),
302 Layer.succeed(
303 ConnectionDriver.ConnectionDriver,
304 ConnectionDriver.ConnectionDriver.of({
305 connect,
306 preflight: (_entry, route) =>
307 options?.preflight?.(route) ??
308 checkRoute(route).pipe(
309 Effect.map((check): ConnectionDriver.RoutePreflight =>
310 check === "answered" ? "ready" : check,
311 ),
312 ),
313 checkHost: (prepared) =>
314 options?.checkHost?.(prepared) ??
315 Effect.succeed<ConnectionDriver.RouteCheck>("unchecked"),
316 }),
317 ),
318 );
319
320 return {
321 dependencies: layerDependencies,
322 prepareCount,
323 sessionCount,
324 releaseCount,
325 setNetworkStatus: (status: NetworkStatus) => SubscriptionRef.set(networkStatus, status),
326 wake: (reason: ConnectionWakeups.ConnectionWakeup) =>
327 SubscriptionRef.update(wakeups, (event) => ({
328 sequence: event.sequence + 1,
329 reason,
330 })),
331 closeLatestSession: Effect.fn("TestConnectionHarness.closeLatestSession")(function* (
332 exit: Exit.Exit<never, ConnectionTransientError> = Exit.fail(transient("Session closed.")),
333 ) {
334 const sessions = yield* Ref.get(closedSessions);
335 const latest = sessions.at(-1);
336 if (latest) {
337 yield* Deferred.done(latest, exit);
338 }
339 }),
340 };
341 });
342
343 describe("EnvironmentSupervisor", () => {
344 it.effect("does not attempt a connection until it is desired", () =>
345 Effect.gen(function* () {
346 const harness = yield* makeHarness();
347 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe(
348 Effect.provide(harness.dependencies),
349 );
350
351 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("available");
352 expect(yield* Ref.get(harness.prepareCount)).toBe(0);
353 }),
354 );
355
356 it.effect("does not let the initial connect signal cancel the first attempt", () =>
357 Effect.gen(function* () {
358 const harness = yield* makeHarness();
359 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY).pipe(
360 Effect.provide(harness.dependencies),
361 );
362
363 yield* supervisor.connect;
364 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
365
366 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
367 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
368 }),
369 );
370
371 it.effect("waits while offline and connects immediately when the network returns", () =>
372 Effect.gen(function* () {
373 const harness = yield* makeHarness({ networkStatus: "offline" });
374 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
375 initiallyDesired: true,
376 }).pipe(Effect.provide(harness.dependencies));
377
378 yield* awaitState(supervisor.state, (state) => state.phase === "offline");
379 expect(yield* Ref.get(harness.prepareCount)).toBe(0);
380
381 yield* harness.setNetworkStatus("online");
382 const ready = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
383
384 expect(ready).toMatchObject({
385 desired: true,
386 network: "online",
387 phase: "connected",
388 attempt: 1,
389 generation: 1,
390 lastFailure: null,
391 });
392 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
393 }),
394 );
395
396 it.effect("explicit retry connects while the network still reports offline", () =>
397 Effect.gen(function* () {
398 const harness = yield* makeHarness({
399 networkStatus: "offline",
400 prepare: () => Effect.sleep("1 second").pipe(Effect.as(PREPARED_CONNECTION)),
401 });
402 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
403 initiallyDesired: true,
404 }).pipe(Effect.provide(harness.dependencies));
405
406 yield* awaitState(supervisor.state, (state) => state.phase === "offline");
407 // The loop wakes on the earlier signal and takes the retry's flag while
408 // RetryRequested is still queued, so that signal must not cancel the attempt.
409 yield* supervisor.connect;
410 yield* supervisor.retryNow;
411 yield* TestClock.adjust("1 second");
412 const settled = yield* awaitState(
413 supervisor.state,
414 (state) => state.phase === "connected" || state.phase === "offline",
415 );
416
417 expect(settled.phase).toBe("connected");
418 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
419 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
420 }),
421 );
422
423 it.effect.each([
424 { label: "cancels connection setup", retry: false, phase: "offline", sessions: 0 },
425 { label: "does not cancel an explicit retry", retry: true, phase: "connected", sessions: 1 },
426 ])("an offline report during setup $label", ({ retry, phase, sessions }) =>
427 Effect.gen(function* () {
428 const firstAttemptStarted = yield* Deferred.make<void>();
429 const harness = yield* makeHarness({
430 prepare: () =>
431 Deferred.succeed(firstAttemptStarted, undefined).pipe(
432 Effect.andThen(Effect.sleep("1 second")),
433 Effect.as(PREPARED_CONNECTION),
434 ),
435 });
436 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
437 initiallyDesired: true,
438 }).pipe(Effect.provide(harness.dependencies));
439
440 yield* Deferred.await(firstAttemptStarted);
441 if (retry) {
442 yield* supervisor.retryNow;
443 }
444 yield* harness.setNetworkStatus("offline");
445 yield* TestClock.adjust("1 second");
446 const settled = yield* awaitState(
447 supervisor.state,
448 (state) => state.phase === "connected" || state.phase === "offline",
449 );
450
451 expect(settled.phase).toBe(phase);
452 expect(yield* Ref.get(harness.sessionCount)).toBe(sessions);
453 }),
454 );
455
456 it.effect("connects to the primary environment while the device reports offline", () =>
457 Effect.gen(function* () {
458 const harness = yield* makeHarness({ networkStatus: "offline" });
459 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
460 initiallyDesired: true,
461 }).pipe(Effect.provide(harness.dependencies));
462
463 const settled = yield* awaitState(
464 supervisor.state,
465 (state) => state.phase === "connected" || state.phase === "offline",
466 );
467
468 expect(settled).toMatchObject({
469 phase: "connected",
470 network: "unknown",
471 attempt: 1,
472 lastFailure: null,
473 });
474 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
475 }),
476 );
477
478 it.effect(
479 "keeps a primary attempt through an offline report and restarts it when the network returns",
480 () =>
481 Effect.gen(function* () {
482 const started = yield* Deferred.make<void>();
483 const harness = yield* makeHarness({
484 prepare: (attempt) =>
485 attempt === 1
486 ? Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never))
487 : Effect.succeed(PREPARED_CONNECTION),
488 });
489 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
490 initiallyDesired: true,
491 }).pipe(Effect.provide(harness.dependencies));
492
493 yield* Deferred.await(started);
494 yield* harness.setNetworkStatus("offline");
495 yield* TestClock.adjust("1 second");
496 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
497
498 yield* harness.setNetworkStatus("online");
499 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
500 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
501 }).pipe(Effect.provide(TestClock.layer())),
502 );
503
504 it.effect("keeps the latest failure visible throughout the next connection attempt", () =>
505 Effect.gen(function* () {
506 const harness = yield* makeHarness({
507 prepare: (attempt) =>
508 attempt === 1 ? Effect.fail(transient("Relay connection timed out.")) : Effect.never,
509 });
510 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
511 initiallyDesired: true,
512 }).pipe(Effect.provide(harness.dependencies));
513
514 yield* awaitState(
515 supervisor.state,
516 (state) => state.phase === "backoff" && state.attempt === 1,
517 );
518 yield* TestClock.adjust("3 seconds");
519
520 const retrying = yield* awaitState(
521 supervisor.state,
522 (state) =>
523 state.phase === "connecting" && state.stage === "preparing" && state.attempt === 2,
524 );
525 expect(retrying).toMatchObject({
526 phase: "connecting",
527 stage: "preparing",
528 attempt: 2,
529 lastFailure: {
530 _tag: "ConnectionTransientError",
531 reason: "transport",
532 message: "Relay connection timed out.",
533 },
534 });
535 }).pipe(Effect.provide(TestClock.layer())),
536 );
537
538 it.effect("checks back on a pending approval every few seconds instead of backing off", () =>
539 Effect.gen(function* () {
540 const pending = new ConnectionTransientError({
541 reason: "approval-pending",
542 detail: "Approve this device on Mac mini. It shows code K7QM-2XFD.",
543 });
544 const harness = yield* makeHarness({
545 prepare: (attempt) =>
546 Effect.fail(attempt <= 8 ? pending : transient("Relay connection timed out.")),
547 });
548 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
549 initiallyDesired: true,
550 }).pipe(Effect.provide(harness.dependencies));
551
552 for (let attempt = 1; attempt <= 8; attempt++) {
553 const waiting = yield* awaitState(
554 supervisor.state,
555 (state) => state.phase === "backoff" && state.attempt === attempt,
556 );
557 expect((waiting.retryAt ?? 0) - (yield* Clock.currentTimeMillis)).toBe(4_000);
558 yield* TestClock.adjust("4 seconds");
559 }
560 // The waits do not count toward backoff, so an outage right after starts at the bottom.
561 const failed = yield* awaitState(
562 supervisor.state,
563 (state) => state.phase === "backoff" && state.attempt === 9,
564 );
565 expect((failed.retryAt ?? 0) - (yield* Clock.currentTimeMillis)).toBeLessThanOrEqual(2_000);
566 }).pipe(Effect.provide(TestClock.layer())),
567 );
568
569 it.effect("backs off a device nobody approved once its request has expired", () =>
570 Effect.gen(function* () {
571 const pending = new ConnectionTransientError({
572 reason: "approval-pending",
573 detail: "Approve this device on Mac mini. It shows code K7QM-2XFD.",
574 });
575 const harness = yield* makeHarness({ prepare: () => Effect.fail(pending) });
576 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
577 initiallyDesired: true,
578 }).pipe(Effect.provide(harness.dependencies));
579
580 const delays: number[] = [];
581 for (let attempt = 1; attempt <= 156; attempt++) {
582 const waiting = yield* awaitState(
583 supervisor.state,
584 (state) => state.phase === "backoff" && state.attempt === attempt,
585 );
586 const delay = (waiting.retryAt ?? 0) - (yield* Clock.currentTimeMillis);
587 delays.push(delay);
588 yield* TestClock.adjust(delay);
589 }
590 // Ten minutes of 4 s checks, then backoff that never polls faster than that.
591 expect(delays.slice(0, 150).every((delay) => delay === 4_000)).toBe(true);
592 expect(delays.slice(150).every((delay) => delay >= 4_000)).toBe(true);
593 expect(delays.at(-1)).toBeGreaterThanOrEqual(32_000);
594 }).pipe(Effect.provide(TestClock.layer())),
595 );
596
597 it.effect("converts unexpected driver defects into retryable failures", () =>
598 Effect.gen(function* () {
599 const harness = yield* makeHarness({
600 prepare: (attempt) =>
601 attempt === 1
602 ? Effect.die(new Error("Native transport defect."))
603 : Effect.succeed(PREPARED_CONNECTION),
604 });
605 const logs: unknown[] = [];
606 const annotations: unknown[] = [];
607 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
608 initiallyDesired: true,
609 }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))));
610
611 const failed = yield* awaitState(
612 supervisor.state,
613 (state) => state.phase === "backoff" && state.attempt === 1,
614 );
615 expect(failed).toMatchObject({
616 lastFailure: {
617 _tag: "ConnectionTransientError",
618 reason: "transport",
619 message: "Test environment connection failed unexpectedly.",
620 },
621 });
622 expect(logs).toEqual([["Connection attempt failed with an unexpected defect."]]);
623 expect(annotations).toMatchObject([
624 { "environment.id": RELAY_TARGET.environmentId, errorName: "Error" },
625 ]);
626
627 yield* TestClock.adjust("3 seconds");
628 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
629 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
630 }).pipe(Effect.provide(TestClock.layer())),
631 );
632
633 it.effect.each([
634 {
635 label: "the attempt's network-status read dies",
636 firstPrepare: Effect.never,
637 statusDefect: true,
638 log: "Connection attempt failed with an unexpected defect.",
639 },
640 {
641 label: "an attempt interrupts itself",
642 firstPrepare: Effect.interrupt,
643 statusDefect: false,
644 log: "Connection attempt failed with an unexpected interruption.",
645 },
646 ])("retries after $label", ({ firstPrepare, statusDefect, log }) =>
647 Effect.gen(function* () {
648 const harness = yield* makeHarness({
649 prepare: (attempt) => (attempt === 1 ? firstPrepare : Effect.succeed(PREPARED_CONNECTION)),
650 });
651 const statusReads = yield* Ref.make(0);
652 const logs: unknown[] = [];
653 const annotations: Array<Record<string, unknown>> = [];
654 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
655 initiallyDesired: true,
656 }).pipe(
657 Effect.provideService(
658 Connectivity.Connectivity,
659 Connectivity.Connectivity.of({
660 // The supervisor reads once at startup; the second read belongs to the first attempt.
661 status: Ref.updateAndGet(statusReads, (count) => count + 1).pipe(
662 Effect.flatMap((count) =>
663 statusDefect && count === 2
664 ? Effect.die(new Error("Platform connectivity defect."))
665 : Effect.succeed("online" as const),
666 ),
667 ),
668 changes: Stream.never,
669 }),
670 ),
671 Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs, annotations))),
672 );
673
674 const failed = yield* awaitState(
675 supervisor.state,
676 (state) => state.phase === "backoff" && state.attempt === 1,
677 );
678 expect(failed).toMatchObject({
679 generation: 0,
680 lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
681 });
682 expect(logs).toEqual([[log]]);
683 expect(Object.hasOwn(annotations[0]!, "errorType")).toBe(statusDefect);
684
685 yield* TestClock.adjust("999 millis");
686 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
687 yield* TestClock.adjust("2 seconds");
688 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
689 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
690 }).pipe(Effect.provide(TestClock.layer())),
691 );
692
693 it.effect.each([
694 {
695 label: "is interrupted",
696 closed: Exit.interrupt(),
697 releaseDefect: false,
698 log: "Connection attempt failed with an unexpected interruption.",
699 },
700 {
701 label: "fails while interrupted",
702 closed: Exit.failCause(
703 Cause.combine(Cause.fail(transient("Session closed.")), Cause.interrupt()),
704 ),
705 releaseDefect: false,
706 log: null,
707 },
708 {
709 label: "closes while its release defects",
710 closed: Exit.fail(transient("Session closed.")),
711 releaseDefect: true,
712 log: "Connection release failed with an unexpected defect.",
713 },
714 ])("reconnects soon after a stable session $label", ({ closed, releaseDefect, log }) =>
715 Effect.gen(function* () {
716 const harness = yield* makeHarness({
717 prepare: (attempt) =>
718 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
719 release: (session) =>
720 releaseDefect && session === 1
721 ? Effect.die(new Error("Session release defect."))
722 : Effect.void,
723 });
724 const logs: unknown[] = [];
725 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
726 initiallyDesired: true,
727 }).pipe(Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))));
728
729 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
730 yield* TestClock.adjust("2 seconds");
731 yield* awaitState(
732 supervisor.state,
733 (state) => state.phase === "connected" && state.generation === 1,
734 );
735 yield* TestClock.adjust("1 minute");
736 yield* harness.closeLatestSession(closed);
737 const lost = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
738 expect(logs).toEqual(log === null ? [] : [[log]]);
739 expect(lost).toMatchObject({ phase: "backoff", generation: 1 });
740 expect(lost.retryAt).toBeLessThanOrEqual((yield* Clock.currentTimeMillis) + 250);
741
742 yield* TestClock.adjust("250 millis");
743 const replaced = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
744 expect(replaced.generation).toBe(2);
745 }).pipe(Effect.provide(TestClock.layer())),
746 );
747
748 it.effect(
749 "does not reuse a live session's generation when its attempt defects after going live",
750 () =>
751 Effect.gen(function* () {
752 const harness = yield* makeHarness({
753 prepare: (attempt) =>
754 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
755 });
756 let logCalls = 0;
757 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
758 initiallyDesired: true,
759 }).pipe(
760 Effect.provide(
761 Layer.merge(
762 harness.dependencies,
763 Logger.layer(
764 [
765 // Logging the lost session's interruption throws, so the attempt itself defects.
766 Logger.make(() => {
767 logCalls += 1;
768 if (logCalls === 1) throw new Error("Logger defect.");
769 }),
770 ],
771 { mergeWithExisting: false },
772 ),
773 ),
774 ),
775 );
776
777 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
778 yield* TestClock.adjust("2 seconds");
779 yield* awaitState(
780 supervisor.state,
781 (state) => state.phase === "connected" && state.generation === 1,
782 );
783 yield* harness.closeLatestSession(Exit.interrupt());
784 const backoff = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
785 expect(backoff).toMatchObject({ generation: 1, lastFailure: null });
786
787 yield* TestClock.adjust("4 seconds");
788 const replaced = yield* awaitState(
789 supervisor.state,
790 (state) => state.phase === "connected",
791 );
792 expect(replaced.generation).toBe(2);
793 }).pipe(Effect.provide(TestClock.layer())),
794 );
795
796 it.effect("restarts the loop after a defect outside an attempt", () =>
797 Effect.gen(function* () {
798 const harness = yield* makeHarness();
799 const logs: unknown[] = [];
800 let randomReads = 0;
801 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
802 initiallyDesired: true,
803 }).pipe(
804 Effect.provideService(Random.Random, {
805 // Both session losses below die here, so the second reaches the restart handler again.
806 nextDoubleUnsafe: () => {
807 randomReads += 1;
808 if (randomReads <= 2) throw new Error("Random defect.");
809 return 1 - Number.EPSILON;
810 },
811 nextIntUnsafe: () => 0,
812 }),
813 Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
814 );
815
816 yield* awaitState(
817 supervisor.state,
818 (state) => state.phase === "connected" && state.generation === 1,
819 );
820 yield* harness.closeLatestSession();
821 const restarted = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
822 expect(logs).toEqual([["Connection loop failed with an unexpected defect."]]);
823 expect(restarted).toMatchObject({
824 attempt: 1,
825 generation: 1,
826 lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
827 retryAt: (yield* Clock.currentTimeMillis) + 300_000,
828 });
829
830 yield* TestClock.adjust("299999 millis");
831 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
832 yield* TestClock.adjust("1 millis");
833 const reconnected = yield* awaitState(
834 supervisor.state,
835 (state) => state.phase === "connected",
836 );
837 expect(reconnected.generation).toBe(2);
838
839 yield* harness.setNetworkStatus("offline");
840 yield* harness.closeLatestSession();
841 const offline = yield* awaitState(supervisor.state, (state) => state.phase !== "connected");
842 expect(offline).toMatchObject({
843 phase: "offline",
844 attempt: 1,
845 lastFailure: { _tag: "ConnectionTransientError", reason: "transport" },
846 });
847
848 yield* supervisor.retryNow;
849 const retried = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
850 expect(retried.generation).toBe(3);
851 }).pipe(Effect.provide(TestClock.layer())),
852 );
853
854 it.effect.each(["connecting", "connected"] as const)(
855 "stops attempting when its scope closes while %s",
856 (phase) =>
857 Effect.gen(function* () {
858 const preparing = yield* Deferred.make<void>();
859 const harness = yield* makeHarness({
860 prepare: () =>
861 Deferred.succeed(preparing, undefined).pipe(
862 Effect.andThen(
863 phase === "connected" ? Effect.succeed(PREPARED_CONNECTION) : Effect.never,
864 ),
865 ),
866 });
867 const logs: unknown[] = [];
868 const scope = yield* Scope.make();
869 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
870 initiallyDesired: true,
871 }).pipe(
872 Effect.provide(Layer.merge(harness.dependencies, recordLogs(logs))),
873 Scope.provide(scope),
874 );
875
876 yield* Deferred.await(preparing);
877 yield* awaitState(supervisor.state, (state) => state.phase === phase);
878 // Yielding between finalizers lets the loop run if the queue shuts down before its interrupt.
879 yield* Scope.close(scope, Exit.void).pipe(
880 Effect.provideService(References.MaxOpsBeforeYield, 1),
881 );
882 yield* TestClock.adjust("10 minutes");
883 expect(logs).toEqual([]);
884 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
885 }).pipe(Effect.provide(TestClock.layer())),
886 );
887
888 it.effect("explicit retry interrupts the current backoff", () =>
889 Effect.gen(function* () {
890 const harness = yield* makeHarness({
891 prepare: (attempt) =>
892 attempt === 1 ? Effect.fail(transient()) : Effect.succeed(PREPARED_CONNECTION),
893 });
894 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
895 initiallyDesired: true,
896 }).pipe(Effect.provide(harness.dependencies));
897
898 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
899 yield* supervisor.retryNow;
900 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
901
902 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
903 }),
904 );
905
906 it.effect.each([
907 { label: "fails", cause: Cause.fail(blocked()) },
908 {
909 label: "fails while interrupted",
910 cause: Cause.combine(Cause.fail(blocked()), Cause.interrupt()),
911 },
912 ])(
913 "keeps a blocked host that $label idle until an external signal requests another attempt",
914 ({ cause }) =>
915 Effect.gen(function* () {
916 const harness = yield* makeHarness();
917 const driver = yield* ConnectionDriver.ConnectionDriver.pipe(
918 Effect.provide(harness.dependencies),
919 );
920 const connects = yield* Ref.make(0);
921 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
922 initiallyDesired: true,
923 }).pipe(
924 // The driver keeps only the typed error of a failed route, so the cause is mixed here.
925 Effect.provideService(ConnectionDriver.ConnectionDriver, {
926 ...driver,
927 connect: (entry, reportProgress) =>
928 Ref.updateAndGet(connects, (count) => count + 1).pipe(
929 Effect.flatMap((count) =>
930 count === 1 ? Effect.failCause(cause) : driver.connect(entry, reportProgress),
931 ),
932 ),
933 }),
934 Effect.provide(harness.dependencies),
935 );
936
937 const failed = yield* awaitState(supervisor.state, (state) => state.lastFailure !== null);
938 expect(failed).toMatchObject({
939 phase: "blocked",
940 lastFailure: { _tag: "ConnectionBlockedError", detail: "Authentication required." },
941 });
942 yield* TestClock.adjust("1 hour");
943 expect(yield* Ref.get(connects)).toBe(1);
944
945 yield* supervisor.retryNow;
946 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
947 expect(yield* Ref.get(connects)).toBe(2);
948 }).pipe(Effect.provide(TestClock.layer())),
949 );
950
951 it.effect("retries a blocked connection when platform credentials change", () =>
952 Effect.gen(function* () {
953 const harness = yield* makeHarness({
954 prepare: (attempt) =>
955 attempt === 1 ? Effect.fail(blocked()) : Effect.succeed(PREPARED_CONNECTION),
956 });
957 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
958 initiallyDesired: true,
959 }).pipe(Effect.provide(harness.dependencies));
960
961 yield* awaitState(supervisor.state, (state) => state.phase === "blocked");
962 yield* harness.wake("credentials-changed");
963 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
964
965 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
966 }),
967 );
968
969 it.effect("does not let platform wakeups reset an in-flight attempt", () =>
970 Effect.gen(function* () {
971 const firstAttemptStarted = yield* Deferred.make<void>();
972 const harness = yield* makeHarness({
973 prepare: () =>
974 Deferred.succeed(firstAttemptStarted, undefined).pipe(Effect.andThen(Effect.never)),
975 });
976 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
977 initiallyDesired: true,
978 }).pipe(Effect.provide(harness.dependencies));
979
980 yield* Deferred.await(firstAttemptStarted);
981 yield* Effect.all(
982 [
983 harness.wake("credentials-changed"),
984 harness.wake("application-active"),
985 harness.wake("credentials-changed"),
986 ],
987 { concurrency: "unbounded" },
988 );
989 yield* Effect.yieldNow;
990
991 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
992
993 yield* TestClock.adjust("15 seconds");
994 const retrying = yield* awaitState(
995 supervisor.state,
996 (state) => state.phase === "backoff" && state.attempt === 1,
997 );
998
999 expect(retrying).toMatchObject({
1000 lastFailure: {
1001 _tag: "ConnectionTransientError",
1002 reason: "timeout",
1003 message: "Test environment did not respond during connection setup.",
1004 },
1005 });
1006 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
1007 expect(yield* Ref.get(harness.sessionCount)).toBe(0);
1008 }).pipe(Effect.provide(TestClock.layer())),
1009 );
1010
1011 it.effect("reports a slow SSH preparation failure instead of the setup timeout", () =>
1012 Effect.gen(function* () {
1013 const preparationFailure = new ConnectionTransientError({
1014 reason: "remote-unavailable",
1015 detail:
1016 "Could not prepare the SSH environment: curl: (28) Operation timed out after 240003 milliseconds",
1017 });
1018 const harness = yield* makeHarness({
1019 prepare: () =>
1020 Effect.sleep("4 minutes").pipe(Effect.andThen(Effect.fail(preparationFailure))),
1021 });
1022 const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, {
1023 initiallyDesired: true,
1024 }).pipe(Effect.provide(harness.dependencies));
1025
1026 yield* awaitState(
1027 supervisor.state,
1028 (state) => state.phase === "connecting" && state.stage === "preparing",
1029 );
1030 const firstFailure = yield* awaitState(
1031 supervisor.state,
1032 (state) => state.phase === "backoff",
1033 ).pipe(Effect.forkChild({ startImmediately: true }));
1034 yield* TestClock.adjust("4 minutes");
1035
1036 expect(yield* Fiber.join(firstFailure)).toMatchObject({
1037 attempt: 1,
1038 lastFailure: {
1039 _tag: "ConnectionTransientError",
1040 reason: "remote-unavailable",
1041 message: preparationFailure.message,
1042 },
1043 });
1044 }).pipe(Effect.provide(TestClock.layer())),
1045 );
1046
1047 it.effect("times out an SSH route that stops answering after a slow preparation", () =>
1048 Effect.gen(function* () {
1049 const harness = yield* makeHarness({
1050 prepare: () => Effect.sleep("4 minutes").pipe(Effect.as(PREPARED_CONNECTION)),
1051 ready: () => Effect.never,
1052 });
1053 const supervisor = yield* EnvironmentSupervisor.make(SSH_ENTRY, {
1054 initiallyDesired: true,
1055 }).pipe(Effect.provide(harness.dependencies));
1056
1057 yield* awaitState(
1058 supervisor.state,
1059 (state) => state.phase === "connecting" && state.stage === "preparing",
1060 );
1061 yield* TestClock.adjust("4 minutes");
1062 yield* awaitState(
1063 supervisor.state,
1064 (state) => state.phase === "connecting" && state.stage === "synchronizing",
1065 );
1066 const firstFailure = yield* awaitState(
1067 supervisor.state,
1068 (state) => state.phase === "backoff",
1069 ).pipe(Effect.forkChild({ startImmediately: true }));
1070 yield* TestClock.adjust("90 seconds");
1071
1072 expect(yield* Fiber.join(firstFailure)).toMatchObject({
1073 attempt: 1,
1074 lastFailure: {
1075 _tag: "ConnectionTransientError",
1076 reason: "timeout",
1077 message: "Test environment did not respond during connection setup.",
1078 },
1079 });
1080 }).pipe(Effect.provide(TestClock.layer())),
1081 );
1082
1083 it.effect("does not time a second SSH route's slow preparation", () =>
1084 Effect.gen(function* () {
1085 const secondSshTarget = new SshConnectionTarget({
1086 environmentId: TARGET.environmentId,
1087 label: TARGET.label,
1088 connectionId: "ssh-connection-2",
1089 });
1090 const firstRouteFailure = transient("The first SSH route dropped while synchronizing.");
1091 const harness = yield* makeHarness({
1092 prepare: (attempt, target) =>
1093 attempt === 1
1094 ? Effect.succeed(preparedFor(target))
1095 : Effect.sleep("4 minutes").pipe(
1096 Effect.andThen(
1097 Effect.fail(
1098 new ConnectionTransientError({
1099 reason: "remote-unavailable",
1100 detail: "Could not prepare the SSH environment.",
1101 }),
1102 ),
1103 ),
1104 ),
1105 ready: () => Effect.fail(firstRouteFailure),
1106 });
1107 const supervisor = yield* EnvironmentSupervisor.make(
1108 { ...SSH_ENTRY, alternateRoutes: [{ target: secondSshTarget, profile: Option.none() }] },
1109 { initiallyDesired: true },
1110 ).pipe(Effect.provide(harness.dependencies));
1111
1112 yield* awaitState(
1113 supervisor.state,
1114 (state) => state.phase === "connecting" && state.stage === "preparing",
1115 );
1116 const firstFailure = yield* awaitState(
1117 supervisor.state,
1118 (state) => state.phase === "backoff",
1119 ).pipe(Effect.forkChild({ startImmediately: true }));
1120 yield* TestClock.adjust("4 minutes");
1121
1122 // The driver reports the first transient failure across its routes.
1123 expect(yield* Fiber.join(firstFailure)).toMatchObject({
1124 attempt: 1,
1125 lastFailure: { reason: "transport", message: firstRouteFailure.message },
1126 });
1127 expect(yield* Ref.get(harness.prepareCount)).toBe(2);
1128 }).pipe(Effect.provide(TestClock.layer())),
1129 );
1130
1131 it.effect("treats an involuntary session close as transient and reconnects", () =>
1132 Effect.gen(function* () {
1133 const harness = yield* makeHarness();
1134 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1135 initiallyDesired: true,
1136 }).pipe(Effect.provide(harness.dependencies));
1137
1138 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1139 yield* harness.closeLatestSession();
1140 yield* awaitState(
1141 supervisor.state,
1142 (state) => state.phase === "backoff" && state.attempt === 1,
1143 );
1144 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
1145
1146 yield* TestClock.adjust("250 millis");
1147 yield* awaitState(
1148 supervisor.state,
1149 (state) => state.phase === "connected" && state.generation === 2,
1150 );
1151
1152 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1153 expect(Option.isSome(yield* SubscriptionRef.get(supervisor.prepared))).toBe(true);
1154 }).pipe(Effect.provide(TestClock.layer())),
1155 );
1156
1157 it.effect("does not report a session lost while suspended as a failed connection", () =>
1158 Effect.gen(function* () {
1159 const allowReconnect = yield* Deferred.make<void>();
1160 const harness = yield* makeHarness({
1161 prepare: (attempt) =>
1162 attempt === 2
1163 ? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION))
1164 : Effect.succeed(PREPARED_CONNECTION),
1165 });
1166 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1167 initiallyDesired: true,
1168 }).pipe(Effect.provide(harness.dependencies));
1169
1170 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1171 // The OS killed the socket while the app was suspended; its close arrives
1172 // before the app reports it is active again.
1173 yield* harness.closeLatestSession();
1174 const waiting = yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
1175 expect(presentConnectionState(waiting)).toEqual({
1176 phase: "reconnecting",
1177 error: null,
1178 traceId: null,
1179 });
1180
1181 yield* harness.wake("application-active");
1182 const replacing = yield* awaitState(
1183 supervisor.state,
1184 (state) => state.phase === "connecting" && state.stage === "preparing",
1185 );
1186 expect(presentConnectionState(replacing).error).toBeNull();
1187
1188 yield* Deferred.succeed(allowReconnect, undefined);
1189 yield* awaitState(
1190 supervisor.state,
1191 (state) => state.phase === "connected" && state.generation === 2,
1192 );
1193 }).pipe(Effect.provide(TestClock.layer())),
1194 );
1195
1196 it.effect("restarts the retry ladder when a resume replaces a silent socket", () =>
1197 Effect.gen(function* () {
1198 const harness = yield* makeHarness({
1199 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1200 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1201 probe: (attempt) => (attempt === 2 ? Effect.never : Effect.void),
1202 checkHost: () => Effect.succeed("answered"),
1203 });
1204 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1205 initiallyDesired: true,
1206 }).pipe(Effect.provide(harness.dependencies));
1207
1208 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1209 yield* harness.closeLatestSession();
1210 yield* awaitState(
1211 supervisor.state,
1212 (state) => state.phase === "backoff" && state.attempt === 1,
1213 );
1214 yield* TestClock.adjust("2 seconds");
1215 yield* awaitState(
1216 supervisor.state,
1217 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 2,
1218 );
1219
1220 yield* harness.wake("application-active-long");
1221 yield* TestClock.adjust("500 millis");
1222 yield* awaitState(
1223 supervisor.state,
1224 (state) => state.phase === "connected" && state.generation === 3 && state.attempt === 1,
1225 );
1226 }).pipe(Effect.provide(TestClock.layer())),
1227 );
1228
1229 it.effect("restarts the retry ladder when a long resume interrupts connection setup", () =>
1230 Effect.gen(function* () {
1231 const harness = yield* makeHarness({
1232 prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)),
1233 });
1234 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1235 initiallyDesired: true,
1236 }).pipe(Effect.provide(harness.dependencies));
1237
1238 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1239 yield* harness.closeLatestSession();
1240 yield* awaitState(
1241 supervisor.state,
1242 (state) => state.phase === "backoff" && state.attempt === 1,
1243 );
1244 yield* TestClock.adjust("3 seconds");
1245 yield* awaitState(
1246 supervisor.state,
1247 (state) => state.phase === "connecting" && state.attempt === 2,
1248 );
1249
1250 yield* harness.wake("application-active-long");
1251 yield* awaitState(
1252 supervisor.state,
1253 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1254 );
1255 }).pipe(Effect.provide(TestClock.layer())),
1256 );
1257
1258 it.effect("probes the active session without reconnecting on application activation", () =>
1259 Effect.gen(function* () {
1260 const probeCount = yield* Ref.make(0);
1261 const probeCalled = yield* Deferred.make<void>();
1262 const harness = yield* makeHarness({
1263 probe: () =>
1264 Ref.update(probeCount, (count) => count + 1).pipe(
1265 Effect.andThen(Deferred.succeed(probeCalled, undefined)),
1266 ),
1267 });
1268 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1269 initiallyDesired: true,
1270 }).pipe(Effect.provide(harness.dependencies));
1271
1272 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1273 yield* harness.wake("application-active");
1274 yield* Deferred.await(probeCalled);
1275
1276 expect(yield* Ref.get(probeCount)).toBe(1);
1277 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1278 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1279 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
1280 }),
1281 );
1282
1283 it.effect("keeps a live mobile session after a long background resume", () =>
1284 Effect.gen(function* () {
1285 const probeCount = yield* Ref.make(0);
1286 const harness = yield* makeHarness({
1287 probe: () => Ref.update(probeCount, (count) => count + 1),
1288 });
1289 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1290 initiallyDesired: true,
1291 }).pipe(Effect.provide(harness.dependencies));
1292
1293 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1294 yield* harness.wake("application-active-long");
1295 yield* TestClock.adjust("5 seconds");
1296
1297 expect(yield* Ref.get(probeCount)).toBe(1);
1298 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1299 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1300 phase: "connected",
1301 generation: 1,
1302 });
1303 }).pipe(Effect.provide(TestClock.layer())),
1304 );
1305
1306 const makeSilentSocketHarness = (options?: {
1307 readonly probe?: (attempt: number) => Effect.Effect<void, ConnectionAttemptError>;
1308 readonly checkHost?: (check: number) => ConnectionDriver.RouteCheck;
1309 }) =>
1310 Effect.gen(function* () {
1311 const hostChecks = yield* Ref.make<ReadonlyArray<string>>([]);
1312 const harness = yield* makeHarness({
1313 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1314 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1315 probe: options?.probe ?? ((attempt) => (attempt === 1 ? Effect.never : Effect.void)),
1316 checkHost: (prepared) =>
1317 Ref.updateAndGet(hostChecks, (checks) => [...checks, prepared.target._tag]).pipe(
1318 Effect.map((checks) => options?.checkHost?.(checks.length) ?? "answered"),
1319 ),
1320 });
1321 return { ...harness, hostChecks };
1322 });
1323
1324 it.effect.each(["application-active-probe", "application-active-long"] as const)(
1325 "replaces a LAN socket that stays silent while its host answers afresh (%s)",
1326 (wakeup) =>
1327 Effect.gen(function* () {
1328 const harness = yield* makeSilentSocketHarness();
1329 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1330 initiallyDesired: true,
1331 }).pipe(Effect.provide(harness.dependencies));
1332
1333 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1334 yield* harness.wake(wakeup);
1335 yield* TestClock.adjust("499 millis");
1336 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1337
1338 yield* TestClock.adjust("1 millis");
1339 yield* awaitState(
1340 supervisor.state,
1341 (state) => state.phase === "connected" && state.generation === 2,
1342 );
1343 expect(yield* Ref.get(harness.hostChecks)).toEqual(["BearerConnectionTarget"]);
1344 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1345 }).pipe(Effect.provide(TestClock.layer())),
1346 );
1347
1348 it.effect("replaces a remote socket only after 2.5 seconds of silence", () =>
1349 Effect.gen(function* () {
1350 const harness = yield* makeSilentSocketHarness();
1351 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1352 initiallyDesired: true,
1353 }).pipe(Effect.provide(harness.dependencies));
1354
1355 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1356 yield* harness.wake("application-active-probe");
1357 yield* TestClock.adjust("2499 millis");
1358 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1359
1360 yield* TestClock.adjust("1 millis");
1361 yield* awaitState(
1362 supervisor.state,
1363 (state) => state.phase === "connected" && state.generation === 2,
1364 );
1365 expect(yield* Ref.get(harness.hostChecks)).toEqual(["RelayConnectionTarget"]);
1366 }).pipe(Effect.provide(TestClock.layer())),
1367 );
1368
1369 it.effect("replaces a silent remote socket without the wait after a long resume", () =>
1370 Effect.gen(function* () {
1371 const harness = yield* makeSilentSocketHarness();
1372 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1373 initiallyDesired: true,
1374 }).pipe(Effect.provide(harness.dependencies));
1375
1376 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1377 yield* harness.wake("application-active-long");
1378 yield* TestClock.adjust("499 millis");
1379 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1380
1381 yield* TestClock.adjust("1 millis");
1382 yield* awaitState(
1383 supervisor.state,
1384 (state) => state.phase === "connected" && state.generation === 2,
1385 );
1386 }).pipe(Effect.provide(TestClock.layer())),
1387 );
1388
1389 it.effect("replaces a silent socket even while the network reports offline", () =>
1390 Effect.gen(function* () {
1391 const harness = yield* makeSilentSocketHarness();
1392 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1393 initiallyDesired: true,
1394 }).pipe(Effect.provide(harness.dependencies));
1395
1396 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1397 yield* harness.setNetworkStatus("offline");
1398 yield* TestClock.adjust("500 millis");
1399 yield* awaitState(
1400 supervisor.state,
1401 (state) => state.phase === "connected" && state.generation === 2,
1402 );
1403 }).pipe(Effect.provide(TestClock.layer())),
1404 );
1405
1406 it.effect("checks the primary environment's socket when the device reports offline", () =>
1407 Effect.gen(function* () {
1408 const harness = yield* makeSilentSocketHarness();
1409 const target = new PrimaryConnectionTarget({
1410 ...TARGET,
1411 httpBaseUrl: "http://127.0.0.1:3773",
1412 wsBaseUrl: "ws://127.0.0.1:3773",
1413 });
1414 const supervisor = yield* EnvironmentSupervisor.make(
1415 { ...TARGET_ENTRY, target },
1416 { initiallyDesired: true },
1417 ).pipe(Effect.provide(harness.dependencies));
1418
1419 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1420 yield* harness.setNetworkStatus("offline");
1421 yield* TestClock.adjust("500 millis");
1422 yield* awaitState(
1423 supervisor.state,
1424 (state) => state.phase === "connected" && state.generation === 2,
1425 );
1426 }).pipe(Effect.provide(TestClock.layer())),
1427 );
1428
1429 it.effect("replaces a remote socket after a long resume while the network reports offline", () =>
1430 Effect.gen(function* () {
1431 const harness = yield* makeSilentSocketHarness();
1432 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1433 initiallyDesired: true,
1434 }).pipe(Effect.provide(harness.dependencies));
1435
1436 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1437 yield* harness.setNetworkStatus("offline");
1438 yield* harness.wake("application-active-long");
1439 yield* TestClock.adjust("2500 millis");
1440 yield* awaitState(
1441 supervisor.state,
1442 (state) => state.phase === "connected" && state.generation === 2,
1443 );
1444 }).pipe(Effect.provide(TestClock.layer())),
1445 );
1446
1447 it.effect("checks the host afresh when the network changes after a frame", () =>
1448 Effect.gen(function* () {
1449 const frames = yield* Ref.make(0);
1450 const harness = yield* makeHarness({
1451 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1452 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1453 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1454 inboundFrames: Ref.get(frames),
1455 checkHost: () => Effect.succeed("answered"),
1456 });
1457 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1458 initiallyDesired: true,
1459 }).pipe(Effect.provide(harness.dependencies));
1460
1461 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1462 yield* harness.wake("application-active-probe");
1463 yield* TestClock.adjust("1 second");
1464 yield* harness.wake("network-changed");
1465 yield* TestClock.adjust("500 millis");
1466 yield* Ref.update(frames, (count) => count + 1);
1467 // The running check sees the frame at 1.6 s, so a fresh one starts then.
1468 yield* TestClock.adjust("2599 millis");
1469 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1470
1471 yield* TestClock.adjust("1 millis");
1472 yield* awaitState(
1473 supervisor.state,
1474 (state) => state.phase === "connected" && state.generation === 2,
1475 );
1476 }).pipe(Effect.provide(TestClock.layer())),
1477 );
1478
1479 it.effect("keeps a running check's window when the network changes during a long resume", () =>
1480 Effect.gen(function* () {
1481 const harness = yield* makeSilentSocketHarness();
1482 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1483 initiallyDesired: true,
1484 }).pipe(Effect.provide(harness.dependencies));
1485
1486 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1487 yield* harness.wake("application-active-long");
1488 yield* TestClock.adjust("1 second");
1489 yield* harness.wake("network-changed");
1490 yield* harness.setNetworkStatus("offline");
1491 yield* TestClock.adjust("1500 millis");
1492 yield* awaitState(
1493 supervisor.state,
1494 (state) => state.phase === "connected" && state.generation === 2,
1495 );
1496 }).pipe(Effect.provide(TestClock.layer())),
1497 );
1498
1499 it.effect("checks the host again when one check goes unanswered", () =>
1500 Effect.gen(function* () {
1501 const harness = yield* makeSilentSocketHarness({
1502 checkHost: (check) => (check === 1 ? "silent" : "answered"),
1503 });
1504 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1505 initiallyDesired: true,
1506 }).pipe(Effect.provide(harness.dependencies));
1507
1508 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1509 yield* harness.wake("application-active-probe");
1510 yield* TestClock.adjust("1499 millis");
1511 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1512
1513 yield* TestClock.adjust("1 millis");
1514 yield* awaitState(
1515 supervisor.state,
1516 (state) => state.phase === "connected" && state.generation === 2,
1517 );
1518 expect(yield* Ref.get(harness.hostChecks)).toHaveLength(2);
1519 }).pipe(Effect.provide(TestClock.layer())),
1520 );
1521
1522 it.effect("leaves a web or desktop foreground probe to its own deadline", () =>
1523 Effect.gen(function* () {
1524 const harness = yield* makeSilentSocketHarness({
1525 probe: (attempt) => (attempt === 1 ? Effect.sleep("4 seconds") : Effect.void),
1526 });
1527 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1528 initiallyDesired: true,
1529 }).pipe(Effect.provide(harness.dependencies));
1530
1531 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1532 yield* harness.wake("application-active");
1533 yield* TestClock.adjust("20 seconds");
1534
1535 expect(yield* Ref.get(harness.hostChecks)).toEqual([]);
1536 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1537 }).pipe(Effect.provide(TestClock.layer())),
1538 );
1539
1540 it.effect("checks the host when a network change arrives during a foreground probe", () =>
1541 Effect.gen(function* () {
1542 const harness = yield* makeSilentSocketHarness();
1543 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1544 initiallyDesired: true,
1545 }).pipe(Effect.provide(harness.dependencies));
1546
1547 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1548 yield* harness.wake("application-active");
1549 yield* TestClock.adjust("100 millis");
1550 yield* harness.wake("network-changed");
1551 yield* TestClock.adjust("499 millis");
1552 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1553
1554 yield* TestClock.adjust("1 millis");
1555 yield* awaitState(
1556 supervisor.state,
1557 (state) => state.phase === "connected" && state.generation === 2,
1558 );
1559 }).pipe(Effect.provide(TestClock.layer())),
1560 );
1561
1562 it.effect.each(["application-active-probe", "application-active-long"] as const)(
1563 "keeps waiting for the probe when a frame arrives during the host check (%s)",
1564 (wakeup) =>
1565 Effect.gen(function* () {
1566 const frames = yield* Ref.make(0);
1567 const harness = yield* makeHarness({
1568 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1569 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1570 probe: (attempt) => (attempt === 1 ? Effect.sleep("2 seconds") : Effect.void),
1571 inboundFrames: Ref.get(frames),
1572 checkHost: () => Ref.update(frames, (count) => count + 1).pipe(Effect.as("answered")),
1573 });
1574 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1575 initiallyDesired: true,
1576 }).pipe(Effect.provide(harness.dependencies));
1577
1578 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1579 yield* harness.wake(wakeup);
1580 yield* TestClock.adjust("5 seconds");
1581
1582 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1583 expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);
1584 }).pipe(Effect.provide(TestClock.layer())),
1585 );
1586
1587 it.effect("keeps a long-resume socket that delivers a frame during a slow host check", () =>
1588 Effect.gen(function* () {
1589 const frames = yield* Ref.make(0);
1590 const harness = yield* makeHarness({
1591 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1592 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1593 probe: (attempt) => (attempt === 1 ? Effect.sleep("2500 millis") : Effect.void),
1594 inboundFrames: Ref.get(frames),
1595 checkHost: () => Effect.sleep("2 seconds").pipe(Effect.as("answered")),
1596 });
1597 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1598 initiallyDesired: true,
1599 }).pipe(Effect.provide(harness.dependencies));
1600
1601 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1602 yield* harness.wake("application-active-long");
1603 yield* TestClock.adjust("700 millis");
1604 yield* Ref.update(frames, (count) => count + 1);
1605 yield* TestClock.adjust("5 seconds");
1606
1607 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1608 expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);
1609 }).pipe(Effect.provide(TestClock.layer())),
1610 );
1611
1612 it.effect("waits out a LAN resume probe when the host does not answer a fresh request", () =>
1613 Effect.gen(function* () {
1614 const harness = yield* makeHarness({
1615 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
1616 initialConfig: () => Effect.succeed(PROBE_CAPABLE_CONFIG),
1617 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1618 checkHost: () => Effect.succeed("silent"),
1619 });
1620 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1621 initiallyDesired: true,
1622 }).pipe(Effect.provide(harness.dependencies));
1623
1624 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1625 yield* harness.wake("application-active-probe");
1626 yield* TestClock.adjust("2999 millis");
1627 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1628
1629 yield* TestClock.adjust("1 millis");
1630 yield* awaitState(
1631 supervisor.state,
1632 (state) => state.phase === "connected" && state.generation === 2,
1633 );
1634 }).pipe(Effect.provide(TestClock.layer())),
1635 );
1636
1637 it.effect.each(["application-active-probe", "network-changed"] as const)(
1638 "gives a live remote socket 15 seconds to answer a %s probe",
1639 (wakeup) =>
1640 Effect.gen(function* () {
1641 // A slow path delays the fresh request as much as the probe.
1642 const harness = yield* makeSilentSocketHarness({
1643 probe: (attempt) => (attempt === 1 ? Effect.sleep("4 seconds") : Effect.void),
1644 checkHost: () => "silent",
1645 });
1646 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1647 initiallyDesired: true,
1648 }).pipe(Effect.provide(harness.dependencies));
1649
1650 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1651 yield* harness.wake(wakeup);
1652 yield* TestClock.adjust("20 seconds");
1653
1654 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1655 expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);
1656 }).pipe(Effect.provide(TestClock.layer())),
1657 );
1658
1659 it.effect("replaces an unanswered long resume probe even while the network reports offline", () =>
1660 Effect.gen(function* () {
1661 const harness = yield* makeHarness({
1662 probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
1663 });
1664 const supervisor = yield* EnvironmentSupervisor.make(LAN_ENTRY, {
1665 initiallyDesired: true,
1666 }).pipe(Effect.provide(harness.dependencies));
1667
1668 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1669 yield* harness.wake("application-active-long");
1670 yield* harness.setNetworkStatus("offline");
1671 yield* TestClock.adjust("1499 millis");
1672 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1673
1674 yield* TestClock.adjust("1 millis");
1675 const reconnected = yield* awaitState(
1676 supervisor.state,
1677 (state) => state.phase === "connected" && state.generation === 2,
1678 );
1679 expect(reconnected.lastFailure).toBeNull();
1680 }).pipe(Effect.provide(TestClock.layer())),
1681 );
1682
1683 it.effect("reconnects immediately when the session closes during a resume probe", () =>
1684 Effect.gen(function* () {
1685 const probeStarted = yield* Deferred.make<void>();
1686 const harness = yield* makeHarness({
1687 probe: (attempt) =>
1688 attempt === 1
1689 ? Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never))
1690 : Effect.void,
1691 });
1692 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1693 initiallyDesired: true,
1694 }).pipe(Effect.provide(harness.dependencies));
1695
1696 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1697 yield* harness.wake("application-active-probe");
1698 yield* Deferred.await(probeStarted);
1699 yield* harness.closeLatestSession();
1700
1701 const reconnected = yield* awaitState(
1702 supervisor.state,
1703 (state) => state.phase === "connected" && state.generation === 2,
1704 );
1705 expect(reconnected.attempt).toBe(1);
1706 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1707 }).pipe(Effect.provide(TestClock.layer())),
1708 );
1709
1710 it.effect("reconnects immediately when the foreground liveness probe fails", () =>
1711 Effect.gen(function* () {
1712 const allowReconnect = yield* Deferred.make<void>();
1713 const harness = yield* makeHarness({
1714 prepare: (attempt) =>
1715 attempt === 2
1716 ? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION))
1717 : Effect.succeed(PREPARED_CONNECTION),
1718 probe: (attempt) =>
1719 attempt === 1 ? Effect.fail(transient("The live session is stale.")) : Effect.void,
1720 });
1721 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1722 initiallyDesired: true,
1723 }).pipe(Effect.provide(harness.dependencies));
1724
1725 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1726 yield* harness.wake("application-active");
1727 const reconnecting = yield* awaitState(
1728 supervisor.state,
1729 (state) => state.phase === "connecting",
1730 );
1731 expect(reconnecting.attempt).toBe(1);
1732 expect(reconnecting.lastFailure).toBeNull();
1733 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
1734
1735 yield* Deferred.succeed(allowReconnect, undefined);
1736 yield* awaitState(
1737 supervisor.state,
1738 (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
1739 );
1740
1741 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1742 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1743 }).pipe(Effect.provide(TestClock.layer())),
1744 );
1745
1746 it.effect("honors an explicit disconnect while a foreground probe is stalled", () =>
1747 Effect.gen(function* () {
1748 const probeStarted = yield* Deferred.make<void>();
1749 const harness = yield* makeHarness({
1750 probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)),
1751 });
1752 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1753 initiallyDesired: true,
1754 }).pipe(Effect.provide(harness.dependencies));
1755
1756 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1757 yield* harness.wake("application-active");
1758 yield* Deferred.await(probeStarted);
1759 yield* supervisor.disconnect;
1760 yield* awaitState(supervisor.state, (state) => state.phase === "available");
1761
1762 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1763 }),
1764 );
1765
1766 it.effect(
1767 "keeps a live session when the automatic follow-up to a slow foreground probe answers",
1768 () =>
1769 Effect.gen(function* () {
1770 const probeCount = yield* Ref.make(0);
1771 const probeStarted = yield* Queue.unbounded<number>();
1772 const harness = yield* makeHarness({
1773 probe: () =>
1774 Ref.updateAndGet(probeCount, (count) => count + 1).pipe(
1775 Effect.tap((call) => Queue.offer(probeStarted, call)),
1776 Effect.flatMap((call) => (call === 1 ? Effect.never : Effect.void)),
1777 ),
1778 });
1779 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1780 initiallyDesired: true,
1781 }).pipe(Effect.provide(harness.dependencies));
1782
1783 yield* awaitState(
1784 supervisor.state,
1785 (state) => state.phase === "connected" && state.generation === 1,
1786 );
1787 yield* harness.wake("application-active");
1788 expect(yield* Queue.take(probeStarted)).toBe(1);
1789 // A busy server misses the foreground probe deadline once while the
1790 // socket stays open; the supervisor asks again on its own.
1791 yield* TestClock.adjust("15 seconds");
1792 expect(yield* Queue.take(probeStarted)).toBe(2);
1793 yield* TestClock.adjust("30 seconds");
1794
1795 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1796 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1797 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
1798 phase: "connected",
1799 generation: 1,
1800 });
1801 }).pipe(Effect.provide(TestClock.layer())),
1802 );
1803
1804 it.effect("replaces a session when the automatic follow-up probe also goes unanswered", () =>
1805 Effect.gen(function* () {
1806 const probeCount = yield* Ref.make(0);
1807 const harness = yield* makeHarness({
1808 probe: () =>
1809 Ref.update(probeCount, (count) => count + 1).pipe(Effect.andThen(Effect.never)),
1810 });
1811 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1812 initiallyDesired: true,
1813 }).pipe(Effect.provide(harness.dependencies));
1814
1815 yield* awaitState(
1816 supervisor.state,
1817 (state) => state.phase === "connected" && state.generation === 1,
1818 );
1819 // One foreground event and no other signal: the follow-up's miss alone
1820 // replaces the session.
1821 yield* harness.wake("application-active");
1822 yield* TestClock.adjust("15 seconds");
1823 yield* TestClock.adjust("15 seconds");
1824 yield* awaitState(
1825 supervisor.state,
1826 (state) => state.phase === "connected" && state.generation === 2,
1827 );
1828
1829 expect(yield* Ref.get(probeCount)).toBe(2);
1830 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1831 }).pipe(Effect.provide(TestClock.layer())),
1832 );
1833
1834 it.effect("replaces a session when a retry lands late in a slow foreground probe", () =>
1835 Effect.gen(function* () {
1836 const probeStarted = yield* Deferred.make<void>();
1837 const harness = yield* makeHarness({
1838 probe: () => Deferred.succeed(probeStarted, undefined).pipe(Effect.andThen(Effect.never)),
1839 });
1840 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1841 initiallyDesired: true,
1842 }).pipe(Effect.provide(harness.dependencies));
1843
1844 yield* awaitState(
1845 supervisor.state,
1846 (state) => state.phase === "connected" && state.generation === 1,
1847 );
1848 yield* harness.wake("application-active");
1849 yield* Deferred.await(probeStarted);
1850 yield* TestClock.adjust("13 seconds");
1851 // The retry's own deadline falls after the foreground one, but it still
1852 // asks for a definite answer, so the first miss is not forgiven.
1853 yield* supervisor.retryNow;
1854 yield* TestClock.adjust("5 seconds");
1855 yield* awaitState(
1856 supervisor.state,
1857 (state) => state.phase === "connected" && state.generation === 2,
1858 );
1859 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1860 }).pipe(Effect.provide(TestClock.layer())),
1861 );
1862
1863 it.effect("does not churn a healthy session when credentials change", () =>
1864 Effect.gen(function* () {
1865 const harness = yield* makeHarness();
1866 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
1867 initiallyDesired: true,
1868 }).pipe(Effect.provide(harness.dependencies));
1869
1870 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1871 yield* harness.wake("credentials-changed");
1872 yield* Effect.yieldNow;
1873
1874 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1875 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
1876 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
1877 }),
1878 );
1879
1880 it.effect("releases and reconnects a relay session when credentials change", () =>
1881 Effect.gen(function* () {
1882 const relaySpans: Array<Tracer.NativeSpan> = [];
1883 const relayTracer = Tracer.make({
1884 span(options) {
1885 const span = new Tracer.NativeSpan(options);
1886 relaySpans.push(span);
1887 return span;
1888 },
1889 });
1890 const harness = yield* makeHarness({
1891 prepare: () => Effect.succeed(preparedFor(RELAY_TARGET)),
1892 });
1893 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
1894 initiallyDesired: true,
1895 }).pipe(
1896 Effect.provide(harness.dependencies),
1897 Effect.provideService(RelayClientTracer, Option.some(relayTracer)),
1898 );
1899
1900 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
1901 yield* harness.wake("credentials-changed");
1902 yield* awaitState(
1903 supervisor.state,
1904 (state) => state.phase === "connected" && state.generation === 2,
1905 );
1906
1907 expect(yield* Ref.get(harness.sessionCount)).toBe(2);
1908 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
1909 const attempts = relaySpans.filter((span) => span.name === "relay.connection.attempt");
1910 expect(attempts).toHaveLength(2);
1911 for (const span of attempts) {
1912 expect(span.attributes.get("environment.id")).toBe(RELAY_TARGET.environmentId);
1913 expect(span.attributes.get("environment.target.kind")).toBe(RELAY_TARGET._tag);
1914 expect(span.attributes.has("environment.label")).toBe(false);
1915 }
1916 }),
1917 );
1918
1919 it.effect("refreshes HTTP authorization without replacing the active relay session", () =>
1920 Effect.gen(function* () {
1921 const endpoint = {
1922 httpBaseUrl: TARGET.httpBaseUrl,
1923 wsBaseUrl: TARGET.wsBaseUrl,
1924 providerKind: "cloudflare_tunnel" as const,
1925 };
1926 const token = yield* Ref.make(
1927 Option.some(
1928 new TokenStore.RemoteDpopAccessToken({
1929 environmentId: TARGET.environmentId,
1930 accountId: "test-account",
1931 label: TARGET.label,
1932 endpoint,
1933 accessToken: "access-token-1",
1934 expiresAtEpochMs: 3_600_000,
1935 dpopThumbprint: "test-thumbprint",
1936 }),
1937 ),
1938 );
1939 const bootstrapFails = yield* Ref.make(false);
1940 const bootstrapCalls = yield* Ref.make(0);
1941 const httpPaths: Array<string> = [];
1942 const sessionAuthorizations: Array<string | null> = [];
1943 const fetchFn = ((input, init) => {
1944 const request = new Request(input, init);
1945 const pathname = new URL(request.url).pathname;
1946 httpPaths.push(pathname);
1947 switch (pathname) {
1948 case "/.well-known/t3/environment":
1949 return Promise.resolve(
1950 Response.json({
1951 environmentId: TARGET.environmentId,
1952 label: TARGET.label,
1953 platform: { os: "linux", arch: "x64" },
1954 serverVersion: "0.0.0-test",
1955 capabilities: { repositoryIdentity: true },
1956 }),
1957 );
1958 case "/oauth/token":
1959 return Promise.resolve(
1960 Response.json({
1961 access_token: "access-token-2",
1962 issued_token_type: "urn:ietf:params:oauth:token-type:access_token",
1963 token_type: "DPoP",
1964 expires_in: 3_600,
1965 scope: AuthStandardClientScopes.join(" "),
1966 }),
1967 );
1968 case "/api/auth/websocket-ticket/rpc":
1969 return Promise.resolve(
1970 Response.json({
1971 ticket: "ws-ticket",
1972 expiresAt: "2026-09-04T01:00:00.000Z",
1973 }),
1974 );
1975 case "/api/auth/session": {
1976 const authorization = request.headers.get("authorization");
1977 sessionAuthorizations.push(authorization);
1978 return Promise.resolve(
1979 Response.json({
1980 authenticated: authorization === "DPoP access-token-2",
1981 auth: {
1982 policy: "loopback-browser",
1983 bootstrapMethods: ["one-time-token"],
1984 sessionMethods: ["dpop-access-token"],
1985 sessionCookieName: "t3_session_test",
1986 },
1987 scopes: AuthStandardClientScopes,
1988 }),
1989 );
1990 }
1991 default:
1992 return Promise.reject(new Error(`Unexpected HTTP request to ${request.url}`));
1993 }
1994 }) satisfies typeof fetch;
1995 const signer = ManagedRelay.ManagedRelayDpopSigner.of({
1996 thumbprint: Effect.succeed("test-thumbprint"),
1997 createProof: () => Effect.succeed("test-proof"),
1998 });
1999 const unused = () => Effect.die("Unexpected relay operation.");
2000 const relay = ManagedRelay.ManagedRelayClient.of({
2001 relayUrl: "https://relay.example.test",
2002 listEnvironments: unused,
2003 listDevices: unused,
2004 createEnvironmentLinkChallenge: unused,
2005 linkEnvironment: unused,
2006 unlinkEnvironment: unused,
2007 getT3ChatLink: unused,
2008 createT3ChatLinkRequest: unused,
2009 unlinkT3Chat: unused,
2010 getEnvironmentStatus: unused,
2011 connectEnvironment: Effect.fn("TestConnectionHttp.connectEnvironment")(function* () {
2012 yield* Ref.update(bootstrapCalls, (count) => count + 1);
2013 if (yield* Ref.get(bootstrapFails)) {
2014 return yield* new ManagedRelay.ManagedRelayRequestTimeoutError({
2015 activity: "Relay environment connection",
2016 timeoutMs: 6_000,
2017 traceId: null,
2018 });
2019 }
2020 return {
2021 environmentId: TARGET.environmentId,
2022 endpoint,
2023 credential: "relay-bootstrap",
2024 expiresAt: "2026-09-04T01:00:00.000Z",
2025 };
2026 }),
2027 registerDevice: unused,
2028 unregisterDevice: unused,
2029 registerLiveActivity: unused,
2030 getAgentActivitySnapshot: unused,
2031 resetTokenCache: Effect.void,
2032 });
2033 const layerHttp = RpcHttp.layerRemoteHttpClient(fetchFn);
2034 const remoteAuthorization = yield* RemoteEnvironmentAuthorization.make.pipe(
2035 Effect.provide(
2036 Layer.mergeAll(
2037 layerHttp,
2038 Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, signer),
2039 Layer.succeed(ManagedRelay.ManagedRelayClient, relay),
2040 Layer.succeed(ClientCapabilities.CloudSession, {
2041 identity: Effect.succeedSome({ accountId: "test-account" }),
2042 clerkToken: Effect.succeed("clerk-token"),
2043 }),
2044 Layer.succeed(ClientCapabilities.RelayDeviceIdentity, {
2045 deviceId: Effect.succeedNone,
2046 }),
2047 Layer.succeed(TokenStore.RemoteDpopAccessTokenStore, {
2048 get: () => Ref.get(token),
2049 put: (value) => Ref.set(token, Option.some(value)),
2050 remove: () => Ref.set(token, Option.none()),
2051 }),
2052 Layer.succeed(ClientCapabilities.ClientPresentation, {
2053 metadata: { label: "Test client", deviceType: "desktop" },
2054 }),
2055 ConnectionCredentialStore.layerMemory(),
2056 ),
2057 ),
2058 );
2059 const harness = yield* makeHarness({
2060 prepare: () =>
2061 remoteAuthorization
2062 .authorizeDpop({ expectedEnvironmentId: TARGET.environmentId })
2063 .pipe(Effect.map((prepared) => ({ ...prepared, target: RELAY_TARGET }))),
2064 });
2065 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
2066 initiallyDesired: true,
2067 }).pipe(Effect.provide(harness.dependencies));
2068 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2069 const session = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session));
2070 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
2071 const readSession = fetchEnvironmentSessionState({
2072 prepared,
2073 signer: Option.some(signer),
2074 remoteAuthorization: Option.some(remoteAuthorization),
2075 }).pipe(Effect.provide(layerHttp));
2076
2077 yield* TestClock.adjust("2 hours");
2078 expect((yield* readSession).authenticated).toBe(true);
2079 expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]);
2080 expect(yield* Ref.get(bootstrapCalls)).toBe(1);
2081 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session);
2082 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
2083 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
2084 phase: "connected",
2085 generation: 1,
2086 });
2087
2088 yield* TestClock.adjust("2 hours");
2089 yield* Ref.set(bootstrapFails, true);
2090 const failure = yield* readSession.pipe(Effect.flip);
2091 expect(failure._tag).toBe("RemoteEnvironmentAuthFetchError");
2092 expect(yield* Ref.get(bootstrapCalls)).toBe(2);
2093 expect(sessionAuthorizations).toEqual(["DPoP access-token-2"]);
2094 expect(httpPaths.filter((path) => path === "/api/auth/websocket-ticket/rpc")).toHaveLength(1);
2095 expect(httpPaths.filter((path) => path === "/oauth/token")).toHaveLength(1);
2096 expect(yield* Ref.get(harness.sessionCount)).toBe(1);
2097 expect(yield* Ref.get(harness.releaseCount)).toBe(0);
2098 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.session))).toBe(session);
2099 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
2100 phase: "connected",
2101 generation: 1,
2102 });
2103 }).pipe(Effect.provide(TestClock.layer())),
2104 );
2105
2106 it.effect("does not lose an explicit disconnect among concurrent wakeup signals", () =>
2107 Effect.gen(function* () {
2108 const harness = yield* makeHarness();
2109 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
2110 initiallyDesired: true,
2111 }).pipe(Effect.provide(harness.dependencies));
2112
2113 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2114 yield* Effect.all(
2115 [
2116 supervisor.disconnect,
2117 harness.wake("credentials-changed"),
2118 harness.wake("application-active"),
2119 harness.wake("credentials-changed"),
2120 ],
2121 { concurrency: "unbounded" },
2122 );
2123 yield* awaitState(supervisor.state, (state) => state.phase === "available");
2124
2125 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
2126 expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
2127 }),
2128 );
2129 });
2130
2131 describe("EnvironmentSupervisor routes", () => {
2132 it.effect("connects over relay before a slow LAN reachability check times out", () =>
2133 Effect.gen(function* () {
2134 const checkingLan = yield* Deferred.make<void>();
2135 const harness = yield* makeHarness({
2136 checkRoute: (route) =>
2137 route.target._tag === "BearerConnectionTarget"
2138 ? Deferred.succeed(checkingLan, undefined).pipe(
2139 Effect.andThen(Effect.sleep("2500 millis")),
2140 Effect.as("silent" as const),
2141 )
2142 : Effect.succeed("unchecked"),
2143 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2144 });
2145 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2146 initiallyDesired: true,
2147 }).pipe(Effect.provide(harness.dependencies));
2148
2149 yield* Deferred.await(checkingLan);
2150 yield* TestClock.adjust("500 millis");
2151 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2152 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2153 "RelayConnectionTarget",
2154 );
2155 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
2156 }).pipe(Effect.provide(TestClock.layer())),
2157 );
2158
2159 it.effect("shares one head start across several slow LAN routes", () =>
2160 Effect.gen(function* () {
2161 const secondLanTarget = new BearerConnectionTarget({
2162 environmentId: TARGET.environmentId,
2163 label: TARGET.label,
2164 connectionId: "bearer:second-lan",
2165 });
2166 const entry: ConnectionCatalogEntry = {
2167 ...LAN_THEN_RELAY_ENTRY,
2168 alternateRoutes: [
2169 {
2170 target: secondLanTarget,
2171 profile: Option.some(
2172 new BearerConnectionProfile({
2173 connectionId: secondLanTarget.connectionId,
2174 environmentId: TARGET.environmentId,
2175 label: TARGET.label,
2176 httpBaseUrl: "http://192.168.56.1:3773/",
2177 wsBaseUrl: "ws://192.168.56.1:3773/",
2178 }),
2179 ),
2180 },
2181 { target: RELAY_TARGET, profile: Option.none() },
2182 ],
2183 };
2184 const checking = yield* Ref.make(0);
2185 const checkingLans = yield* Deferred.make<void>();
2186 const harness = yield* makeHarness({
2187 checkRoute: (route) =>
2188 route.target._tag === "BearerConnectionTarget"
2189 ? Ref.updateAndGet(checking, (count) => count + 1).pipe(
2190 Effect.flatMap((count) =>
2191 count === 2 ? Deferred.succeed(checkingLans, undefined) : Effect.void,
2192 ),
2193 Effect.andThen(Effect.sleep("2500 millis")),
2194 Effect.as("silent" as const),
2195 )
2196 : Effect.succeed("unchecked"),
2197 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2198 });
2199 const supervisor = yield* EnvironmentSupervisor.make(entry, {
2200 initiallyDesired: true,
2201 }).pipe(Effect.provide(harness.dependencies));
2202
2203 yield* Deferred.await(checkingLans);
2204 yield* TestClock.adjust("500 millis");
2205 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2206 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2207 "RelayConnectionTarget",
2208 );
2209 }).pipe(Effect.provide(TestClock.layer())),
2210 );
2211
2212 it.effect("still prefers a LAN route that responds within its head start", () =>
2213 Effect.gen(function* () {
2214 const checkingLan = yield* Deferred.make<void>();
2215 const harness = yield* makeHarness({
2216 checkRoute: (route) =>
2217 route.target._tag === "BearerConnectionTarget"
2218 ? Deferred.succeed(checkingLan, undefined).pipe(
2219 Effect.andThen(Effect.sleep("100 millis")),
2220 Effect.as("answered" as const),
2221 )
2222 : Effect.succeed("unchecked"),
2223 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2224 });
2225 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2226 initiallyDesired: true,
2227 }).pipe(Effect.provide(harness.dependencies));
2228
2229 yield* Deferred.await(checkingLan);
2230 yield* TestClock.adjust("100 millis");
2231 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2232 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2233 "BearerConnectionTarget",
2234 );
2235 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
2236 }).pipe(Effect.provide(TestClock.layer())),
2237 );
2238
2239 it.effect("tries a slow tailnet route that answered before a dead LAN route", () =>
2240 Effect.gen(function* () {
2241 const tailnetTarget = new BearerConnectionTarget({
2242 environmentId: TARGET.environmentId,
2243 label: TARGET.label,
2244 connectionId: "bearer:tailnet",
2245 });
2246 // No relay: a Tailscale-only client away from the LAN.
2247 const entry: ConnectionCatalogEntry = {
2248 ...LAN_THEN_RELAY_ENTRY,
2249 alternateRoutes: [
2250 {
2251 target: tailnetTarget,
2252 profile: Option.some(
2253 new BearerConnectionProfile({
2254 connectionId: tailnetTarget.connectionId,
2255 environmentId: TARGET.environmentId,
2256 label: TARGET.label,
2257 httpBaseUrl: "http://100.100.1.2:3773/",
2258 wsBaseUrl: "ws://100.100.1.2:3773/",
2259 network: "tailscale",
2260 }),
2261 ),
2262 },
2263 ],
2264 };
2265 const checking = yield* Ref.make(0);
2266 const checkingBoth = yield* Deferred.make<void>();
2267 const harness = yield* makeHarness({
2268 checkRoute: (route) =>
2269 Ref.updateAndGet(checking, (count) => count + 1).pipe(
2270 Effect.flatMap((count) =>
2271 count === 2 ? Deferred.succeed(checkingBoth, undefined) : Effect.void,
2272 ),
2273 Effect.andThen(
2274 connectionRouteId(route.target) === LAN_TARGET.connectionId
2275 ? Effect.sleep("2500 millis").pipe(Effect.as("silent" as const))
2276 : Effect.sleep("700 millis").pipe(Effect.as("answered" as const)),
2277 ),
2278 ),
2279 // The dead LAN address never answers the connection itself either.
2280 prepare: (_attempt, target) =>
2281 target._tag === "BearerConnectionTarget" &&
2282 target.connectionId === LAN_TARGET.connectionId
2283 ? Effect.never
2284 : Effect.succeed(preparedFor(target)),
2285 });
2286 const supervisor = yield* EnvironmentSupervisor.make(entry, {
2287 initiallyDesired: true,
2288 }).pipe(Effect.provide(harness.dependencies));
2289
2290 yield* Deferred.await(checkingBoth);
2291 yield* TestClock.adjust("2500 millis");
2292 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2293 expect(
2294 connectionRouteId(
2295 Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target,
2296 ),
2297 ).toBe("bearer:tailnet");
2298 }).pipe(Effect.provide(TestClock.layer())),
2299 );
2300
2301 it.effect("still tries a slow LAN route when relay fails", () =>
2302 Effect.gen(function* () {
2303 const checkingLan = yield* Deferred.make<void>();
2304 const attempted: string[] = [];
2305 const harness = yield* makeHarness({
2306 checkRoute: (route) =>
2307 route.target._tag === "BearerConnectionTarget"
2308 ? Deferred.succeed(checkingLan, undefined).pipe(
2309 Effect.andThen(Effect.sleep("2500 millis")),
2310 Effect.as("answered" as const),
2311 )
2312 : Effect.succeed("unchecked"),
2313 prepare: (_attempt, target) => {
2314 attempted.push(target._tag);
2315 return target._tag === "RelayConnectionTarget"
2316 ? Effect.fail(blocked("Sign in to T3 Connect."))
2317 : Effect.succeed(preparedFor(target));
2318 },
2319 });
2320 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2321 initiallyDesired: true,
2322 }).pipe(Effect.provide(harness.dependencies));
2323
2324 yield* Deferred.await(checkingLan);
2325 yield* TestClock.adjust("2500 millis");
2326 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2327 expect(attempted).toEqual(["RelayConnectionTarget", "BearerConnectionTarget"]);
2328 }).pipe(Effect.provide(TestClock.layer())),
2329 );
2330
2331 it.effect("retries instead of waiting on a LAN route that stayed silent after relay failed", () =>
2332 Effect.gen(function* () {
2333 const checkingLan = yield* Deferred.make<void>();
2334 const attempted: string[] = [];
2335 const harness = yield* makeHarness({
2336 checkRoute: (route) =>
2337 route.target._tag === "BearerConnectionTarget"
2338 ? Deferred.succeed(checkingLan, undefined).pipe(
2339 Effect.andThen(Effect.sleep("2500 millis")),
2340 Effect.as("silent" as const),
2341 )
2342 : Effect.succeed("unchecked"),
2343 prepare: (_attempt, target) => {
2344 attempted.push(target._tag);
2345 return target._tag === "RelayConnectionTarget"
2346 ? Effect.fail(transient("Relay socket closed."))
2347 : Effect.never;
2348 },
2349 });
2350 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2351 initiallyDesired: true,
2352 }).pipe(Effect.provide(harness.dependencies));
2353
2354 yield* Deferred.await(checkingLan);
2355 yield* TestClock.adjust("2500 millis");
2356 yield* awaitState(supervisor.state, (state) => state.phase === "backoff");
2357 expect(attempted).toEqual(["RelayConnectionTarget"]);
2358 }).pipe(Effect.provide(TestClock.layer())),
2359 );
2360
2361 it.effect("skips a silent LAN route and connects over T3 Connect", () =>
2362 Effect.gen(function* () {
2363 const harness = yield* makeHarness({
2364 checkRoute: (route) =>
2365 Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"),
2366 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2367 });
2368 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2369 initiallyDesired: true,
2370 }).pipe(Effect.provide(harness.dependencies));
2371
2372 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2373 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
2374 expect(prepared.target._tag).toBe("RelayConnectionTarget");
2375 expect(yield* Ref.get(harness.prepareCount)).toBe(1);
2376 }),
2377 );
2378
2379 it.effect("moves to a blocked route's fallback instead of stopping", () =>
2380 Effect.gen(function* () {
2381 const harness = yield* makeHarness({
2382 checkRoute: () => Effect.succeed("answered"),
2383 prepare: (_attempt, target) =>
2384 target._tag === "BearerConnectionTarget"
2385 ? Effect.fail(blocked("The environment credential is invalid."))
2386 : Effect.succeed(preparedFor(target)),
2387 });
2388 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2389 initiallyDesired: true,
2390 }).pipe(Effect.provide(harness.dependencies));
2391
2392 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2393 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
2394 expect(prepared.target._tag).toBe("RelayConnectionTarget");
2395 }),
2396 );
2397
2398 it.effect("does not report a route that failed before reaching its host's sign-in", () =>
2399 Effect.gen(function* () {
2400 const failures = [
2401 new ConnectionBlockedError({
2402 reason: "permission",
2403 detail: "T3 Connect is not enabled on this machine.",
2404 hop: "tunnel",
2405 failureCause: "tunnel_not_enabled",
2406 }),
2407 new ConnectionBlockedError({
2408 reason: "authentication",
2409 detail: "This device's clock is wrong.",
2410 hop: "server",
2411 failureCause: "clock_skew",
2412 }),
2413 ];
2414 for (const failure of failures) {
2415 const harness = yield* makeHarness({
2416 checkRoute: () => Effect.succeed("answered"),
2417 prepare: (_attempt, target) =>
2418 target._tag === "BearerConnectionTarget"
2419 ? Effect.fail(failure)
2420 : Effect.succeed(preparedFor(target)),
2421 });
2422 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2423 initiallyDesired: true,
2424 }).pipe(Effect.provide(harness.dependencies));
2425
2426 const state = yield* awaitState(supervisor.state, (value) => value.phase === "connected");
2427 expect(state.refusedRouteIds).toEqual([]);
2428 }
2429 }),
2430 );
2431
2432 it.effect("reports a preferred route whose host refuses sign-in once it answers again", () =>
2433 Effect.gen(function* () {
2434 const lanReachable = yield* Ref.make(false);
2435 const lanRefuses = yield* Ref.make(true);
2436 const checkRoute = (route: ConnectionRoute) =>
2437 route.target._tag === "BearerConnectionTarget"
2438 ? Ref.get(lanReachable).pipe(
2439 Effect.map((reachable): ConnectionDriver.RouteCheck =>
2440 reachable ? "answered" : "silent",
2441 ),
2442 )
2443 : Effect.succeed<ConnectionDriver.RouteCheck>("unchecked");
2444 const harness = yield* makeHarness({
2445 checkRoute,
2446 preflight: (route) =>
2447 ConnectionDriver.preflightRoute(
2448 checkRoute(route),
2449 Ref.get(lanRefuses).pipe(
2450 Effect.flatMap((refuses) =>
2451 refuses && route.target._tag === "BearerConnectionTarget"
2452 ? Effect.fail(hostRefusal())
2453 : Effect.succeed(preparedFor(route.target)),
2454 ),
2455 ),
2456 ),
2457 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2458 });
2459 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2460 initiallyDesired: true,
2461 }).pipe(Effect.provide(harness.dependencies));
2462
2463 const connected = yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2464 expect(connected.refusedRouteIds).toEqual([]);
2465
2466 yield* Ref.set(lanReachable, true);
2467 yield* harness.wake("network-changed");
2468 const state = yield* awaitState(
2469 supervisor.state,
2470 (value) => value.refusedRouteIds.length > 0,
2471 );
2472 expect(state).toMatchObject({
2473 phase: "connected",
2474 generation: connected.generation,
2475 refusedRouteIds: [LAN_TARGET.connectionId],
2476 });
2477 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2478 "RelayConnectionTarget",
2479 );
2480
2481 yield* Ref.set(lanRefuses, false);
2482 yield* harness.wake("network-changed");
2483 const overLan = yield* awaitState(
2484 supervisor.state,
2485 (value) => value.phase === "connected" && value.generation > connected.generation,
2486 );
2487 expect(
2488 connectionRouteId(
2489 Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target,
2490 ),
2491 ).toBe(LAN_TARGET.connectionId);
2492 expect(overLan.refusedRouteIds).toEqual([]);
2493 }),
2494 );
2495
2496 it.effect("keeps reporting a refused route after moving to a better route", () =>
2497 Effect.gen(function* () {
2498 const loopbackTarget = new BearerConnectionTarget({
2499 environmentId: TARGET.environmentId,
2500 label: TARGET.label,
2501 connectionId: "bearer:loopback",
2502 });
2503 const entry: ConnectionCatalogEntry = {
2504 target: loopbackTarget,
2505 profile: Option.some(
2506 new BearerConnectionProfile({
2507 connectionId: loopbackTarget.connectionId,
2508 environmentId: TARGET.environmentId,
2509 label: TARGET.label,
2510 httpBaseUrl: "http://127.0.0.1:3773/",
2511 wsBaseUrl: "ws://127.0.0.1:3773/",
2512 }),
2513 ),
2514 alternateRoutes: [LAN_ROUTE, { target: RELAY_TARGET, profile: Option.none() }],
2515 enabled: true,
2516 };
2517 const lanReachable = yield* Ref.make(false);
2518 const checkRoute = (route: ConnectionRoute) =>
2519 connectionRouteId(route.target) === LAN_TARGET.connectionId
2520 ? Ref.get(lanReachable).pipe(
2521 Effect.map((reachable): ConnectionDriver.RouteCheck =>
2522 reachable ? "answered" : "silent",
2523 ),
2524 )
2525 : Effect.succeed<ConnectionDriver.RouteCheck>(
2526 route.target._tag === "BearerConnectionTarget" ? "answered" : "unchecked",
2527 );
2528 const prepare = (target: ConnectionTarget) =>
2529 connectionRouteId(target) === loopbackTarget.connectionId
2530 ? Effect.fail(hostRefusal())
2531 : Effect.succeed(preparedFor(target));
2532 const harness = yield* makeHarness({
2533 checkRoute,
2534 preflight: (route) =>
2535 ConnectionDriver.preflightRoute(checkRoute(route), prepare(route.target)),
2536 prepare: (_attempt, target) => prepare(target),
2537 });
2538 const supervisor = yield* EnvironmentSupervisor.make(entry, {
2539 initiallyDesired: true,
2540 }).pipe(Effect.provide(harness.dependencies));
2541
2542 const overRelay = yield* awaitState(
2543 supervisor.state,
2544 (state) => state.phase === "connected" && state.generation === 1,
2545 );
2546 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2547 "RelayConnectionTarget",
2548 );
2549 expect(overRelay.refusedRouteIds).toEqual([loopbackTarget.connectionId]);
2550
2551 yield* Ref.set(lanReachable, true);
2552 yield* harness.wake("network-changed");
2553 const overLan = yield* awaitState(
2554 supervisor.state,
2555 (state) => state.phase === "connected" && state.generation === 2,
2556 );
2557 expect(
2558 connectionRouteId(
2559 Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target,
2560 ),
2561 ).toBe(LAN_TARGET.connectionId);
2562 expect(overLan.refusedRouteIds).toEqual([loopbackTarget.connectionId]);
2563 }),
2564 );
2565
2566 it.effect("tries every route when none answers the reachability check", () =>
2567 Effect.gen(function* () {
2568 const harness = yield* makeHarness({
2569 checkRoute: () => Effect.succeed("silent"),
2570 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2571 });
2572 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2573 initiallyDesired: true,
2574 }).pipe(Effect.provide(harness.dependencies));
2575
2576 yield* awaitState(supervisor.state, (state) => state.phase === "connected");
2577 const prepared = Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared));
2578 expect(prepared.target._tag).toBe("BearerConnectionTarget");
2579 }),
2580 );
2581
2582 it.effect("keeps retrying when one route is blocked and another failed transiently", () =>
2583 Effect.gen(function* () {
2584 const harness = yield* makeHarness({
2585 checkRoute: () => Effect.succeed("answered"),
2586 prepare: (_attempt, target) =>
2587 target._tag === "RelayConnectionTarget"
2588 ? Effect.fail(blocked("Sign in to T3 Connect."))
2589 : Effect.fail(transient("LAN socket refused.")),
2590 });
2591 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2592 initiallyDesired: true,
2593 }).pipe(Effect.provide(harness.dependencies));
2594
2595 const state = yield* awaitState(supervisor.state, (value) => value.phase === "backoff");
2596 expect(state.lastFailure?._tag).toBe("ConnectionTransientError");
2597 }),
2598 );
2599
2600 it.effect("moves back to the LAN route when the network changes and it answers again", () =>
2601 Effect.gen(function* () {
2602 const lanReachable = yield* Ref.make(false);
2603 const harness = yield* makeHarness({
2604 checkRoute: (route) =>
2605 route.target._tag === "BearerConnectionTarget"
2606 ? Ref.get(lanReachable).pipe(
2607 Effect.map((reachable) => (reachable ? "answered" : "silent")),
2608 )
2609 : Effect.succeed("unchecked"),
2610 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2611 });
2612 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2613 initiallyDesired: true,
2614 }).pipe(Effect.provide(harness.dependencies));
2615
2616 yield* awaitState(
2617 supervisor.state,
2618 (state) => state.phase === "connected" && state.generation === 1,
2619 );
2620 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2621 "RelayConnectionTarget",
2622 );
2623
2624 yield* Ref.set(lanReachable, true);
2625 yield* harness.setNetworkStatus("unknown");
2626 yield* harness.setNetworkStatus("online");
2627
2628 yield* awaitState(
2629 supervisor.state,
2630 (state) => state.phase === "connected" && state.generation === 2,
2631 );
2632 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2633 "BearerConnectionTarget",
2634 );
2635 expect(yield* Ref.get(harness.releaseCount)).toBe(1);
2636 }),
2637 );
2638
2639 it.effect("checks a dead LAN route less often until the network changes", () =>
2640 Effect.gen(function* () {
2641 const lanReachable = yield* Ref.make(false);
2642 const lanChecks = yield* Ref.make(0);
2643 const harness = yield* makeHarness({
2644 checkRoute: (route) =>
2645 route.target._tag === "BearerConnectionTarget"
2646 ? Ref.update(lanChecks, (count) => count + 1).pipe(
2647 Effect.andThen(Ref.get(lanReachable)),
2648 Effect.map((reachable) => (reachable ? "answered" : "silent")),
2649 )
2650 : Effect.succeed("unchecked"),
2651 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2652 });
2653 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2654 initiallyDesired: true,
2655 }).pipe(Effect.provide(harness.dependencies));
2656
2657 yield* awaitState(
2658 supervisor.state,
2659 (state) => state.phase === "connected" && state.generation === 1,
2660 );
2661 yield* Ref.set(lanChecks, 0);
2662
2663 // Checked at 1, 3 and 7 minutes instead of every minute.
2664 for (let minute = 0; minute < 10; minute += 1) {
2665 yield* TestClock.adjust("60 seconds");
2666 }
2667 expect(yield* Ref.get(lanChecks)).toBe(3);
2668
2669 yield* Ref.set(lanReachable, true);
2670 yield* harness.wake("network-changed");
2671 yield* awaitState(
2672 supervisor.state,
2673 (state) => state.phase === "connected" && state.generation === 2,
2674 );
2675 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2676 "BearerConnectionTarget",
2677 );
2678 }),
2679 );
2680
2681 it.effect("keeps checking a LAN route every minute when it answers but cannot be prepared", () =>
2682 Effect.gen(function* () {
2683 const lanPreflights = yield* Ref.make(0);
2684 const harness = yield* makeHarness({
2685 checkRoute: (route) =>
2686 Effect.succeed(route.target._tag === "BearerConnectionTarget" ? "silent" : "unchecked"),
2687 preflight: (route) =>
2688 route.target._tag === "BearerConnectionTarget"
2689 ? Ref.update(lanPreflights, (count) => count + 1).pipe(Effect.as("unprepared" as const))
2690 : Effect.succeed("unchecked"),
2691 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2692 });
2693 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2694 initiallyDesired: true,
2695 }).pipe(Effect.provide(harness.dependencies));
2696
2697 yield* awaitState(
2698 supervisor.state,
2699 (state) => state.phase === "connected" && state.generation === 1,
2700 );
2701 for (let minute = 0; minute < 10; minute += 1) {
2702 yield* TestClock.adjust("60 seconds");
2703 }
2704 expect(yield* Ref.get(lanPreflights)).toBe(10);
2705 }),
2706 );
2707
2708 it.effect("ignores a dead LAN check that was running when the network changed", () =>
2709 Effect.gen(function* () {
2710 const lanChecks = yield* Ref.make(0);
2711 const checkStarted = yield* Deferred.make<void>();
2712 const finishCheck = yield* Deferred.make<void>();
2713 const woke = yield* Ref.make(false);
2714 const wakeupHandled = yield* Deferred.make<void>();
2715 const harness = yield* makeHarness({
2716 checkRoute: (route) =>
2717 route.target._tag === "BearerConnectionTarget"
2718 ? Ref.updateAndGet(lanChecks, (count) => count + 1).pipe(
2719 Effect.flatMap((check) =>
2720 check === 1
2721 ? Effect.succeed("silent" as const)
2722 : check === 2
2723 ? Deferred.succeed(checkStarted, undefined).pipe(
2724 Effect.andThen(Deferred.await(finishCheck)),
2725 Effect.as("silent" as const),
2726 )
2727 : Effect.succeed("answered" as const),
2728 ),
2729 )
2730 : Effect.succeed("unchecked"),
2731 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2732 // The wakeup probes the session only after it forgets the silent routes.
2733 probe: () =>
2734 Ref.get(woke).pipe(
2735 Effect.flatMap((afterWake) =>
2736 afterWake ? Deferred.succeed(wakeupHandled, undefined) : Effect.void,
2737 ),
2738 Effect.asVoid,
2739 ),
2740 });
2741 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2742 initiallyDesired: true,
2743 }).pipe(Effect.provide(harness.dependencies));
2744
2745 yield* awaitState(
2746 supervisor.state,
2747 (state) => state.phase === "connected" && state.generation === 1,
2748 );
2749 yield* TestClock.adjust("60 seconds");
2750 yield* Deferred.await(checkStarted);
2751 yield* Ref.set(woke, true);
2752 yield* harness.wake("network-changed");
2753 yield* Deferred.await(wakeupHandled);
2754 yield* Deferred.succeed(finishCheck, undefined);
2755
2756 yield* awaitState(
2757 supervisor.state,
2758 (state) => state.phase === "connected" && state.generation === 2,
2759 );
2760 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2761 "BearerConnectionTarget",
2762 );
2763 }),
2764 );
2765
2766 it.effect("moves off a dead LAN socket when the phone switches to cellular", () =>
2767 Effect.gen(function* () {
2768 const lanReachable = yield* Ref.make(true);
2769 const harness = yield* makeHarness({
2770 checkRoute: (route) =>
2771 route.target._tag === "BearerConnectionTarget"
2772 ? Ref.get(lanReachable).pipe(
2773 Effect.map((reachable) => (reachable ? "answered" : "silent")),
2774 )
2775 : Effect.succeed("unchecked"),
2776 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2777 probe: () =>
2778 Ref.get(lanReachable).pipe(
2779 Effect.flatMap((reachable) => (reachable ? Effect.void : Effect.never)),
2780 ),
2781 });
2782 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
2783 initiallyDesired: true,
2784 }).pipe(Effect.provide(harness.dependencies));
2785
2786 yield* awaitState(
2787 supervisor.state,
2788 (state) => state.phase === "connected" && state.generation === 1,
2789 );
2790 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2791 "BearerConnectionTarget",
2792 );
2793
2794 yield* Ref.set(lanReachable, false);
2795 yield* harness.wake("network-changed");
2796 yield* TestClock.adjust("3 seconds");
2797
2798 yield* awaitState(
2799 supervisor.state,
2800 (state) => state.phase === "connected" && state.generation === 2,
2801 );
2802 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2803 "RelayConnectionTarget",
2804 );
2805 }),
2806 );
2807
2808 it.effect.each(["initial", "late", "withdrawn", "withdrawn-upgrade", "omitted"] as const)(
2809 "learns the %s direct endpoint over T3 Connect",
2810 (discovery) =>
2811 Effect.gen(function* () {
2812 const relayEntry: ConnectionCatalogEntry =
2813 discovery.startsWith("withdrawn") || discovery === "omitted"
2814 ? {
2815 target: LAN_ROUTE.target,
2816 profile: Option.some(
2817 new BearerConnectionProfile({
2818 ...Option.getOrThrow(LAN_ROUTE.profile),
2819 ...(discovery === "withdrawn-upgrade"
2820 ? {
2821 httpBaseUrl: "https://old.example.test/",
2822 wsBaseUrl: "wss://old.example.test/",
2823 }
2824 : {}),
2825 learned: true,
2826 authorization: "t3-connect",
2827 }),
2828 ),
2829 alternateRoutes: [{ target: RELAY_TARGET, profile: Option.none() }],
2830 enabled: true,
2831 }
2832 : {
2833 target: RELAY_TARGET,
2834 profile: Option.none(),
2835 enabled: true,
2836 };
2837 const initialProcessed = yield* Deferred.make<void>();
2838 const discoveryProcessed = yield* Deferred.make<void>();
2839 const discoveryStarted = yield* Deferred.make<void>();
2840 const finishDiscovery = yield* Deferred.make<void>();
2841 const lateAddress = "https://fixture.tail1234.ts.net/";
2842 const learned = yield* Ref.make<ReadonlyArray<string>>([]);
2843 const harness = yield* makeHarness({
2844 configEvents:
2845 discovery !== "initial"
2846 ? Stream.concat(
2847 Stream.make({
2848 version: 1 as const,
2849 type: "snapshot" as const,
2850 config: (discovery === "omitted"
2851 ? {}
2852 : { directEndpoints: [] }) as unknown as ServerConfig,
2853 }).pipe(Stream.ensuring(Deferred.succeed(initialProcessed, undefined))),
2854 Stream.fromEffect(
2855 Deferred.await(finishDiscovery).pipe(
2856 Effect.as({
2857 version: 1 as const,
2858 type: "snapshot" as const,
2859 config: {
2860 directEndpoints: [{ kind: "tailnet", httpBaseUrl: lateAddress }],
2861 } as unknown as ServerConfig,
2862 }),
2863 ),
2864 ),
2865 ).pipe(Stream.ensuring(Deferred.succeed(discoveryProcessed, undefined)))
2866 : undefined,
2867 checkRoute: (route) =>
2868 route.target._tag === "BearerConnectionTarget"
2869 ? Effect.succeed("answered")
2870 : Effect.succeed("unchecked"),
2871 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
2872 initialConfig: () =>
2873 Effect.succeed({
2874 directEndpoints:
2875 discovery !== "initial"
2876 ? []
2877 : [{ kind: "lan", httpBaseUrl: "http://192.168.1.10:3773/" }],
2878 } as unknown as ServerConfig),
2879 });
2880 const current = yield* Ref.make(relayEntry);
2881 const supervisor = yield* EnvironmentSupervisor.make(relayEntry, {
2882 initiallyDesired: true,
2883 learnRoutes: ({ activeRoute, reported }) =>
2884 Effect.gen(function* () {
2885 yield* Deferred.succeed(discoveryStarted, undefined);
2886 const entry = yield* Ref.get(current);
2887 const routes = mergeLearnedRoutes({
2888 entry,
2889 activeRoute,
2890 reported,
2891 allowInsecure: true,
2892 });
2893 if (routes === null) return Option.none();
2894 const next = entryWithRoutes(entry, routes);
2895 yield* Ref.set(current, next);
2896 yield* Ref.update(learned, (all) => [
2897 ...all,
2898 ...routes.filter(isLearned).map((route) => connectionRouteId(route.target)),
2899 ]);
2900 return Option.some(next);
2901 }),
2902 }).pipe(Effect.provide(harness.dependencies));
2903
2904 if (discovery === "omitted") {
2905 yield* Deferred.await(initialProcessed);
2906 expect(yield* Ref.get(current)).toEqual(relayEntry);
2907 expect(yield* Ref.get(learned)).toEqual([]);
2908 } else {
2909 yield* Deferred.await(discoveryStarted);
2910 }
2911 yield* Deferred.succeed(finishDiscovery, undefined);
2912 if (discovery === "withdrawn-upgrade") {
2913 yield* Deferred.await(discoveryProcessed);
2914 yield* TestClock.adjust("60 seconds");
2915 expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(2);
2916 return;
2917 }
2918 if (discovery === "withdrawn" || discovery === "omitted") {
2919 yield* Deferred.await(discoveryProcessed);
2920 expect(yield* Ref.get(learned)).toContain(
2921 `learned:${TARGET.environmentId}:https://fixture.tail1234.ts.net`,
2922 );
2923 expect((yield* SubscriptionRef.get(supervisor.state)).generation).toBe(1);
2924 return;
2925 }
2926 yield* awaitState(
2927 supervisor.state,
2928 (state) => state.phase === "connected" && state.generation === 2,
2929 );
2930 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
2931 "BearerConnectionTarget",
2932 );
2933 expect(yield* Ref.get(learned)).toEqual([
2934 `learned:${TARGET.environmentId}:${discovery !== "initial" ? "https://fixture.tail1234.ts.net" : "http://192.168.1.10:3773"}`,
2935 ]);
2936 }),
2937 );
2938 });
2939
2940 describe("EnvironmentSupervisor background fibers", () => {
2941 it.effect("hears the network return after its connectivity stream died", () =>
2942 Effect.gen(function* () {
2943 const harness = yield* makeHarness();
2944 const status = yield* SubscriptionRef.make<NetworkStatus>("offline");
2945 const subscriptions = yield* Ref.make(0);
2946 const dying = Connectivity.Connectivity.of({
2947 status: SubscriptionRef.get(status),
2948 changes: Stream.unwrap(
2949 Ref.updateAndGet(subscriptions, (count) => count + 1).pipe(
2950 Effect.map((count) =>
2951 count === 1
2952 ? Stream.die(new Error("platform connectivity stream died"))
2953 : SubscriptionRef.changes(status),
2954 ),
2955 ),
2956 ),
2957 });
2958 const supervisor = yield* EnvironmentSupervisor.make(RELAY_ENTRY, {
2959 initiallyDesired: true,
2960 }).pipe(
2961 Effect.provideService(Connectivity.Connectivity, dying),
2962 Effect.provide(harness.dependencies),
2963 );
2964 yield* TestClock.adjust("10 minutes");
2965 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("offline");
2966
2967 yield* SubscriptionRef.set(status, "online");
2968 yield* TestClock.adjust("10 minutes");
2969 expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected");
2970 }).pipe(Effect.provide(TestClock.layer())),
2971 );
2972
2973 it.effect("replaces a stalled session on return to the app after its wakeup stream died", () =>
2974 Effect.gen(function* () {
2975 const harness = yield* makeHarness({ probe: () => Effect.never });
2976 const returns = yield* SubscriptionRef.make(0);
2977 const subscriptions = yield* Ref.make(0);
2978 const dying = ConnectionWakeups.ConnectionWakeups.of({
2979 changes: Stream.unwrap(
2980 Ref.updateAndGet(subscriptions, (count) => count + 1).pipe(
2981 Effect.map((count) =>
2982 count === 1
2983 ? Stream.die(new Error("platform wakeup stream died"))
2984 : SubscriptionRef.changes(returns).pipe(
2985 Stream.drop(1),
2986 Stream.map(() => "application-active" as const),
2987 ),
2988 ),
2989 ),
2990 ),
2991 });
2992 const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, {
2993 initiallyDesired: true,
2994 }).pipe(
2995 Effect.provideService(ConnectionWakeups.ConnectionWakeups, dying),
2996 Effect.provide(harness.dependencies),
2997 );
2998 yield* TestClock.adjust("10 minutes");
2999 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
3000 phase: "connected",
3001 generation: 1,
3002 });
3003
3004 // The user returns to the app; the session's probe never answers.
3005 yield* SubscriptionRef.update(returns, (count) => count + 1);
3006 yield* TestClock.adjust("15 seconds");
3007 yield* TestClock.adjust("15 seconds");
3008 yield* TestClock.adjust("10 minutes");
3009 expect(yield* SubscriptionRef.get(supervisor.state)).toMatchObject({
3010 phase: "connected",
3011 generation: 2,
3012 });
3013 }).pipe(Effect.provide(TestClock.layer())),
3014 );
3015
3016 it.effect("still moves to a better route after one route check crashed", () =>
3017 Effect.gen(function* () {
3018 const lanReachable = yield* Ref.make(false);
3019 const preflights = yield* Ref.make(0);
3020 const harness = yield* makeHarness({
3021 checkRoute: (route) =>
3022 route.target._tag === "BearerConnectionTarget"
3023 ? Ref.get(lanReachable).pipe(
3024 Effect.map((reachable) => (reachable ? "answered" : "silent")),
3025 )
3026 : Effect.succeed("unchecked"),
3027 preflight: () =>
3028 Ref.updateAndGet(preflights, (count) => count + 1).pipe(
3029 Effect.flatMap((count) =>
3030 count === 1
3031 ? Effect.die(new Error("route check threw"))
3032 : Ref.get(lanReachable).pipe(
3033 Effect.map((reachable) => (reachable ? "ready" : "silent")),
3034 ),
3035 ),
3036 ),
3037 prepare: (_attempt, target) => Effect.succeed(preparedFor(target)),
3038 });
3039 const supervisor = yield* EnvironmentSupervisor.make(LAN_THEN_RELAY_ENTRY, {
3040 initiallyDesired: true,
3041 }).pipe(Effect.provide(harness.dependencies));
3042 yield* awaitState(
3043 supervisor.state,
3044 (state) => state.phase === "connected" && state.generation === 1,
3045 );
3046 yield* TestClock.adjust("60 seconds");
3047 expect(yield* Ref.get(preflights)).toBeGreaterThanOrEqual(1);
3048
3049 yield* Ref.set(lanReachable, true);
3050 for (let minute = 0; minute < 60; minute += 1) {
3051 yield* TestClock.adjust("60 seconds");
3052 }
3053 expect(Option.getOrThrow(yield* SubscriptionRef.get(supervisor.prepared)).target._tag).toBe(
3054 "BearerConnectionTarget",
3055 );
3056 }).pipe(Effect.provide(TestClock.layer())),
3057 );
3058 });