The best way to control Claude Code, Codex and more from any computer or phone

main
ts 910 lines 36.4 KB
Raw
1 import { mFallback, msg } from "gt-i18n";
2 import type { MFunctionType } from "gt-i18n/types";
3 import {
4 ORCHESTRATION_V2_WS_METHODS,
5 type EnvironmentId,
6 type ExecutionEnvironmentDescriptor,
7 type OrchestrationV2ShellSnapshot,
8 type OrchestrationV2ShellStreamItem,
9 type OrchestrationV2ThreadShell,
10 type ServerConfig,
11 type ThreadId,
12 type ThreadPullRequestLink,
13 } from "@t3tools/contracts";
14 import * as Cause from "effect/Cause";
15 import * as Clock from "effect/Clock";
16 import * as Effect from "effect/Effect";
17 import * as Exit from "effect/Exit";
18 import * as Fiber from "effect/Fiber";
19 import * as Option from "effect/Option";
20 import * as Queue from "effect/Queue";
21 import * as Ref from "effect/Ref";
22 import * as Semaphore from "effect/Semaphore";
23 import * as Stream from "effect/Stream";
24 import * as SubscriptionRef from "effect/SubscriptionRef";
25 import * as SynchronizedRef from "effect/SynchronizedRef";
26 import { AsyncResult, Atom } from "effect/reactivity";
27
28 import * as EnvironmentRegistry from "../connection/registry.ts";
29 import { connectionProjectionPhase, type PreparedConnection } from "../connection/model.ts";
30 import * as EnvironmentSupervisor from "../connection/supervisor.ts";
31 import * as ConnectionWakeups from "../connection/wakeups.ts";
32 import { safeErrorLogAttributes } from "../errors/safeLog.ts";
33 import * as Persistence from "../platform/persistence.ts";
34 import { runCachePersistence } from "./cachePersistence.ts";
35 import { subscribeDynamicWithSession, subscriptionRetryDelay } from "../rpc/client.ts";
36 import type { RpcSession } from "../rpc/session.ts";
37 import * as ShellSnapshotLoader from "./shellSnapshotHttp.ts";
38 import type { DeferredShellSnapshot } from "./shellPullRequests.ts";
39 import {
40 applyShellStreamEvents,
41 type ShellStreamEvent,
42 mergeShellSnapshotProjects,
43 reuseUnchangedThreadShells,
44 sameThreadPullRequests,
45 } from "./shellReducer.ts";
46 import { type EnvironmentCatalogState, enabledEnvironmentIds } from "./connections.ts";
47 import { followStreamInEnvironment } from "./runtime.ts";
48
49 export type EnvironmentShellStatus = "empty" | "cached" | "synchronizing" | "live";
50
51 export interface EnvironmentShellState {
52 readonly snapshot: Option.Option<OrchestrationV2ShellSnapshot>;
53 readonly status: EnvironmentShellStatus;
54 readonly error: Option.Option<string>;
55 /**
56 * Changes with each authoritative snapshot (not enrichment refreshes); 0 until the first one.
57 * Thread families read for an older generation may miss changes from before the latest snapshot,
58 * so they are read again.
59 */
60 readonly generation: number;
61 /** Sequences of the thread removals applied since the latest authoritative snapshot. */
62 readonly removedAt: ReadonlyMap<ThreadId, number>;
63 }
64
65 const EMPTY_REMOVALS: ReadonlyMap<ThreadId, number> = new Map();
66 // Shared by every shell state, so a rebuilt one never reuses a generation another already used.
67 let lastShellGeneration = 0;
68
69 export const EMPTY_SHELL_STATE: EnvironmentShellState = {
70 snapshot: Option.none(),
71 status: "empty",
72 error: Option.none(),
73 generation: 0,
74 removedAt: EMPTY_REMOVALS,
75 };
76
77 function shellStatusForSnapshot(
78 snapshot: Option.Option<OrchestrationV2ShellSnapshot>,
79 ): EnvironmentShellStatus {
80 return Option.isSome(snapshot) ? "cached" : "empty";
81 }
82
83 function withoutDeferredPullRequests(value: DeferredShellSnapshot): OrchestrationV2ShellSnapshot {
84 if (value.loadPullRequests === undefined) return value;
85 const { loadPullRequests: _deferred, ...snapshot } = value;
86 return snapshot;
87 }
88
89 const SHELL_SYNCHRONIZATION_ERROR_MESSAGE = msg("Could not synchronize environment data.");
90
91 /** @public The shell's sync error in the active language; other messages stay as they are. */
92 export function shellErrorText(error: string, resolve: MFunctionType = mFallback): string {
93 return error === SHELL_SYNCHRONIZATION_ERROR_MESSAGE ? resolve(error) : error;
94 }
95
96 /**
97 * Replay does not refresh the older-settled cut, rows that left the newest window, or
98 * projections the server rebuilt, so drift grows with the age of the last server snapshot.
99 */
100 const CACHE_RESUME_MAX_AGE_MILLIS = 12 * 60 * 60 * 1000;
101
102 // How long a ready socket waits for the early HTTP answer before it resumes held rows itself.
103 // The supervisor's quick probe allows a live host the same time. A proxy that drops `/api`
104 // without an answer would otherwise hold each connect for the full HTTP timeout.
105 const EARLY_LOAD_GRACE = "3 seconds";
106
107 type ShellOriginIdentity = Omit<Persistence.ShellCacheOrigin, "snapshotAt">;
108
109 interface ShellSubscription {
110 readonly session: RpcSession;
111 /** What a snapshot from this subscription descends from; null when the server did not say. */
112 readonly identity: ShellOriginIdentity | null;
113 }
114
115 function shellIdentity(
116 environment: Pick<ExecutionEnvironmentDescriptor, "serverVersion" | "databaseMigration">,
117 flags: { readonly shellThreadFamilies?: boolean; readonly shellOlderSettled?: boolean },
118 ): ShellOriginIdentity {
119 const omitIdleSubagents = flags.shellThreadFamilies === true;
120 return {
121 serverVersion: environment.serverVersion,
122 // Migrations rewrite projections without events, so a migrated server reloads.
123 databaseMigration: environment.databaseMigration ?? null,
124 omitIdleSubagents,
125 omitOlderSettled: flags.shellOlderSettled === true && omitIdleSubagents,
126 };
127 }
128
129 function sameConnection(left: PreparedConnection, right: PreparedConnection): boolean {
130 const keys = Object.keys({ ...left, ...right }) as Array<keyof PreparedConnection>;
131 return keys.every((key) => key === "httpAuthorization" || left[key] === right[key]);
132 }
133
134 /** Null for a server whose descriptor does not name its shell flags; only its config does. */
135 function descriptorShellIdentity(
136 descriptor: ExecutionEnvironmentDescriptor | undefined,
137 ): ShellOriginIdentity | null {
138 if (descriptor?.capabilities.shellThreadFamilies === undefined) return null;
139 return shellIdentity(descriptor, descriptor.capabilities);
140 }
141
142 function sameShellIdentity(left: ShellOriginIdentity, right: ShellOriginIdentity): boolean {
143 return (
144 left.serverVersion === right.serverVersion &&
145 left.databaseMigration === right.databaseMigration &&
146 left.omitIdleSubagents === right.omitIdleSubagents &&
147 left.omitOlderSettled === right.omitOlderSettled
148 );
149 }
150
151 function canResumeFrom(
152 origin: Persistence.ShellCacheOrigin | undefined,
153 identity: ShellOriginIdentity | null,
154 now: number,
155 ): boolean {
156 if (origin === undefined || identity === null) return false;
157 const age = now - origin.snapshotAt;
158 return sameShellIdentity(origin, identity) && age >= 0 && age < CACHE_RESUME_MAX_AGE_MILLIS;
159 }
160
161 type HttpShellOutcome =
162 | { readonly kind: "snapshot"; readonly projectIdentities: string | undefined }
163 /** The server can replay from the rows the client holds. */
164 | { readonly kind: "resumable" }
165 | { readonly kind: "failed" };
166
167 interface EarlyShellLoad {
168 readonly prepared: PreparedConnection;
169 readonly identity: ShellOriginIdentity;
170 readonly fiber: Fiber.Fiber<HttpShellOutcome>;
171 // One subscription uses a load; a later one on the same connection decides again.
172 readonly claimed: boolean;
173 }
174
175 interface LiveShell {
176 readonly snapshot: OrchestrationV2ShellSnapshot;
177 // Read together with the rows, so a save never pairs them with another snapshot's origin.
178 readonly origin: Persistence.ShellCacheOrigin | undefined;
179 }
180
181 export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")(function* () {
182 const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor;
183 const cache = yield* Persistence.EnvironmentCacheStore;
184 const snapshotLoader = yield* ShellSnapshotLoader.ShellSnapshotLoader;
185 const wakeups = yield* Effect.serviceOption(ConnectionWakeups.ConnectionWakeups);
186 const environmentId = supervisor.target.environmentId;
187 const cached = yield* cache.loadShell(environmentId).pipe(
188 Effect.catch((error) =>
189 Effect.logWarning("Could not load cached environment shell.").pipe(
190 Effect.annotateLogs({
191 environmentId,
192 ...safeErrorLogAttributes(error),
193 }),
194 Effect.as(Option.none<Persistence.CachedShellSnapshot>()),
195 ),
196 ),
197 );
198 const cachedOrigin = Option.isSome(cached) ? cached.value.origin : undefined;
199 const cachedSnapshot = Option.map(cached, ({ origin: _origin, ...rows }) =>
200 withoutDeferredPullRequests(rows),
201 );
202 const state = yield* SubscriptionRef.make<EnvironmentShellState>({
203 ...EMPTY_SHELL_STATE,
204 snapshot: cachedSnapshot,
205 status: shellStatusForSnapshot(cachedSnapshot),
206 });
207 const awaitingCompletion = yield* Ref.make(false);
208 const activeSubscription = yield* Ref.make<ShellSubscription | null>(null);
209 // The server snapshot the in-memory rows descend from, directly or through the cache.
210 // Only a server snapshot renews it; `snapshot-required` drops it.
211 const origin = yield* Ref.make(cachedOrigin);
212 const reloadRequests = yield* Queue.unbounded<void>();
213 let defectRetries = 0;
214 const latestLive = yield* Ref.make<Option.Option<LiveShell>>(Option.none());
215 const persistence = yield* Queue.sliding<OrchestrationV2ShellSnapshot>(1);
216
217 const persistenceLock = yield* Semaphore.make(1);
218 let lastPersisted: LiveShell | undefined;
219 // Rows still waiting for their deferred pull request links, which must not be saved:
220 // the cache would keep them without links until the next server refresh.
221 let pendingLinkRows: ReadonlySet<OrchestrationV2ThreadShell> = new Set();
222 const persistLatest = Effect.fn("EnvironmentShellState.persistLatest")(function* (
223 flush: boolean,
224 ) {
225 while (true) {
226 const latest = yield* Ref.get(latestLive);
227 if (Option.isNone(latest) || latest.value === lastPersisted) return;
228 const { snapshot, origin } = latest.value;
229 if (snapshot.threads.some((thread) => pendingLinkRows.has(thread))) return;
230 if (
231 lastPersisted !== undefined &&
232 origin === lastPersisted.origin &&
233 snapshot.snapshotSequence >= lastPersisted.snapshot.snapshotSequence &&
234 snapshot.schemaVersion === lastPersisted.snapshot.schemaVersion &&
235 snapshot.projects === lastPersisted.snapshot.projects &&
236 snapshot.threads === lastPersisted.snapshot.threads &&
237 snapshot.archivedThreads === lastPersisted.snapshot.archivedThreads
238 ) {
239 // Keep the live cursor without rewriting the rows. Remember it here too so a
240 // later authoritative reset to a lower sequence still reaches the cache.
241 lastPersisted = latest.value;
242 return;
243 }
244 const saved = yield* cache.saveShell(environmentId, snapshot, origin).pipe(
245 Effect.as(true),
246 Effect.catch((error) =>
247 Effect.logWarning("Could not persist environment shell cache.").pipe(
248 Effect.annotateLogs({ environmentId, ...safeErrorLogAttributes(error) }),
249 Effect.as(false),
250 ),
251 ),
252 );
253 if (!saved) return;
254 lastPersisted = latest.value;
255 if (!flush) return;
256 }
257 }, persistenceLock.withPermit);
258 const persist = () => persistLatest(false);
259 const flushLiveShellSnapshot = persistLatest(true);
260
261 // Register before scoped worker fibers so reverse finalizer order interrupts
262 // those fibers first and this flush sees a stable latestLiveSnapshot.
263 yield* Effect.addFinalizer(() => flushLiveShellSnapshot);
264
265 yield* runCachePersistence(persistence, persist).pipe(Effect.forkScoped);
266
267 const setDisconnected = Ref.set(awaitingCompletion, false).pipe(
268 Effect.andThen(
269 SubscriptionRef.update(state, (current) => ({
270 ...current,
271 status: shellStatusForSnapshot(current.snapshot),
272 })),
273 ),
274 Effect.andThen(flushLiveShellSnapshot.pipe(Effect.forkScoped)),
275 Effect.asVoid,
276 );
277 const setSynchronizing = SubscriptionRef.update(state, (current) => ({
278 ...current,
279 status: "synchronizing" as const,
280 error: Option.none(),
281 }));
282 const setReady = SubscriptionRef.update(state, (current) =>
283 current.status === "live"
284 ? current
285 : {
286 ...current,
287 status: "synchronizing" as const,
288 error: Option.none(),
289 },
290 );
291 const setStreamError = (error: unknown) =>
292 Ref.set(awaitingCompletion, false).pipe(
293 Effect.andThen(Effect.logWarning("Could not synchronize the environment shell.")),
294 Effect.annotateLogs({
295 environmentId,
296 ...safeErrorLogAttributes(error),
297 }),
298 Effect.andThen(
299 SubscriptionRef.update(state, (current) => ({
300 ...current,
301 status: shellStatusForSnapshot(current.snapshot),
302 error: Option.some(SHELL_SYNCHRONIZATION_ERROR_MESSAGE),
303 })),
304 ),
305 );
306
307 // Server items and deferred link fills both read the snapshot and write it back; one at a
308 // time, so neither can overwrite what the other wrote in between.
309 const snapshotWriteLock = yield* Semaphore.make(1);
310
311 const applyLocked = Effect.fnUntraced(function* (
312 items: ReadonlyArray<OrchestrationV2ShellStreamItem>,
313 /** What an authoritative snapshot among the items descends from. */
314 identity: ShellOriginIdentity | null,
315 /** Set for an HTTP snapshot: the rows that still wait for their pull request links. */
316 pendingLinkIds?: ReadonlySet<ThreadId>,
317 ) {
318 const initial = yield* SubscriptionRef.get(state);
319 // No socket has confirmed an HTTP snapshot, so it never makes the shell live.
320 const fromHttp = pendingLinkIds !== undefined;
321 const disconnected = initial.status === "empty" || initial.status === "cached";
322 let waiting = yield* Ref.get(awaitingCompletion);
323 let next = initial;
324 let receivedAuthoritativeSnapshot = false;
325 let snapshotRequired = false;
326 let deltas: Array<ShellStreamEvent> = [];
327 const flushDeltas = () => {
328 if (deltas.length > 0 && Option.isSome(next.snapshot)) {
329 let sequence = next.snapshot.value.snapshotSequence;
330 let removedAt: Map<ThreadId, number> | undefined;
331 for (const delta of deltas) {
332 if (delta.sequence <= sequence) continue;
333 switch (delta.kind) {
334 case "thread.removed":
335 removedAt ??= new Map(next.removedAt);
336 removedAt.set(delta.threadId, delta.sequence);
337 break;
338 case "thread.updated":
339 case "project.updated":
340 case "project.removed":
341 break;
342 default:
343 continue;
344 }
345 sequence = delta.sequence;
346 }
347 next = {
348 ...next,
349 removedAt: removedAt ?? next.removedAt,
350 snapshot: Option.some(applyShellStreamEvents(next.snapshot.value, deltas)),
351 status: waiting ? "synchronizing" : "live",
352 error: Option.none(),
353 };
354 }
355 deltas = [];
356 };
357 for (const item of items) {
358 if (item.kind === "snapshot-required") {
359 snapshotRequired = true;
360 continue;
361 }
362 if (item.kind !== "snapshot" && item.kind !== "synchronized") {
363 deltas.push(item);
364 continue;
365 }
366 // Snapshots can reset the cursor; completion markers change the sync status.
367 flushDeltas();
368 if (item.kind === "synchronized") {
369 waiting = false;
370 if (Option.isSome(next.snapshot)) {
371 next = {
372 ...next,
373 status: "live",
374 error: Option.none(),
375 // A cache resume that caught up is as current as a snapshot for generation readers.
376 generation: next.generation === 0 ? ++lastShellGeneration : next.generation,
377 };
378 }
379 continue;
380 }
381 const authoritative = item.resolvedRepositoryIdentityRoots === undefined;
382 const nextSnapshot = mergeShellSnapshotProjects(
383 Option.getOrNull(next.snapshot),
384 item.snapshot,
385 item.resolvedRepositoryIdentityRoots === undefined
386 ? undefined
387 : { resolvedRepositoryIdentityRoots: item.resolvedRepositoryIdentityRoots },
388 );
389 if (authoritative) receivedAuthoritativeSnapshot = true;
390 next = {
391 snapshot: Option.some(nextSnapshot),
392 status:
393 fromHttp && disconnected ? "cached" : waiting || fromHttp ? "synchronizing" : "live",
394 error: Option.none(),
395 generation: authoritative ? ++lastShellGeneration : next.generation,
396 removedAt: authoritative ? EMPTY_REMOVALS : next.removedAt,
397 };
398 }
399 flushDeltas();
400 yield* Ref.set(awaitingCompletion, waiting);
401 let nextOrigin = yield* Ref.get(origin);
402 if (snapshotRequired) {
403 // The rows stay on screen but reload before the next resume; the cache keeps the old
404 // origin until that reload saves, which costs at most one more refusal.
405 nextOrigin = undefined;
406 yield* Ref.set(origin, nextOrigin);
407 yield* Queue.offer(reloadRequests, undefined);
408 } else if (receivedAuthoritativeSnapshot) {
409 nextOrigin =
410 identity === null ? undefined : { ...identity, snapshotAt: yield* Clock.currentTimeMillis };
411 yield* Ref.set(origin, nextOrigin);
412 }
413 if (next === initial) return;
414 if (pendingLinkIds !== undefined && Option.isSome(next.snapshot)) {
415 pendingLinkRows = new Set(
416 next.snapshot.value.threads.filter((thread) => pendingLinkIds.has(thread.id)),
417 );
418 }
419 if (Option.isSome(next.snapshot)) {
420 yield* Ref.set(
421 latestLive,
422 Option.some({ snapshot: next.snapshot.value, origin: nextOrigin }),
423 );
424 }
425 yield* SubscriptionRef.set(state, next);
426 if (next.snapshot !== initial.snapshot && Option.isSome(next.snapshot)) {
427 yield* Queue.offer(persistence, next.snapshot.value);
428 }
429 });
430
431 const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* (
432 entries: ReadonlyArray<readonly [RpcSession, OrchestrationV2ShellStreamItem]>,
433 ) {
434 // Read under the lock: items buffered across a session change must not land on the
435 // rows the next session loaded.
436 const subscription = yield* Ref.get(activeSubscription);
437 yield* applyLocked(
438 entries.flatMap(([session, item]) => (session === subscription?.session ? [item] : [])),
439 subscription?.identity ?? null,
440 );
441 }, snapshotWriteLock.withPermit);
442
443 const stateScope = yield* Effect.scope;
444 // Each full snapshot supersedes the previous one's fill; links from an older snapshot must
445 // not land on rows a newer snapshot reused.
446
447 let activeFill: Fiber.Fiber<void> | undefined;
448 const cancelActiveFill = Effect.suspend(() => {
449 const fill = activeFill;
450 activeFill = undefined;
451 return fill === undefined ? Effect.void : Fiber.interrupt(fill);
452 });
453 const fillDeferredPullRequests = Effect.fnUntraced(function* (
454 source: DeferredShellSnapshot,
455 applied: OrchestrationV2ShellSnapshot,
456 ) {
457 yield* cancelActiveFill;
458 if (source.loadPullRequests === undefined) return;
459 const arrivedRows = new Set(applied.threads);
460 const arrivedIds = new Set(source.threads.map((thread) => thread.id));
461 pendingLinkRows = new Set(applied.threads.filter((thread) => arrivedIds.has(thread.id)));
462 activeFill = yield* source.loadPullRequests.pipe(
463 Effect.flatMap((linksByThreadId) =>
464 writeFilledLinks(linksByThreadId, arrivedRows, arrivedIds),
465 ),
466 Effect.forkIn(stateScope),
467 );
468 });
469 const writeFilledLinks = Effect.fnUntraced(
470 function* (
471 linksByThreadId: ReadonlyMap<ThreadId, ReadonlyArray<ThreadPullRequestLink>>,
472 arrivedRows: ReadonlySet<OrchestrationV2ThreadShell>,
473 arrivedIds: ReadonlySet<ThreadId>,
474 ) {
475 const filled = yield* SubscriptionRef.modify(
476 state,
477 (current): [Option.Option<OrchestrationV2ShellSnapshot>, EnvironmentShellState] => {
478 if (Option.isNone(current.snapshot)) return [Option.none(), current];
479 const snapshot = current.snapshot.value;
480 let changed = false;
481 const threads = snapshot.threads.map((thread) => {
482 if (!arrivedRows.has(thread) || !arrivedIds.has(thread.id)) return thread;
483 const links = linksByThreadId.get(thread.id);
484 if (sameThreadPullRequests(thread.pullRequests, links)) return thread;
485 changed = true;
486 if (links === undefined) {
487 const { pullRequests: _previous, ...row } = thread;
488 return row;
489 }
490 return { ...thread, pullRequests: links };
491 });
492 if (!changed) return [Option.none(), current];
493 const next = { ...snapshot, threads };
494 return [Option.some(next), { ...current, snapshot: Option.some(next) }];
495 },
496 );
497 pendingLinkRows = new Set();
498 const live = yield* Ref.get(latestLive);
499 if (Option.isNone(live)) return;
500 const latest = Option.isSome(filled) ? filled.value : live.value.snapshot;
501 yield* Ref.set(latestLive, Option.some({ snapshot: latest, origin: live.value.origin }));
502 yield* Queue.offer(persistence, latest);
503 },
504 (effect) => snapshotWriteLock.withPermit(effect),
505 );
506 if (Option.isSome(cached) && Option.isSome(cachedSnapshot)) {
507 yield* fillDeferredPullRequests(cached.value, cachedSnapshot.value);
508 }
509
510 const applyHttpSnapshot = Effect.fnUntraced(function* (
511 loaded: ShellSnapshotLoader.ShellHttpSnapshot,
512 identity: ShellOriginIdentity | null,
513 ) {
514 const { projectIdentities: _identities, ...deferred } = loaded;
515 yield* cancelActiveFill;
516 const previous = yield* SubscriptionRef.get(state);
517 const snapshot = reuseUnchangedThreadShells(
518 Option.getOrNull(previous.snapshot),
519 withoutDeferredPullRequests(deferred),
520 );
521 const pendingLinkIds = new Set(
522 deferred.loadPullRequests === undefined ? [] : deferred.threads.map((thread) => thread.id),
523 );
524 yield* snapshotWriteLock.withPermit(
525 applyLocked([{ kind: "snapshot", snapshot }], identity, pendingLinkIds),
526 );
527 const current = yield* SubscriptionRef.get(state);
528 if (Option.isSome(current.snapshot)) {
529 yield* fillDeferredPullRequests(deferred, current.snapshot.value);
530 }
531 }, Effect.uninterruptible);
532
533 /**
534 * One request to the HTTP shell route. Held rows that may resume are named, so the server
535 * answers that they still can, in a few bytes, or sends the snapshot that replaces them.
536 */
537 const loadOverHttp = Effect.fn("EnvironmentShellState.loadOverHttp")(function* (
538 prepared: PreparedConnection,
539 identity: ShellOriginIdentity | null,
540 ) {
541 const held = yield* SubscriptionRef.get(state);
542 const resumeFrom =
543 Option.isSome(held.snapshot) &&
544 canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis)
545 ? held.snapshot.value.snapshotSequence
546 : undefined;
547 // HTTP and the socket must agree, or omitted children would return through the socket.
548 const answer = yield* snapshotLoader.load(prepared, {
549 omitIdleSubagents: identity?.omitIdleSubagents === true,
550 omitOlderSettled: identity?.omitOlderSettled === true,
551 ...(resumeFrom === undefined ? {} : { resumeFrom }),
552 });
553 if (Option.isNone(answer)) return { kind: "failed" } satisfies HttpShellOutcome;
554 if ("resumable" in answer.value) return { kind: "resumable" } satisfies HttpShellOutcome;
555 yield* applyHttpSnapshot(answer.value, identity);
556 return {
557 kind: "snapshot",
558 projectIdentities: answer.value.projectIdentities,
559 } satisfies HttpShellOutcome;
560 });
561
562 // A server whose descriptor names its shell flags is asked for the shell as soon as the
563 // connection is prepared, while the socket opens and the config downloads. The subscription
564 // claims that load, and reloads when the live config disagrees with the descriptor.
565 const earlyLoad = yield* SynchronizedRef.make<EarlyShellLoad | null>(null);
566 const useEarlyLoad = (prepared: PreparedConnection | null, claim: boolean) =>
567 SynchronizedRef.modifyEffect(
568 earlyLoad,
569 Effect.fnUntraced(function* (current) {
570 // A rotated credential replaces the prepared connection of a session that stays up.
571 if (current !== null && prepared !== null && sameConnection(current.prepared, prepared)) {
572 const next = { ...current, prepared, claimed: current.claimed || claim };
573 // A claimed load that still runs is handed out again. A second request could
574 // answer first, and this one would then replace its rows under a passed cursor.
575 const done = current.fiber.pollUnsafe() !== undefined;
576 return [current.claimed && done ? null : current, next] as const;
577 }
578 if (current !== null) yield* Fiber.interrupt(current.fiber);
579 const identity = descriptorShellIdentity(prepared?.descriptor);
580 if (prepared === null || identity === null) return [null, null] as const;
581 // Items a closed session still has in flight must not land on the rows this loads.
582 yield* Ref.set(activeSubscription, null);
583 const fiber = yield* loadOverHttp(prepared, identity).pipe(Effect.forkIn(stateScope));
584 const next: EarlyShellLoad = { prepared, identity, fiber, claimed: claim };
585 return [next, next] as const;
586 }),
587 ).pipe(
588 // Covers the store too. A cancel that lands during the wait above would otherwise take
589 // effect before the store, and the load just started would run on with no record.
590 Effect.uninterruptible,
591 );
592 /**
593 * The subscription waits for the early load, so its answer is applied before the socket
594 * moves the rows on. Held rows that can resume wait a limited time only: after that the
595 * load is ended, so a late answer applies nothing, and the socket resumes them.
596 */
597 const awaitEarlyLoad = Effect.fnUntraced(function* (
598 early: EarlyShellLoad,
599 identity: ShellOriginIdentity | null,
600 ) {
601 const resumable =
602 Option.isSome((yield* SubscriptionRef.get(state)).snapshot) &&
603 canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis);
604 if (!resumable) return yield* Fiber.await(early.fiber);
605 const answered = yield* Fiber.await(early.fiber).pipe(Effect.timeoutOption(EARLY_LOAD_GRACE));
606 if (Option.isSome(answered)) return answered.value;
607 // Returns after an answer that is being applied has finished, so the cursor below is read
608 // from rows that no longer change under it.
609 yield* Fiber.interrupt(early.fiber);
610 return yield* Fiber.await(early.fiber);
611 });
612
613 yield* SubscriptionRef.changes(supervisor.prepared).pipe(
614 // Reads the connection of this moment: a change handled late must not start a load for a
615 // connection that ended, which would also drop the items of the session that replaced it.
616 Stream.runForEach(() =>
617 SubscriptionRef.get(supervisor.prepared).pipe(
618 Effect.flatMap((prepared) => useEarlyLoad(Option.getOrNull(prepared), false)),
619 ),
620 ),
621 Effect.forkScoped,
622 );
623
624 const resubscriptions = Stream.merge(
625 Stream.fromQueue(reloadRequests),
626 Option.match(wakeups, {
627 onNone: () => Stream.never,
628 onSome: (service) =>
629 service.changes.pipe(Stream.filter(ConnectionWakeups.shouldResubscribeAfterWakeup)),
630 }),
631 );
632
633 yield* setSynchronizing;
634 yield* Effect.forkScoped(
635 subscribeDynamicWithSession(
636 ORCHESTRATION_V2_WS_METHODS.subscribeShell,
637 Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) {
638 const identity = yield* session.initialConfig.pipe(
639 Effect.map((config) => shellIdentity(config.environment, config)),
640 Effect.orElseSucceed(() => null),
641 );
642 const { omitIdleSubagents, omitOlderSettled } = identity ?? {
643 omitIdleSubagents: false,
644 omitOlderSettled: false,
645 };
646 const early = yield* useEarlyLoad(
647 Option.getOrNull(yield* SubscriptionRef.get(supervisor.prepared)),
648 true,
649 );
650 const earlyExit = early === null ? null : yield* awaitEarlyLoad(early, identity);
651 // Rows the descriptor's flags or build loaded are not kept when the live config differs:
652 // their origin then cannot resume, so the load below replaces them.
653 let loaded =
654 early !== null &&
655 earlyExit !== null &&
656 Exit.isSuccess(earlyExit) &&
657 identity !== null &&
658 sameShellIdentity(early.identity, identity)
659 ? earlyExit.value
660 : null;
661 yield* Ref.set(activeSubscription, { session, identity });
662 yield* Ref.set(awaitingCompletion, true);
663 yield* setSynchronizing;
664
665 // Rows from a recent server snapshot with the same omissions and server build resume
666 // from their cursor, in memory or from the cache. Anything else reloads the HTTP
667 // snapshot so a valid cursor cannot preserve stale or incomplete data. A `resumable`
668 // answer is checked again too: the rows can pass their age limit while the socket opens.
669 const resumingHeldRows =
670 loaded?.kind !== "snapshot" &&
671 Option.isSome((yield* SubscriptionRef.get(state)).snapshot) &&
672 canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis);
673 if (!resumingHeldRows && loaded === null) {
674 const prepared = yield* SubscriptionRef.get(supervisor.prepared).pipe(
675 Effect.flatMap(
676 Option.match({
677 onSome: Effect.succeed,
678 onNone: () =>
679 SubscriptionRef.changes(supervisor.prepared).pipe(
680 Stream.filter(Option.isSome),
681 Stream.map((value) => value.value),
682 Stream.runHead,
683 Effect.map(Option.getOrThrow),
684 ),
685 }),
686 ),
687 );
688 loaded = yield* loadOverHttp(prepared, identity);
689 }
690
691 const current = yield* SubscriptionRef.get(state);
692 const omission = {
693 ...(omitIdleSubagents ? { omitIdleSubagents: true as const } : {}),
694 ...(omitOlderSettled ? { omitOlderSettled: true as const } : {}),
695 };
696 if (Option.isNone(current.snapshot) || !(resumingHeldRows || loaded?.kind === "snapshot")) {
697 return { ...omission, requestCompletionMarker: true as const };
698 }
699 return {
700 afterSequence: current.snapshot.value.snapshotSequence,
701 // Only rows the client already held may be refused; a cursor from the snapshot just
702 // loaded falls back to the socket snapshot, so a refusal cannot loop.
703 ...(resumingHeldRows ? { httpSnapshotFallback: true as const } : {}),
704 // No stream item has touched the snapshot just loaded, so its identities still hold.
705 ...(loaded?.kind === "snapshot" && loaded.projectIdentities !== undefined
706 ? { knownProjectIdentities: loaded.projectIdentities }
707 : {}),
708 ...omission,
709 requestCompletionMarker: true as const,
710 };
711 }),
712 {
713 // A defect parks the stream, so resubscribe after a backoff rather than wait for a wakeup.
714 onDefect: (cause) =>
715 setStreamError(Cause.squash(cause)).pipe(
716 Effect.andThen(Effect.sleep(subscriptionRetryDelay("250 millis", defectRetries++))),
717 Effect.andThen(Queue.offer(reloadRequests, undefined)),
718 ),
719 onExpectedFailure: (cause) => setStreamError(Cause.squash(cause)),
720 retryExpectedFailureAfter: "250 millis",
721 resubscribe: resubscriptions,
722 },
723 ).pipe(
724 Stream.runForEachArray((entries) =>
725 applyItems(entries).pipe(
726 Effect.andThen(
727 Effect.sync(() => {
728 defectRetries = 0;
729 }),
730 ),
731 ),
732 ),
733 ),
734 );
735 yield* SubscriptionRef.changes(supervisor.state).pipe(
736 Stream.runForEach((connectionState) => {
737 switch (connectionProjectionPhase(connectionState)) {
738 case "synchronizing":
739 return setSynchronizing;
740 case "disconnected":
741 return setDisconnected;
742 case "ready":
743 return setReady;
744 }
745 }),
746 Effect.forkScoped,
747 );
748
749 return state;
750 });
751
752 function shellStateChanges(environmentId: EnvironmentId) {
753 return followStreamInEnvironment(
754 environmentId,
755 Stream.unwrap(makeEnvironmentShellState().pipe(Effect.map(SubscriptionRef.changes))),
756 );
757 }
758
759 export interface EnvironmentShellSummary {
760 readonly hasSnapshot: boolean;
761 readonly hasSynchronizingShell: boolean;
762 readonly firstError: string | null;
763 }
764
765 const EMPTY_ENVIRONMENT_SHELL_SUMMARY: EnvironmentShellSummary = Object.freeze({
766 hasSnapshot: false,
767 hasSynchronizingShell: false,
768 firstError: null,
769 });
770
771 const EMPTY_SERVER_CONFIGS: ReadonlyMap<EnvironmentId, ServerConfig> = new Map();
772
773 function shellSummariesEqual(
774 left: EnvironmentShellSummary,
775 right: EnvironmentShellSummary,
776 ): boolean {
777 return (
778 left.hasSnapshot === right.hasSnapshot &&
779 left.hasSynchronizingShell === right.hasSynchronizingShell &&
780 left.firstError === right.firstError
781 );
782 }
783
784 function mapsEqual<K, V>(left: ReadonlyMap<K, V>, right: ReadonlyMap<K, V>): boolean {
785 if (left.size !== right.size) {
786 return false;
787 }
788 for (const [key, value] of left) {
789 if (right.get(key) !== value) {
790 return false;
791 }
792 }
793 return true;
794 }
795
796 export function createEnvironmentShellSummaryAtom(input: {
797 readonly catalogValueAtom: Atom.Atom<EnvironmentCatalogState>;
798 readonly shellStateValueAtom: (environmentId: EnvironmentId) => Atom.Atom<EnvironmentShellState>;
799 }) {
800 let previousSummary = EMPTY_ENVIRONMENT_SHELL_SUMMARY;
801 return Atom.make((get) => {
802 let hasSnapshot = false;
803 let hasSynchronizingShell = false;
804 let firstError: string | null = null;
805
806 for (const environmentId of enabledEnvironmentIds(get(input.catalogValueAtom))) {
807 const state = get(input.shellStateValueAtom(environmentId));
808 hasSynchronizingShell ||= state.status === "synchronizing";
809 if (firstError === null) {
810 firstError = Option.getOrNull(state.error);
811 }
812 if (Option.isNone(state.snapshot)) {
813 continue;
814 }
815 hasSnapshot = true;
816 }
817
818 const next: EnvironmentShellSummary = {
819 hasSnapshot,
820 hasSynchronizingShell,
821 firstError,
822 };
823 if (shellSummariesEqual(previousSummary, next)) {
824 return previousSummary;
825 }
826 previousSummary = next;
827 return previousSummary;
828 }).pipe(Atom.withLabel("environment-shell-summary"));
829 }
830
831 export function createEnvironmentServerConfigsAtom(input: {
832 readonly catalogValueAtom: Atom.Atom<EnvironmentCatalogState>;
833 readonly serverConfigValueAtom: (environmentId: EnvironmentId) => Atom.Atom<ServerConfig | null>;
834 }) {
835 let previousServerConfigs = EMPTY_SERVER_CONFIGS;
836 return Atom.make((get) => {
837 const next = new Map<EnvironmentId, ServerConfig>();
838 for (const environmentId of enabledEnvironmentIds(get(input.catalogValueAtom))) {
839 const config = get(input.serverConfigValueAtom(environmentId));
840 if (config !== null) {
841 next.set(environmentId, config);
842 }
843 }
844 if (mapsEqual(previousServerConfigs, next)) {
845 return previousServerConfigs;
846 }
847 previousServerConfigs = next;
848 return previousServerConfigs;
849 }).pipe(Atom.withLabel("environment-server-configs"));
850 }
851
852 /**
853 * Every shell push replaces the state, so views that gate on load progress read these
854 * primitives instead and stay put while threads stream.
855 */
856 function createEnvironmentShellProgressAtoms(
857 stateValueAtom: (environmentId: EnvironmentId) => Atom.Atom<EnvironmentShellState>,
858 ) {
859 return {
860 statusAtom: Atom.family((environmentId: EnvironmentId) =>
861 Atom.make((get) => get(stateValueAtom(environmentId)).status).pipe(
862 Atom.withLabel(`environment-shell-status:${environmentId}`),
863 ),
864 ),
865 hasSnapshotAtom: Atom.family((environmentId: EnvironmentId) =>
866 Atom.make((get) => Option.isSome(get(stateValueAtom(environmentId)).snapshot)).pipe(
867 Atom.withLabel(`environment-shell-has-snapshot:${environmentId}`),
868 ),
869 ),
870 };
871 }
872
873 export function createEnvironmentShellAtoms<R, E>(
874 runtime: Atom.AtomRuntime<
875 | EnvironmentRegistry.EnvironmentRegistry
876 | Persistence.EnvironmentCacheStore
877 | ShellSnapshotLoader.ShellSnapshotLoader
878 | R,
879 E
880 >,
881 ) {
882 const stateAtom = Atom.family((environmentId: EnvironmentId) =>
883 runtime.atom(shellStateChanges(environmentId), {
884 initialValue: EMPTY_SHELL_STATE,
885 }),
886 );
887
888 const stateValueAtom = Atom.family((environmentId: EnvironmentId) =>
889 Atom.make((get) =>
890 AsyncResult.getOrElse(get(stateAtom(environmentId)), () => EMPTY_SHELL_STATE),
891 ).pipe(Atom.withLabel(`environment-shell-state-value:${environmentId}`)),
892 );
893
894 const { statusAtom, hasSnapshotAtom } = createEnvironmentShellProgressAtoms(stateValueAtom);
895
896 return {
897 stateAtom,
898 stateValueAtom,
899 statusAtom,
900 hasSnapshotAtom,
901 };
902 }
903
904 export * from "./models.ts";
905 export * from "./shellCommands.ts";
906 export * from "./shellReducer.ts";
907 export * from "./shellPullRequests.ts";
908 export * as ShellSnapshotLoader from "./shellSnapshotHttp.ts";
909 export * from "./snapshots.ts";
910 export * from "./steadyShells.ts";