The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { mFallback, msg } from "gt-i18n"; |
| 2 | import type { MFunctionType } from "gt-i18n/types"; |
| 3 | import { |
| 4 | ORCHESTRATION_V2_WS_METHODS, |
| 5 | type EnvironmentId, |
| 6 | type ExecutionEnvironmentDescriptor, |
| 7 | type OrchestrationV2ShellSnapshot, |
| 8 | type OrchestrationV2ShellStreamItem, |
| 9 | type OrchestrationV2ThreadShell, |
| 10 | type ServerConfig, |
| 11 | type ThreadId, |
| 12 | type ThreadPullRequestLink, |
| 13 | } from "@t3tools/contracts"; |
| 14 | import * as Cause from "effect/Cause"; |
| 15 | import * as Clock from "effect/Clock"; |
| 16 | import * as Effect from "effect/Effect"; |
| 17 | import * as Exit from "effect/Exit"; |
| 18 | import * as Fiber from "effect/Fiber"; |
| 19 | import * as Option from "effect/Option"; |
| 20 | import * as Queue from "effect/Queue"; |
| 21 | import * as Ref from "effect/Ref"; |
| 22 | import * as Semaphore from "effect/Semaphore"; |
| 23 | import * as Stream from "effect/Stream"; |
| 24 | import * as SubscriptionRef from "effect/SubscriptionRef"; |
| 25 | import * as SynchronizedRef from "effect/SynchronizedRef"; |
| 26 | import { AsyncResult, Atom } from "effect/reactivity"; |
| 27 | |
| 28 | import * as EnvironmentRegistry from "../connection/registry.ts"; |
| 29 | import { connectionProjectionPhase, type PreparedConnection } from "../connection/model.ts"; |
| 30 | import * as EnvironmentSupervisor from "../connection/supervisor.ts"; |
| 31 | import * as ConnectionWakeups from "../connection/wakeups.ts"; |
| 32 | import { safeErrorLogAttributes } from "../errors/safeLog.ts"; |
| 33 | import * as Persistence from "../platform/persistence.ts"; |
| 34 | import { runCachePersistence } from "./cachePersistence.ts"; |
| 35 | import { subscribeDynamicWithSession, subscriptionRetryDelay } from "../rpc/client.ts"; |
| 36 | import type { RpcSession } from "../rpc/session.ts"; |
| 37 | import * as ShellSnapshotLoader from "./shellSnapshotHttp.ts"; |
| 38 | import type { DeferredShellSnapshot } from "./shellPullRequests.ts"; |
| 39 | import { |
| 40 | applyShellStreamEvents, |
| 41 | type ShellStreamEvent, |
| 42 | mergeShellSnapshotProjects, |
| 43 | reuseUnchangedThreadShells, |
| 44 | sameThreadPullRequests, |
| 45 | } from "./shellReducer.ts"; |
| 46 | import { type EnvironmentCatalogState, enabledEnvironmentIds } from "./connections.ts"; |
| 47 | import { followStreamInEnvironment } from "./runtime.ts"; |
| 48 | |
| 49 | export type EnvironmentShellStatus = "empty" | "cached" | "synchronizing" | "live"; |
| 50 | |
| 51 | export interface EnvironmentShellState { |
| 52 | readonly snapshot: Option.Option<OrchestrationV2ShellSnapshot>; |
| 53 | readonly status: EnvironmentShellStatus; |
| 54 | readonly error: Option.Option<string>; |
| 55 | /** |
| 56 | * Changes with each authoritative snapshot (not enrichment refreshes); 0 until the first one. |
| 57 | * Thread families read for an older generation may miss changes from before the latest snapshot, |
| 58 | * so they are read again. |
| 59 | */ |
| 60 | readonly generation: number; |
| 61 | /** Sequences of the thread removals applied since the latest authoritative snapshot. */ |
| 62 | readonly removedAt: ReadonlyMap<ThreadId, number>; |
| 63 | } |
| 64 | |
| 65 | const EMPTY_REMOVALS: ReadonlyMap<ThreadId, number> = new Map(); |
| 66 | // Shared by every shell state, so a rebuilt one never reuses a generation another already used. |
| 67 | let lastShellGeneration = 0; |
| 68 | |
| 69 | export const EMPTY_SHELL_STATE: EnvironmentShellState = { |
| 70 | snapshot: Option.none(), |
| 71 | status: "empty", |
| 72 | error: Option.none(), |
| 73 | generation: 0, |
| 74 | removedAt: EMPTY_REMOVALS, |
| 75 | }; |
| 76 | |
| 77 | function shellStatusForSnapshot( |
| 78 | snapshot: Option.Option<OrchestrationV2ShellSnapshot>, |
| 79 | ): EnvironmentShellStatus { |
| 80 | return Option.isSome(snapshot) ? "cached" : "empty"; |
| 81 | } |
| 82 | |
| 83 | function withoutDeferredPullRequests(value: DeferredShellSnapshot): OrchestrationV2ShellSnapshot { |
| 84 | if (value.loadPullRequests === undefined) return value; |
| 85 | const { loadPullRequests: _deferred, ...snapshot } = value; |
| 86 | return snapshot; |
| 87 | } |
| 88 | |
| 89 | const SHELL_SYNCHRONIZATION_ERROR_MESSAGE = msg("Could not synchronize environment data."); |
| 90 | |
| 91 | /** @public The shell's sync error in the active language; other messages stay as they are. */ |
| 92 | export function shellErrorText(error: string, resolve: MFunctionType = mFallback): string { |
| 93 | return error === SHELL_SYNCHRONIZATION_ERROR_MESSAGE ? resolve(error) : error; |
| 94 | } |
| 95 | |
| 96 | /** |
| 97 | * Replay does not refresh the older-settled cut, rows that left the newest window, or |
| 98 | * projections the server rebuilt, so drift grows with the age of the last server snapshot. |
| 99 | */ |
| 100 | const CACHE_RESUME_MAX_AGE_MILLIS = 12 * 60 * 60 * 1000; |
| 101 | |
| 102 | // How long a ready socket waits for the early HTTP answer before it resumes held rows itself. |
| 103 | // The supervisor's quick probe allows a live host the same time. A proxy that drops `/api` |
| 104 | // without an answer would otherwise hold each connect for the full HTTP timeout. |
| 105 | const EARLY_LOAD_GRACE = "3 seconds"; |
| 106 | |
| 107 | type ShellOriginIdentity = Omit<Persistence.ShellCacheOrigin, "snapshotAt">; |
| 108 | |
| 109 | interface ShellSubscription { |
| 110 | readonly session: RpcSession; |
| 111 | /** What a snapshot from this subscription descends from; null when the server did not say. */ |
| 112 | readonly identity: ShellOriginIdentity | null; |
| 113 | } |
| 114 | |
| 115 | function shellIdentity( |
| 116 | environment: Pick<ExecutionEnvironmentDescriptor, "serverVersion" | "databaseMigration">, |
| 117 | flags: { readonly shellThreadFamilies?: boolean; readonly shellOlderSettled?: boolean }, |
| 118 | ): ShellOriginIdentity { |
| 119 | const omitIdleSubagents = flags.shellThreadFamilies === true; |
| 120 | return { |
| 121 | serverVersion: environment.serverVersion, |
| 122 | // Migrations rewrite projections without events, so a migrated server reloads. |
| 123 | databaseMigration: environment.databaseMigration ?? null, |
| 124 | omitIdleSubagents, |
| 125 | omitOlderSettled: flags.shellOlderSettled === true && omitIdleSubagents, |
| 126 | }; |
| 127 | } |
| 128 | |
| 129 | function sameConnection(left: PreparedConnection, right: PreparedConnection): boolean { |
| 130 | const keys = Object.keys({ ...left, ...right }) as Array<keyof PreparedConnection>; |
| 131 | return keys.every((key) => key === "httpAuthorization" || left[key] === right[key]); |
| 132 | } |
| 133 | |
| 134 | /** Null for a server whose descriptor does not name its shell flags; only its config does. */ |
| 135 | function descriptorShellIdentity( |
| 136 | descriptor: ExecutionEnvironmentDescriptor | undefined, |
| 137 | ): ShellOriginIdentity | null { |
| 138 | if (descriptor?.capabilities.shellThreadFamilies === undefined) return null; |
| 139 | return shellIdentity(descriptor, descriptor.capabilities); |
| 140 | } |
| 141 | |
| 142 | function sameShellIdentity(left: ShellOriginIdentity, right: ShellOriginIdentity): boolean { |
| 143 | return ( |
| 144 | left.serverVersion === right.serverVersion && |
| 145 | left.databaseMigration === right.databaseMigration && |
| 146 | left.omitIdleSubagents === right.omitIdleSubagents && |
| 147 | left.omitOlderSettled === right.omitOlderSettled |
| 148 | ); |
| 149 | } |
| 150 | |
| 151 | function canResumeFrom( |
| 152 | origin: Persistence.ShellCacheOrigin | undefined, |
| 153 | identity: ShellOriginIdentity | null, |
| 154 | now: number, |
| 155 | ): boolean { |
| 156 | if (origin === undefined || identity === null) return false; |
| 157 | const age = now - origin.snapshotAt; |
| 158 | return sameShellIdentity(origin, identity) && age >= 0 && age < CACHE_RESUME_MAX_AGE_MILLIS; |
| 159 | } |
| 160 | |
| 161 | type HttpShellOutcome = |
| 162 | | { readonly kind: "snapshot"; readonly projectIdentities: string | undefined } |
| 163 | /** The server can replay from the rows the client holds. */ |
| 164 | | { readonly kind: "resumable" } |
| 165 | | { readonly kind: "failed" }; |
| 166 | |
| 167 | interface EarlyShellLoad { |
| 168 | readonly prepared: PreparedConnection; |
| 169 | readonly identity: ShellOriginIdentity; |
| 170 | readonly fiber: Fiber.Fiber<HttpShellOutcome>; |
| 171 | // One subscription uses a load; a later one on the same connection decides again. |
| 172 | readonly claimed: boolean; |
| 173 | } |
| 174 | |
| 175 | interface LiveShell { |
| 176 | readonly snapshot: OrchestrationV2ShellSnapshot; |
| 177 | // Read together with the rows, so a save never pairs them with another snapshot's origin. |
| 178 | readonly origin: Persistence.ShellCacheOrigin | undefined; |
| 179 | } |
| 180 | |
| 181 | export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")(function* () { |
| 182 | const supervisor = yield* EnvironmentSupervisor.EnvironmentSupervisor; |
| 183 | const cache = yield* Persistence.EnvironmentCacheStore; |
| 184 | const snapshotLoader = yield* ShellSnapshotLoader.ShellSnapshotLoader; |
| 185 | const wakeups = yield* Effect.serviceOption(ConnectionWakeups.ConnectionWakeups); |
| 186 | const environmentId = supervisor.target.environmentId; |
| 187 | const cached = yield* cache.loadShell(environmentId).pipe( |
| 188 | Effect.catch((error) => |
| 189 | Effect.logWarning("Could not load cached environment shell.").pipe( |
| 190 | Effect.annotateLogs({ |
| 191 | environmentId, |
| 192 | ...safeErrorLogAttributes(error), |
| 193 | }), |
| 194 | Effect.as(Option.none<Persistence.CachedShellSnapshot>()), |
| 195 | ), |
| 196 | ), |
| 197 | ); |
| 198 | const cachedOrigin = Option.isSome(cached) ? cached.value.origin : undefined; |
| 199 | const cachedSnapshot = Option.map(cached, ({ origin: _origin, ...rows }) => |
| 200 | withoutDeferredPullRequests(rows), |
| 201 | ); |
| 202 | const state = yield* SubscriptionRef.make<EnvironmentShellState>({ |
| 203 | ...EMPTY_SHELL_STATE, |
| 204 | snapshot: cachedSnapshot, |
| 205 | status: shellStatusForSnapshot(cachedSnapshot), |
| 206 | }); |
| 207 | const awaitingCompletion = yield* Ref.make(false); |
| 208 | const activeSubscription = yield* Ref.make<ShellSubscription | null>(null); |
| 209 | // The server snapshot the in-memory rows descend from, directly or through the cache. |
| 210 | // Only a server snapshot renews it; `snapshot-required` drops it. |
| 211 | const origin = yield* Ref.make(cachedOrigin); |
| 212 | const reloadRequests = yield* Queue.unbounded<void>(); |
| 213 | let defectRetries = 0; |
| 214 | const latestLive = yield* Ref.make<Option.Option<LiveShell>>(Option.none()); |
| 215 | const persistence = yield* Queue.sliding<OrchestrationV2ShellSnapshot>(1); |
| 216 | |
| 217 | const persistenceLock = yield* Semaphore.make(1); |
| 218 | let lastPersisted: LiveShell | undefined; |
| 219 | // Rows still waiting for their deferred pull request links, which must not be saved: |
| 220 | // the cache would keep them without links until the next server refresh. |
| 221 | let pendingLinkRows: ReadonlySet<OrchestrationV2ThreadShell> = new Set(); |
| 222 | const persistLatest = Effect.fn("EnvironmentShellState.persistLatest")(function* ( |
| 223 | flush: boolean, |
| 224 | ) { |
| 225 | while (true) { |
| 226 | const latest = yield* Ref.get(latestLive); |
| 227 | if (Option.isNone(latest) || latest.value === lastPersisted) return; |
| 228 | const { snapshot, origin } = latest.value; |
| 229 | if (snapshot.threads.some((thread) => pendingLinkRows.has(thread))) return; |
| 230 | if ( |
| 231 | lastPersisted !== undefined && |
| 232 | origin === lastPersisted.origin && |
| 233 | snapshot.snapshotSequence >= lastPersisted.snapshot.snapshotSequence && |
| 234 | snapshot.schemaVersion === lastPersisted.snapshot.schemaVersion && |
| 235 | snapshot.projects === lastPersisted.snapshot.projects && |
| 236 | snapshot.threads === lastPersisted.snapshot.threads && |
| 237 | snapshot.archivedThreads === lastPersisted.snapshot.archivedThreads |
| 238 | ) { |
| 239 | // Keep the live cursor without rewriting the rows. Remember it here too so a |
| 240 | // later authoritative reset to a lower sequence still reaches the cache. |
| 241 | lastPersisted = latest.value; |
| 242 | return; |
| 243 | } |
| 244 | const saved = yield* cache.saveShell(environmentId, snapshot, origin).pipe( |
| 245 | Effect.as(true), |
| 246 | Effect.catch((error) => |
| 247 | Effect.logWarning("Could not persist environment shell cache.").pipe( |
| 248 | Effect.annotateLogs({ environmentId, ...safeErrorLogAttributes(error) }), |
| 249 | Effect.as(false), |
| 250 | ), |
| 251 | ), |
| 252 | ); |
| 253 | if (!saved) return; |
| 254 | lastPersisted = latest.value; |
| 255 | if (!flush) return; |
| 256 | } |
| 257 | }, persistenceLock.withPermit); |
| 258 | const persist = () => persistLatest(false); |
| 259 | const flushLiveShellSnapshot = persistLatest(true); |
| 260 | |
| 261 | // Register before scoped worker fibers so reverse finalizer order interrupts |
| 262 | // those fibers first and this flush sees a stable latestLiveSnapshot. |
| 263 | yield* Effect.addFinalizer(() => flushLiveShellSnapshot); |
| 264 | |
| 265 | yield* runCachePersistence(persistence, persist).pipe(Effect.forkScoped); |
| 266 | |
| 267 | const setDisconnected = Ref.set(awaitingCompletion, false).pipe( |
| 268 | Effect.andThen( |
| 269 | SubscriptionRef.update(state, (current) => ({ |
| 270 | ...current, |
| 271 | status: shellStatusForSnapshot(current.snapshot), |
| 272 | })), |
| 273 | ), |
| 274 | Effect.andThen(flushLiveShellSnapshot.pipe(Effect.forkScoped)), |
| 275 | Effect.asVoid, |
| 276 | ); |
| 277 | const setSynchronizing = SubscriptionRef.update(state, (current) => ({ |
| 278 | ...current, |
| 279 | status: "synchronizing" as const, |
| 280 | error: Option.none(), |
| 281 | })); |
| 282 | const setReady = SubscriptionRef.update(state, (current) => |
| 283 | current.status === "live" |
| 284 | ? current |
| 285 | : { |
| 286 | ...current, |
| 287 | status: "synchronizing" as const, |
| 288 | error: Option.none(), |
| 289 | }, |
| 290 | ); |
| 291 | const setStreamError = (error: unknown) => |
| 292 | Ref.set(awaitingCompletion, false).pipe( |
| 293 | Effect.andThen(Effect.logWarning("Could not synchronize the environment shell.")), |
| 294 | Effect.annotateLogs({ |
| 295 | environmentId, |
| 296 | ...safeErrorLogAttributes(error), |
| 297 | }), |
| 298 | Effect.andThen( |
| 299 | SubscriptionRef.update(state, (current) => ({ |
| 300 | ...current, |
| 301 | status: shellStatusForSnapshot(current.snapshot), |
| 302 | error: Option.some(SHELL_SYNCHRONIZATION_ERROR_MESSAGE), |
| 303 | })), |
| 304 | ), |
| 305 | ); |
| 306 | |
| 307 | // Server items and deferred link fills both read the snapshot and write it back; one at a |
| 308 | // time, so neither can overwrite what the other wrote in between. |
| 309 | const snapshotWriteLock = yield* Semaphore.make(1); |
| 310 | |
| 311 | const applyLocked = Effect.fnUntraced(function* ( |
| 312 | items: ReadonlyArray<OrchestrationV2ShellStreamItem>, |
| 313 | /** What an authoritative snapshot among the items descends from. */ |
| 314 | identity: ShellOriginIdentity | null, |
| 315 | /** Set for an HTTP snapshot: the rows that still wait for their pull request links. */ |
| 316 | pendingLinkIds?: ReadonlySet<ThreadId>, |
| 317 | ) { |
| 318 | const initial = yield* SubscriptionRef.get(state); |
| 319 | // No socket has confirmed an HTTP snapshot, so it never makes the shell live. |
| 320 | const fromHttp = pendingLinkIds !== undefined; |
| 321 | const disconnected = initial.status === "empty" || initial.status === "cached"; |
| 322 | let waiting = yield* Ref.get(awaitingCompletion); |
| 323 | let next = initial; |
| 324 | let receivedAuthoritativeSnapshot = false; |
| 325 | let snapshotRequired = false; |
| 326 | let deltas: Array<ShellStreamEvent> = []; |
| 327 | const flushDeltas = () => { |
| 328 | if (deltas.length > 0 && Option.isSome(next.snapshot)) { |
| 329 | let sequence = next.snapshot.value.snapshotSequence; |
| 330 | let removedAt: Map<ThreadId, number> | undefined; |
| 331 | for (const delta of deltas) { |
| 332 | if (delta.sequence <= sequence) continue; |
| 333 | switch (delta.kind) { |
| 334 | case "thread.removed": |
| 335 | removedAt ??= new Map(next.removedAt); |
| 336 | removedAt.set(delta.threadId, delta.sequence); |
| 337 | break; |
| 338 | case "thread.updated": |
| 339 | case "project.updated": |
| 340 | case "project.removed": |
| 341 | break; |
| 342 | default: |
| 343 | continue; |
| 344 | } |
| 345 | sequence = delta.sequence; |
| 346 | } |
| 347 | next = { |
| 348 | ...next, |
| 349 | removedAt: removedAt ?? next.removedAt, |
| 350 | snapshot: Option.some(applyShellStreamEvents(next.snapshot.value, deltas)), |
| 351 | status: waiting ? "synchronizing" : "live", |
| 352 | error: Option.none(), |
| 353 | }; |
| 354 | } |
| 355 | deltas = []; |
| 356 | }; |
| 357 | for (const item of items) { |
| 358 | if (item.kind === "snapshot-required") { |
| 359 | snapshotRequired = true; |
| 360 | continue; |
| 361 | } |
| 362 | if (item.kind !== "snapshot" && item.kind !== "synchronized") { |
| 363 | deltas.push(item); |
| 364 | continue; |
| 365 | } |
| 366 | // Snapshots can reset the cursor; completion markers change the sync status. |
| 367 | flushDeltas(); |
| 368 | if (item.kind === "synchronized") { |
| 369 | waiting = false; |
| 370 | if (Option.isSome(next.snapshot)) { |
| 371 | next = { |
| 372 | ...next, |
| 373 | status: "live", |
| 374 | error: Option.none(), |
| 375 | // A cache resume that caught up is as current as a snapshot for generation readers. |
| 376 | generation: next.generation === 0 ? ++lastShellGeneration : next.generation, |
| 377 | }; |
| 378 | } |
| 379 | continue; |
| 380 | } |
| 381 | const authoritative = item.resolvedRepositoryIdentityRoots === undefined; |
| 382 | const nextSnapshot = mergeShellSnapshotProjects( |
| 383 | Option.getOrNull(next.snapshot), |
| 384 | item.snapshot, |
| 385 | item.resolvedRepositoryIdentityRoots === undefined |
| 386 | ? undefined |
| 387 | : { resolvedRepositoryIdentityRoots: item.resolvedRepositoryIdentityRoots }, |
| 388 | ); |
| 389 | if (authoritative) receivedAuthoritativeSnapshot = true; |
| 390 | next = { |
| 391 | snapshot: Option.some(nextSnapshot), |
| 392 | status: |
| 393 | fromHttp && disconnected ? "cached" : waiting || fromHttp ? "synchronizing" : "live", |
| 394 | error: Option.none(), |
| 395 | generation: authoritative ? ++lastShellGeneration : next.generation, |
| 396 | removedAt: authoritative ? EMPTY_REMOVALS : next.removedAt, |
| 397 | }; |
| 398 | } |
| 399 | flushDeltas(); |
| 400 | yield* Ref.set(awaitingCompletion, waiting); |
| 401 | let nextOrigin = yield* Ref.get(origin); |
| 402 | if (snapshotRequired) { |
| 403 | // The rows stay on screen but reload before the next resume; the cache keeps the old |
| 404 | // origin until that reload saves, which costs at most one more refusal. |
| 405 | nextOrigin = undefined; |
| 406 | yield* Ref.set(origin, nextOrigin); |
| 407 | yield* Queue.offer(reloadRequests, undefined); |
| 408 | } else if (receivedAuthoritativeSnapshot) { |
| 409 | nextOrigin = |
| 410 | identity === null ? undefined : { ...identity, snapshotAt: yield* Clock.currentTimeMillis }; |
| 411 | yield* Ref.set(origin, nextOrigin); |
| 412 | } |
| 413 | if (next === initial) return; |
| 414 | if (pendingLinkIds !== undefined && Option.isSome(next.snapshot)) { |
| 415 | pendingLinkRows = new Set( |
| 416 | next.snapshot.value.threads.filter((thread) => pendingLinkIds.has(thread.id)), |
| 417 | ); |
| 418 | } |
| 419 | if (Option.isSome(next.snapshot)) { |
| 420 | yield* Ref.set( |
| 421 | latestLive, |
| 422 | Option.some({ snapshot: next.snapshot.value, origin: nextOrigin }), |
| 423 | ); |
| 424 | } |
| 425 | yield* SubscriptionRef.set(state, next); |
| 426 | if (next.snapshot !== initial.snapshot && Option.isSome(next.snapshot)) { |
| 427 | yield* Queue.offer(persistence, next.snapshot.value); |
| 428 | } |
| 429 | }); |
| 430 | |
| 431 | const applyItems = Effect.fn("EnvironmentShellState.applyItems")(function* ( |
| 432 | entries: ReadonlyArray<readonly [RpcSession, OrchestrationV2ShellStreamItem]>, |
| 433 | ) { |
| 434 | // Read under the lock: items buffered across a session change must not land on the |
| 435 | // rows the next session loaded. |
| 436 | const subscription = yield* Ref.get(activeSubscription); |
| 437 | yield* applyLocked( |
| 438 | entries.flatMap(([session, item]) => (session === subscription?.session ? [item] : [])), |
| 439 | subscription?.identity ?? null, |
| 440 | ); |
| 441 | }, snapshotWriteLock.withPermit); |
| 442 | |
| 443 | const stateScope = yield* Effect.scope; |
| 444 | // Each full snapshot supersedes the previous one's fill; links from an older snapshot must |
| 445 | // not land on rows a newer snapshot reused. |
| 446 | |
| 447 | let activeFill: Fiber.Fiber<void> | undefined; |
| 448 | const cancelActiveFill = Effect.suspend(() => { |
| 449 | const fill = activeFill; |
| 450 | activeFill = undefined; |
| 451 | return fill === undefined ? Effect.void : Fiber.interrupt(fill); |
| 452 | }); |
| 453 | const fillDeferredPullRequests = Effect.fnUntraced(function* ( |
| 454 | source: DeferredShellSnapshot, |
| 455 | applied: OrchestrationV2ShellSnapshot, |
| 456 | ) { |
| 457 | yield* cancelActiveFill; |
| 458 | if (source.loadPullRequests === undefined) return; |
| 459 | const arrivedRows = new Set(applied.threads); |
| 460 | const arrivedIds = new Set(source.threads.map((thread) => thread.id)); |
| 461 | pendingLinkRows = new Set(applied.threads.filter((thread) => arrivedIds.has(thread.id))); |
| 462 | activeFill = yield* source.loadPullRequests.pipe( |
| 463 | Effect.flatMap((linksByThreadId) => |
| 464 | writeFilledLinks(linksByThreadId, arrivedRows, arrivedIds), |
| 465 | ), |
| 466 | Effect.forkIn(stateScope), |
| 467 | ); |
| 468 | }); |
| 469 | const writeFilledLinks = Effect.fnUntraced( |
| 470 | function* ( |
| 471 | linksByThreadId: ReadonlyMap<ThreadId, ReadonlyArray<ThreadPullRequestLink>>, |
| 472 | arrivedRows: ReadonlySet<OrchestrationV2ThreadShell>, |
| 473 | arrivedIds: ReadonlySet<ThreadId>, |
| 474 | ) { |
| 475 | const filled = yield* SubscriptionRef.modify( |
| 476 | state, |
| 477 | (current): [Option.Option<OrchestrationV2ShellSnapshot>, EnvironmentShellState] => { |
| 478 | if (Option.isNone(current.snapshot)) return [Option.none(), current]; |
| 479 | const snapshot = current.snapshot.value; |
| 480 | let changed = false; |
| 481 | const threads = snapshot.threads.map((thread) => { |
| 482 | if (!arrivedRows.has(thread) || !arrivedIds.has(thread.id)) return thread; |
| 483 | const links = linksByThreadId.get(thread.id); |
| 484 | if (sameThreadPullRequests(thread.pullRequests, links)) return thread; |
| 485 | changed = true; |
| 486 | if (links === undefined) { |
| 487 | const { pullRequests: _previous, ...row } = thread; |
| 488 | return row; |
| 489 | } |
| 490 | return { ...thread, pullRequests: links }; |
| 491 | }); |
| 492 | if (!changed) return [Option.none(), current]; |
| 493 | const next = { ...snapshot, threads }; |
| 494 | return [Option.some(next), { ...current, snapshot: Option.some(next) }]; |
| 495 | }, |
| 496 | ); |
| 497 | pendingLinkRows = new Set(); |
| 498 | const live = yield* Ref.get(latestLive); |
| 499 | if (Option.isNone(live)) return; |
| 500 | const latest = Option.isSome(filled) ? filled.value : live.value.snapshot; |
| 501 | yield* Ref.set(latestLive, Option.some({ snapshot: latest, origin: live.value.origin })); |
| 502 | yield* Queue.offer(persistence, latest); |
| 503 | }, |
| 504 | (effect) => snapshotWriteLock.withPermit(effect), |
| 505 | ); |
| 506 | if (Option.isSome(cached) && Option.isSome(cachedSnapshot)) { |
| 507 | yield* fillDeferredPullRequests(cached.value, cachedSnapshot.value); |
| 508 | } |
| 509 | |
| 510 | const applyHttpSnapshot = Effect.fnUntraced(function* ( |
| 511 | loaded: ShellSnapshotLoader.ShellHttpSnapshot, |
| 512 | identity: ShellOriginIdentity | null, |
| 513 | ) { |
| 514 | const { projectIdentities: _identities, ...deferred } = loaded; |
| 515 | yield* cancelActiveFill; |
| 516 | const previous = yield* SubscriptionRef.get(state); |
| 517 | const snapshot = reuseUnchangedThreadShells( |
| 518 | Option.getOrNull(previous.snapshot), |
| 519 | withoutDeferredPullRequests(deferred), |
| 520 | ); |
| 521 | const pendingLinkIds = new Set( |
| 522 | deferred.loadPullRequests === undefined ? [] : deferred.threads.map((thread) => thread.id), |
| 523 | ); |
| 524 | yield* snapshotWriteLock.withPermit( |
| 525 | applyLocked([{ kind: "snapshot", snapshot }], identity, pendingLinkIds), |
| 526 | ); |
| 527 | const current = yield* SubscriptionRef.get(state); |
| 528 | if (Option.isSome(current.snapshot)) { |
| 529 | yield* fillDeferredPullRequests(deferred, current.snapshot.value); |
| 530 | } |
| 531 | }, Effect.uninterruptible); |
| 532 | |
| 533 | /** |
| 534 | * One request to the HTTP shell route. Held rows that may resume are named, so the server |
| 535 | * answers that they still can, in a few bytes, or sends the snapshot that replaces them. |
| 536 | */ |
| 537 | const loadOverHttp = Effect.fn("EnvironmentShellState.loadOverHttp")(function* ( |
| 538 | prepared: PreparedConnection, |
| 539 | identity: ShellOriginIdentity | null, |
| 540 | ) { |
| 541 | const held = yield* SubscriptionRef.get(state); |
| 542 | const resumeFrom = |
| 543 | Option.isSome(held.snapshot) && |
| 544 | canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis) |
| 545 | ? held.snapshot.value.snapshotSequence |
| 546 | : undefined; |
| 547 | // HTTP and the socket must agree, or omitted children would return through the socket. |
| 548 | const answer = yield* snapshotLoader.load(prepared, { |
| 549 | omitIdleSubagents: identity?.omitIdleSubagents === true, |
| 550 | omitOlderSettled: identity?.omitOlderSettled === true, |
| 551 | ...(resumeFrom === undefined ? {} : { resumeFrom }), |
| 552 | }); |
| 553 | if (Option.isNone(answer)) return { kind: "failed" } satisfies HttpShellOutcome; |
| 554 | if ("resumable" in answer.value) return { kind: "resumable" } satisfies HttpShellOutcome; |
| 555 | yield* applyHttpSnapshot(answer.value, identity); |
| 556 | return { |
| 557 | kind: "snapshot", |
| 558 | projectIdentities: answer.value.projectIdentities, |
| 559 | } satisfies HttpShellOutcome; |
| 560 | }); |
| 561 | |
| 562 | // A server whose descriptor names its shell flags is asked for the shell as soon as the |
| 563 | // connection is prepared, while the socket opens and the config downloads. The subscription |
| 564 | // claims that load, and reloads when the live config disagrees with the descriptor. |
| 565 | const earlyLoad = yield* SynchronizedRef.make<EarlyShellLoad | null>(null); |
| 566 | const useEarlyLoad = (prepared: PreparedConnection | null, claim: boolean) => |
| 567 | SynchronizedRef.modifyEffect( |
| 568 | earlyLoad, |
| 569 | Effect.fnUntraced(function* (current) { |
| 570 | // A rotated credential replaces the prepared connection of a session that stays up. |
| 571 | if (current !== null && prepared !== null && sameConnection(current.prepared, prepared)) { |
| 572 | const next = { ...current, prepared, claimed: current.claimed || claim }; |
| 573 | // A claimed load that still runs is handed out again. A second request could |
| 574 | // answer first, and this one would then replace its rows under a passed cursor. |
| 575 | const done = current.fiber.pollUnsafe() !== undefined; |
| 576 | return [current.claimed && done ? null : current, next] as const; |
| 577 | } |
| 578 | if (current !== null) yield* Fiber.interrupt(current.fiber); |
| 579 | const identity = descriptorShellIdentity(prepared?.descriptor); |
| 580 | if (prepared === null || identity === null) return [null, null] as const; |
| 581 | // Items a closed session still has in flight must not land on the rows this loads. |
| 582 | yield* Ref.set(activeSubscription, null); |
| 583 | const fiber = yield* loadOverHttp(prepared, identity).pipe(Effect.forkIn(stateScope)); |
| 584 | const next: EarlyShellLoad = { prepared, identity, fiber, claimed: claim }; |
| 585 | return [next, next] as const; |
| 586 | }), |
| 587 | ).pipe( |
| 588 | // Covers the store too. A cancel that lands during the wait above would otherwise take |
| 589 | // effect before the store, and the load just started would run on with no record. |
| 590 | Effect.uninterruptible, |
| 591 | ); |
| 592 | /** |
| 593 | * The subscription waits for the early load, so its answer is applied before the socket |
| 594 | * moves the rows on. Held rows that can resume wait a limited time only: after that the |
| 595 | * load is ended, so a late answer applies nothing, and the socket resumes them. |
| 596 | */ |
| 597 | const awaitEarlyLoad = Effect.fnUntraced(function* ( |
| 598 | early: EarlyShellLoad, |
| 599 | identity: ShellOriginIdentity | null, |
| 600 | ) { |
| 601 | const resumable = |
| 602 | Option.isSome((yield* SubscriptionRef.get(state)).snapshot) && |
| 603 | canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis); |
| 604 | if (!resumable) return yield* Fiber.await(early.fiber); |
| 605 | const answered = yield* Fiber.await(early.fiber).pipe(Effect.timeoutOption(EARLY_LOAD_GRACE)); |
| 606 | if (Option.isSome(answered)) return answered.value; |
| 607 | // Returns after an answer that is being applied has finished, so the cursor below is read |
| 608 | // from rows that no longer change under it. |
| 609 | yield* Fiber.interrupt(early.fiber); |
| 610 | return yield* Fiber.await(early.fiber); |
| 611 | }); |
| 612 | |
| 613 | yield* SubscriptionRef.changes(supervisor.prepared).pipe( |
| 614 | // Reads the connection of this moment: a change handled late must not start a load for a |
| 615 | // connection that ended, which would also drop the items of the session that replaced it. |
| 616 | Stream.runForEach(() => |
| 617 | SubscriptionRef.get(supervisor.prepared).pipe( |
| 618 | Effect.flatMap((prepared) => useEarlyLoad(Option.getOrNull(prepared), false)), |
| 619 | ), |
| 620 | ), |
| 621 | Effect.forkScoped, |
| 622 | ); |
| 623 | |
| 624 | const resubscriptions = Stream.merge( |
| 625 | Stream.fromQueue(reloadRequests), |
| 626 | Option.match(wakeups, { |
| 627 | onNone: () => Stream.never, |
| 628 | onSome: (service) => |
| 629 | service.changes.pipe(Stream.filter(ConnectionWakeups.shouldResubscribeAfterWakeup)), |
| 630 | }), |
| 631 | ); |
| 632 | |
| 633 | yield* setSynchronizing; |
| 634 | yield* Effect.forkScoped( |
| 635 | subscribeDynamicWithSession( |
| 636 | ORCHESTRATION_V2_WS_METHODS.subscribeShell, |
| 637 | Effect.fn("EnvironmentShellState.makeSubscribeInput")(function* (session) { |
| 638 | const identity = yield* session.initialConfig.pipe( |
| 639 | Effect.map((config) => shellIdentity(config.environment, config)), |
| 640 | Effect.orElseSucceed(() => null), |
| 641 | ); |
| 642 | const { omitIdleSubagents, omitOlderSettled } = identity ?? { |
| 643 | omitIdleSubagents: false, |
| 644 | omitOlderSettled: false, |
| 645 | }; |
| 646 | const early = yield* useEarlyLoad( |
| 647 | Option.getOrNull(yield* SubscriptionRef.get(supervisor.prepared)), |
| 648 | true, |
| 649 | ); |
| 650 | const earlyExit = early === null ? null : yield* awaitEarlyLoad(early, identity); |
| 651 | // Rows the descriptor's flags or build loaded are not kept when the live config differs: |
| 652 | // their origin then cannot resume, so the load below replaces them. |
| 653 | let loaded = |
| 654 | early !== null && |
| 655 | earlyExit !== null && |
| 656 | Exit.isSuccess(earlyExit) && |
| 657 | identity !== null && |
| 658 | sameShellIdentity(early.identity, identity) |
| 659 | ? earlyExit.value |
| 660 | : null; |
| 661 | yield* Ref.set(activeSubscription, { session, identity }); |
| 662 | yield* Ref.set(awaitingCompletion, true); |
| 663 | yield* setSynchronizing; |
| 664 | |
| 665 | // Rows from a recent server snapshot with the same omissions and server build resume |
| 666 | // from their cursor, in memory or from the cache. Anything else reloads the HTTP |
| 667 | // snapshot so a valid cursor cannot preserve stale or incomplete data. A `resumable` |
| 668 | // answer is checked again too: the rows can pass their age limit while the socket opens. |
| 669 | const resumingHeldRows = |
| 670 | loaded?.kind !== "snapshot" && |
| 671 | Option.isSome((yield* SubscriptionRef.get(state)).snapshot) && |
| 672 | canResumeFrom(yield* Ref.get(origin), identity, yield* Clock.currentTimeMillis); |
| 673 | if (!resumingHeldRows && loaded === null) { |
| 674 | const prepared = yield* SubscriptionRef.get(supervisor.prepared).pipe( |
| 675 | Effect.flatMap( |
| 676 | Option.match({ |
| 677 | onSome: Effect.succeed, |
| 678 | onNone: () => |
| 679 | SubscriptionRef.changes(supervisor.prepared).pipe( |
| 680 | Stream.filter(Option.isSome), |
| 681 | Stream.map((value) => value.value), |
| 682 | Stream.runHead, |
| 683 | Effect.map(Option.getOrThrow), |
| 684 | ), |
| 685 | }), |
| 686 | ), |
| 687 | ); |
| 688 | loaded = yield* loadOverHttp(prepared, identity); |
| 689 | } |
| 690 | |
| 691 | const current = yield* SubscriptionRef.get(state); |
| 692 | const omission = { |
| 693 | ...(omitIdleSubagents ? { omitIdleSubagents: true as const } : {}), |
| 694 | ...(omitOlderSettled ? { omitOlderSettled: true as const } : {}), |
| 695 | }; |
| 696 | if (Option.isNone(current.snapshot) || !(resumingHeldRows || loaded?.kind === "snapshot")) { |
| 697 | return { ...omission, requestCompletionMarker: true as const }; |
| 698 | } |
| 699 | return { |
| 700 | afterSequence: current.snapshot.value.snapshotSequence, |
| 701 | // Only rows the client already held may be refused; a cursor from the snapshot just |
| 702 | // loaded falls back to the socket snapshot, so a refusal cannot loop. |
| 703 | ...(resumingHeldRows ? { httpSnapshotFallback: true as const } : {}), |
| 704 | // No stream item has touched the snapshot just loaded, so its identities still hold. |
| 705 | ...(loaded?.kind === "snapshot" && loaded.projectIdentities !== undefined |
| 706 | ? { knownProjectIdentities: loaded.projectIdentities } |
| 707 | : {}), |
| 708 | ...omission, |
| 709 | requestCompletionMarker: true as const, |
| 710 | }; |
| 711 | }), |
| 712 | { |
| 713 | // A defect parks the stream, so resubscribe after a backoff rather than wait for a wakeup. |
| 714 | onDefect: (cause) => |
| 715 | setStreamError(Cause.squash(cause)).pipe( |
| 716 | Effect.andThen(Effect.sleep(subscriptionRetryDelay("250 millis", defectRetries++))), |
| 717 | Effect.andThen(Queue.offer(reloadRequests, undefined)), |
| 718 | ), |
| 719 | onExpectedFailure: (cause) => setStreamError(Cause.squash(cause)), |
| 720 | retryExpectedFailureAfter: "250 millis", |
| 721 | resubscribe: resubscriptions, |
| 722 | }, |
| 723 | ).pipe( |
| 724 | Stream.runForEachArray((entries) => |
| 725 | applyItems(entries).pipe( |
| 726 | Effect.andThen( |
| 727 | Effect.sync(() => { |
| 728 | defectRetries = 0; |
| 729 | }), |
| 730 | ), |
| 731 | ), |
| 732 | ), |
| 733 | ), |
| 734 | ); |
| 735 | yield* SubscriptionRef.changes(supervisor.state).pipe( |
| 736 | Stream.runForEach((connectionState) => { |
| 737 | switch (connectionProjectionPhase(connectionState)) { |
| 738 | case "synchronizing": |
| 739 | return setSynchronizing; |
| 740 | case "disconnected": |
| 741 | return setDisconnected; |
| 742 | case "ready": |
| 743 | return setReady; |
| 744 | } |
| 745 | }), |
| 746 | Effect.forkScoped, |
| 747 | ); |
| 748 | |
| 749 | return state; |
| 750 | }); |
| 751 | |
| 752 | function shellStateChanges(environmentId: EnvironmentId) { |
| 753 | return followStreamInEnvironment( |
| 754 | environmentId, |
| 755 | Stream.unwrap(makeEnvironmentShellState().pipe(Effect.map(SubscriptionRef.changes))), |
| 756 | ); |
| 757 | } |
| 758 | |
| 759 | export interface EnvironmentShellSummary { |
| 760 | readonly hasSnapshot: boolean; |
| 761 | readonly hasSynchronizingShell: boolean; |
| 762 | readonly firstError: string | null; |
| 763 | } |
| 764 | |
| 765 | const EMPTY_ENVIRONMENT_SHELL_SUMMARY: EnvironmentShellSummary = Object.freeze({ |
| 766 | hasSnapshot: false, |
| 767 | hasSynchronizingShell: false, |
| 768 | firstError: null, |
| 769 | }); |
| 770 | |
| 771 | const EMPTY_SERVER_CONFIGS: ReadonlyMap<EnvironmentId, ServerConfig> = new Map(); |
| 772 | |
| 773 | function shellSummariesEqual( |
| 774 | left: EnvironmentShellSummary, |
| 775 | right: EnvironmentShellSummary, |
| 776 | ): boolean { |
| 777 | return ( |
| 778 | left.hasSnapshot === right.hasSnapshot && |
| 779 | left.hasSynchronizingShell === right.hasSynchronizingShell && |
| 780 | left.firstError === right.firstError |
| 781 | ); |
| 782 | } |
| 783 | |
| 784 | function mapsEqual<K, V>(left: ReadonlyMap<K, V>, right: ReadonlyMap<K, V>): boolean { |
| 785 | if (left.size !== right.size) { |
| 786 | return false; |
| 787 | } |
| 788 | for (const [key, value] of left) { |
| 789 | if (right.get(key) !== value) { |
| 790 | return false; |
| 791 | } |
| 792 | } |
| 793 | return true; |
| 794 | } |
| 795 | |
| 796 | export function createEnvironmentShellSummaryAtom(input: { |
| 797 | readonly catalogValueAtom: Atom.Atom<EnvironmentCatalogState>; |
| 798 | readonly shellStateValueAtom: (environmentId: EnvironmentId) => Atom.Atom<EnvironmentShellState>; |
| 799 | }) { |
| 800 | let previousSummary = EMPTY_ENVIRONMENT_SHELL_SUMMARY; |
| 801 | return Atom.make((get) => { |
| 802 | let hasSnapshot = false; |
| 803 | let hasSynchronizingShell = false; |
| 804 | let firstError: string | null = null; |
| 805 | |
| 806 | for (const environmentId of enabledEnvironmentIds(get(input.catalogValueAtom))) { |
| 807 | const state = get(input.shellStateValueAtom(environmentId)); |
| 808 | hasSynchronizingShell ||= state.status === "synchronizing"; |
| 809 | if (firstError === null) { |
| 810 | firstError = Option.getOrNull(state.error); |
| 811 | } |
| 812 | if (Option.isNone(state.snapshot)) { |
| 813 | continue; |
| 814 | } |
| 815 | hasSnapshot = true; |
| 816 | } |
| 817 | |
| 818 | const next: EnvironmentShellSummary = { |
| 819 | hasSnapshot, |
| 820 | hasSynchronizingShell, |
| 821 | firstError, |
| 822 | }; |
| 823 | if (shellSummariesEqual(previousSummary, next)) { |
| 824 | return previousSummary; |
| 825 | } |
| 826 | previousSummary = next; |
| 827 | return previousSummary; |
| 828 | }).pipe(Atom.withLabel("environment-shell-summary")); |
| 829 | } |
| 830 | |
| 831 | export function createEnvironmentServerConfigsAtom(input: { |
| 832 | readonly catalogValueAtom: Atom.Atom<EnvironmentCatalogState>; |
| 833 | readonly serverConfigValueAtom: (environmentId: EnvironmentId) => Atom.Atom<ServerConfig | null>; |
| 834 | }) { |
| 835 | let previousServerConfigs = EMPTY_SERVER_CONFIGS; |
| 836 | return Atom.make((get) => { |
| 837 | const next = new Map<EnvironmentId, ServerConfig>(); |
| 838 | for (const environmentId of enabledEnvironmentIds(get(input.catalogValueAtom))) { |
| 839 | const config = get(input.serverConfigValueAtom(environmentId)); |
| 840 | if (config !== null) { |
| 841 | next.set(environmentId, config); |
| 842 | } |
| 843 | } |
| 844 | if (mapsEqual(previousServerConfigs, next)) { |
| 845 | return previousServerConfigs; |
| 846 | } |
| 847 | previousServerConfigs = next; |
| 848 | return previousServerConfigs; |
| 849 | }).pipe(Atom.withLabel("environment-server-configs")); |
| 850 | } |
| 851 | |
| 852 | /** |
| 853 | * Every shell push replaces the state, so views that gate on load progress read these |
| 854 | * primitives instead and stay put while threads stream. |
| 855 | */ |
| 856 | function createEnvironmentShellProgressAtoms( |
| 857 | stateValueAtom: (environmentId: EnvironmentId) => Atom.Atom<EnvironmentShellState>, |
| 858 | ) { |
| 859 | return { |
| 860 | statusAtom: Atom.family((environmentId: EnvironmentId) => |
| 861 | Atom.make((get) => get(stateValueAtom(environmentId)).status).pipe( |
| 862 | Atom.withLabel(`environment-shell-status:${environmentId}`), |
| 863 | ), |
| 864 | ), |
| 865 | hasSnapshotAtom: Atom.family((environmentId: EnvironmentId) => |
| 866 | Atom.make((get) => Option.isSome(get(stateValueAtom(environmentId)).snapshot)).pipe( |
| 867 | Atom.withLabel(`environment-shell-has-snapshot:${environmentId}`), |
| 868 | ), |
| 869 | ), |
| 870 | }; |
| 871 | } |
| 872 | |
| 873 | export function createEnvironmentShellAtoms<R, E>( |
| 874 | runtime: Atom.AtomRuntime< |
| 875 | | EnvironmentRegistry.EnvironmentRegistry |
| 876 | | Persistence.EnvironmentCacheStore |
| 877 | | ShellSnapshotLoader.ShellSnapshotLoader |
| 878 | | R, |
| 879 | E |
| 880 | >, |
| 881 | ) { |
| 882 | const stateAtom = Atom.family((environmentId: EnvironmentId) => |
| 883 | runtime.atom(shellStateChanges(environmentId), { |
| 884 | initialValue: EMPTY_SHELL_STATE, |
| 885 | }), |
| 886 | ); |
| 887 | |
| 888 | const stateValueAtom = Atom.family((environmentId: EnvironmentId) => |
| 889 | Atom.make((get) => |
| 890 | AsyncResult.getOrElse(get(stateAtom(environmentId)), () => EMPTY_SHELL_STATE), |
| 891 | ).pipe(Atom.withLabel(`environment-shell-state-value:${environmentId}`)), |
| 892 | ); |
| 893 | |
| 894 | const { statusAtom, hasSnapshotAtom } = createEnvironmentShellProgressAtoms(stateValueAtom); |
| 895 | |
| 896 | return { |
| 897 | stateAtom, |
| 898 | stateValueAtom, |
| 899 | statusAtom, |
| 900 | hasSnapshotAtom, |
| 901 | }; |
| 902 | } |
| 903 | |
| 904 | export * from "./models.ts"; |
| 905 | export * from "./shellCommands.ts"; |
| 906 | export * from "./shellReducer.ts"; |
| 907 | export * from "./shellPullRequests.ts"; |
| 908 | export * as ShellSnapshotLoader from "./shellSnapshotHttp.ts"; |
| 909 | export * from "./snapshots.ts"; |
| 910 | export * from "./steadyShells.ts"; |