The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,223 lines 43.8 KB
Raw
1 import * as Brand from "effect/Brand";
2 import * as Cause from "effect/Cause";
3 import * as Deferred from "effect/Deferred";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import * as Exit from "effect/Exit";
7 import * as Fiber from "effect/Fiber";
8 import * as FileSystem from "effect/FileSystem";
9 import * as Option from "effect/Option";
10 import * as PlatformError from "effect/PlatformError";
11 import * as Ref from "effect/Ref";
12 import * as SubscriptionRef from "effect/SubscriptionRef";
13 import * as Schema from "effect/Schema";
14 import * as Semaphore from "effect/Semaphore";
15 import * as Scope from "effect/Scope";
16 import * as Stream from "effect/Stream";
17 import { HttpClient } from "effect/http";
18 import { ChildProcess, ChildProcessSpawner } from "effect/process";
19
20 import {
21 SERVER_EXIT_CODE_MANAGED_POLICY,
22 SERVER_EXIT_CODE_STATE_DIR_OWNED,
23 DesktopBackendBootstrap,
24 type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
25 PRIMARY_LOCAL_ENVIRONMENT_ID,
26 DesktopTelemetryControlMessage,
27 type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue,
28 } from "@t3tools/contracts";
29 import { isRunningDatabaseMigrations } from "@t3tools/shared/databaseMigrationLog";
30 import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness";
31
32 import * as DesktopObservability from "../app/DesktopObservability.ts";
33 import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts";
34 import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts";
35 import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";
36
37 const INITIAL_RESTART_DELAY = Duration.millis(500);
38 const MAX_RESTART_DELAY = Duration.seconds(10);
39 const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
40 // Large state databases can take over a minute to become ready.
41 const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(3);
42 const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100);
43 const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1);
44 // Match the service launcher's grace period so backends can stop provider sessions.
45 const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5);
46 // Allow SIGKILL and the spawner exit wait after the SIGTERM grace so quit cannot
47 // orphan a backend whose shutdown hangs.
48 export const BACKEND_QUIT_STOP_TIMEOUT = Duration.sum(
49 DEFAULT_BACKEND_TERMINATE_GRACE,
50 Duration.seconds(2),
51 );
52 const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5);
53 const BACKEND_READINESS_PATH = "/.well-known/t3/environment";
54 // Three restarts without readiness is about 2 s of backoff: past a blip, early enough to help.
55 const STARTUP_FAILURE_RESTART_ATTEMPTS = 3;
56 const STARTUP_FAILURE_OUTPUT_CHARS = 4_000;
57 const { logWarning: logBackendProcessWarning } =
58 DesktopObservability.makeComponentLogger("desktop-backend-process");
59
60 type BackendProcessLayerServices = ChildProcessSpawner.ChildProcessSpawner | HttpClient.HttpClient;
61
62 type BackendProcessRunRequirements = BackendProcessLayerServices | Scope.Scope;
63
64 export type BackendProcessOutputStream = "stdout" | "stderr";
65
66 export interface BackendProcessContext {
67 readonly executablePath: string;
68 readonly entryPath: string;
69 readonly cwd: string;
70 readonly httpBaseUrl: URL;
71 }
72
73 export type DesktopBackendBootstrapDelivery = "fd3" | "stdin";
74
75 export interface DesktopBackendStartConfig extends BackendProcessContext {
76 readonly args: ReadonlyArray<string>;
77 readonly env: Record<string, string | undefined>;
78 /** False passes env verbatim, preventing the Windows T3CODE_HOME from leaking into WSL. */
79 readonly extendEnv: boolean;
80 readonly bootstrap: DesktopBackendBootstrapValue;
81 readonly bootstrapDelivery: DesktopBackendBootstrapDelivery;
82 readonly httpBaseUrl: URL;
83 readonly captureOutput: boolean;
84 readonly preflightFailure: Option.Option<PreflightFailure>;
85 readonly runningDistro?: string;
86 readonly wslRuntimeId?: string;
87 readonly wslT3Home?: string | null;
88 }
89
90 export interface PreflightFailure {
91 readonly reason: string;
92 readonly fatal: boolean;
93 readonly retryLimit?: number;
94 }
95
96 interface BackendProcessExit {
97 readonly code: Option.Option<number>;
98 readonly reason: string;
99 }
100
101 const backendProcessContextSchema = {
102 executablePath: Schema.String,
103 entryPath: Schema.String,
104 cwd: Schema.String,
105 httpBaseUrl: Schema.URL,
106 };
107
108 export class BackendReadinessTimeoutError extends Schema.TaggedError<BackendReadinessTimeoutError>()(
109 "BackendReadinessTimeoutError",
110 {
111 ...backendProcessContextSchema,
112 readinessUrl: Schema.URL,
113 timeoutMs: Schema.Number,
114 cause: Schema.Defect(),
115 },
116 ) {
117 override get message(): string {
118 return `Timed out after ${this.timeoutMs}ms waiting for desktop backend readiness at ${this.readinessUrl.href}.`;
119 }
120 }
121
122 export class BackendProcessBootstrapEncodeError extends Schema.TaggedError<BackendProcessBootstrapEncodeError>()(
123 "BackendProcessBootstrapEncodeError",
124 {
125 ...backendProcessContextSchema,
126 cause: Schema.Defect(),
127 },
128 ) {
129 override get message(): string {
130 return `Failed to encode the desktop backend bootstrap payload for ${this.entryPath}.`;
131 }
132 }
133
134 export class BackendProcessSpawnError extends Schema.TaggedError<BackendProcessSpawnError>()(
135 "BackendProcessSpawnError",
136 {
137 ...backendProcessContextSchema,
138 cause: Schema.Defect(),
139 },
140 ) {
141 override get message(): string {
142 return `Failed to spawn desktop backend entry ${this.entryPath} with ${this.executablePath}.`;
143 }
144 }
145
146 export class BackendProcessOutputReadError extends Schema.TaggedError<BackendProcessOutputReadError>()(
147 "BackendProcessOutputReadError",
148 {
149 ...backendProcessContextSchema,
150 pid: Schema.Number,
151 streamName: Schema.Literals(["stdout", "stderr"]),
152 cause: Schema.Defect(),
153 },
154 ) {
155 override get message(): string {
156 return `Failed to read ${this.streamName} from desktop backend process ${this.pid}.`;
157 }
158 }
159
160 export class BackendProcessOutputHandlingError extends Schema.TaggedError<BackendProcessOutputHandlingError>()(
161 "BackendProcessOutputHandlingError",
162 {
163 ...backendProcessContextSchema,
164 pid: Schema.Number,
165 streamName: Schema.Literals(["stdout", "stderr"]),
166 chunkByteLength: Schema.Number,
167 cause: Schema.Defect(),
168 },
169 ) {
170 override get message(): string {
171 return `Failed to handle ${this.chunkByteLength} bytes from ${this.streamName} of desktop backend process ${this.pid}.`;
172 }
173 }
174
175 export type BackendProcessOutputError =
176 | BackendProcessOutputReadError
177 | BackendProcessOutputHandlingError;
178
179 export class BackendProcessExitStatusError extends Schema.TaggedError<BackendProcessExitStatusError>()(
180 "BackendProcessExitStatusError",
181 {
182 ...backendProcessContextSchema,
183 pid: Schema.Number,
184 cause: Schema.Defect(),
185 },
186 ) {
187 override get message(): string {
188 return `Failed to read the exit status of desktop backend process ${this.pid}.`;
189 }
190 }
191
192 export const BackendProcessError = Schema.Union([
193 BackendProcessBootstrapEncodeError,
194 BackendProcessSpawnError,
195 BackendProcessExitStatusError,
196 ]);
197 export type BackendProcessError = typeof BackendProcessError.Type;
198
199 interface RunBackendProcessOptions extends DesktopBackendStartConfig {
200 readonly desktopTelemetryStream: Stream.Stream<Uint8Array>;
201 readonly desktopBrowserStream?: Stream.Stream<Uint8Array>;
202 readonly onDesktopBrowserCommand?: (line: string) => Effect.Effect<void>;
203 readonly onDesktopTelemetryControl?: (
204 message: DesktopTelemetryControlMessageValue,
205 ) => Effect.Effect<void>;
206 readonly readinessTimeout?: Duration.Duration;
207 readonly outputDrainTimeout?: Duration.Duration;
208 readonly onStarted?: (pid: number) => Effect.Effect<void>;
209 readonly onExitObserved?: () => Effect.Effect<void>;
210 readonly onReady?: (pid: number) => Effect.Effect<void>;
211 readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect<void>;
212 readonly onOutput?: (
213 streamName: BackendProcessOutputStream,
214 chunk: Uint8Array,
215 ) => Effect.Effect<void, Error>;
216 readonly onOutputFailure?: (error: BackendProcessOutputError) => Effect.Effect<void>;
217 }
218
219 export interface DesktopBackendSnapshot {
220 readonly desiredRunning: boolean;
221 readonly ready: boolean;
222 readonly activePid: Option.Option<number>;
223 readonly restartAttempt: number;
224 readonly restartScheduled: boolean;
225 }
226
227 export type BackendInstanceId = string & Brand.Brand<"BackendInstanceId">;
228 export const BackendInstanceId = Brand.nominal<BackendInstanceId>();
229
230 export const PRIMARY_INSTANCE_ID: BackendInstanceId = BackendInstanceId(
231 PRIMARY_LOCAL_ENVIRONMENT_ID,
232 );
233
234 export interface BackendStartupFailure {
235 readonly reason: string;
236 /** The server refused to start under this machine's managed policy. */
237 readonly managedPolicy: boolean;
238 /** Tail of the last run's stdout and stderr. */
239 readonly output: string;
240 }
241
242 export interface DesktopBackendInstance {
243 readonly id: BackendInstanceId;
244 readonly label: Effect.Effect<string>;
245 readonly start: Effect.Effect<void>;
246 /** Timeout controls the warning; stop still waits for process teardown and exit. */
247 readonly stop: (options?: { readonly timeout?: Duration.Duration }) => Effect.Effect<void>;
248 readonly currentConfig: Effect.Effect<Option.Option<DesktopBackendStartConfig>>;
249 readonly snapshot: Effect.Effect<DesktopBackendSnapshot>;
250 readonly waitForReady: (timeout: Duration.Duration) => Effect.Effect<boolean>;
251 }
252
253 export interface BackendInstanceSpec {
254 readonly id: BackendInstanceId;
255 readonly label: Effect.Effect<string>;
256 readonly configResolve: Effect.Effect<DesktopBackendStartConfig, PlatformError.PlatformError>;
257 /** Receives the ready run's URL and pid; reading current config could race a subsequent restart. */
258 readonly onReady?: (httpBaseUrl: URL, pid: number) => Effect.Effect<void>;
259 readonly onShutdown?: () => Effect.Effect<void>;
260 readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect<boolean>;
261 readonly onStateDirOwned?: () => Effect.Effect<void>;
262 /** Fires on each repeated crash or readiness timeout until the backend becomes ready. */
263 readonly onStartupFailure?: (failure: BackendStartupFailure) => Effect.Effect<void>;
264 }
265
266 interface ActiveBackendRun {
267 readonly id: number;
268 readonly scope: Scope.Closeable;
269 readonly fiber: Option.Option<Fiber.Fiber<void, never>>;
270 readonly pid: Option.Option<number>;
271 readonly exitObserved: boolean;
272 readonly stopRequested: boolean;
273 }
274
275 interface BackendManagerState {
276 readonly desiredRunning: boolean;
277 readonly ready: boolean;
278 readonly config: Option.Option<DesktopBackendStartConfig>;
279 readonly active: Option.Option<ActiveBackendRun>;
280 readonly restartAttempt: number;
281 readonly preflightFailureAttempt: number;
282 readonly restartFiber: Option.Option<Fiber.Fiber<void, never>>;
283 readonly nextRunId: number;
284 }
285
286 const initialState: BackendManagerState = {
287 desiredRunning: false,
288 ready: false,
289 config: Option.none(),
290 active: Option.none(),
291 restartAttempt: 0,
292 preflightFailureAttempt: 0,
293 restartFiber: Option.none(),
294 nextRunId: 1,
295 };
296
297 const activePid = (active: Option.Option<ActiveBackendRun>): Option.Option<number> =>
298 Option.flatMap(active, (run) => run.pid);
299
300 const withActiveRun =
301 (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
302 (state: BackendManagerState): BackendManagerState => ({
303 ...state,
304 active: Option.map(state.active, (run) => (run.id === runId ? f(run) : run)),
305 });
306
307 const calculateRestartDelay = (attempt: number): Duration.Duration =>
308 Duration.min(Duration.times(INITIAL_RESTART_DELAY, 2 ** attempt), MAX_RESTART_DELAY);
309
310 const closeRun = (
311 run: ActiveBackendRun,
312 parentScope: Scope.Scope,
313 options?: { readonly timeout?: Duration.Duration },
314 ) => {
315 const waitForFiber = Option.match(run.fiber, {
316 onNone: () => Effect.void,
317 onSome: (fiber) => Fiber.await(fiber).pipe(Effect.asVoid),
318 });
319 const close = Scope.close(run.scope, Exit.void).pipe(
320 Effect.withSpan("desktop.backendInstance.closeScope"),
321 Effect.andThen(waitForFiber.pipe(Effect.withSpan("desktop.backendInstance.awaitRun"))),
322 );
323 const timeout = options?.timeout;
324
325 if (!timeout) {
326 return close;
327 }
328
329 return Effect.forkIn(close, parentScope, { startImmediately: true }).pipe(
330 Effect.flatMap((closeFiber) =>
331 Fiber.await(closeFiber).pipe(
332 Effect.timeoutOption(timeout),
333 Effect.tap((completed) =>
334 Option.isNone(completed)
335 ? logBackendProcessWarning("backend shutdown exceeded timeout; waiting for exit", {
336 timeoutMs: Duration.toMillis(timeout),
337 })
338 : Effect.void,
339 ),
340 // Parent-scope interruption still finishes release; do not cancel sibling stops.
341 Effect.andThen(Fiber.await(closeFiber)),
342 Effect.asVoid,
343 ),
344 ),
345 );
346 };
347
348 const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "localhost", "[::1]"]);
349 // Windows takes about a second to fail a refused connection to a non-loopback address such as a
350 // WSL distro IP, so one sequential poller notices readiness up to a second late.
351 const STAGGERED_READINESS_POLLERS = 4;
352 const READINESS_POLLER_STAGGER = Duration.millis(250);
353
354 const waitForHttpReady = (
355 options: BackendProcessContext & { readonly timeout: Duration.Duration },
356 ): Effect.Effect<void, BackendReadinessTimeoutError, HttpClient.HttpClient> => {
357 const poll = pollHttpReady(options);
358 if (LOOPBACK_HOSTNAMES.has(options.httpBaseUrl.hostname)) return poll;
359 return Effect.raceAll(
360 Array.from({ length: STAGGERED_READINESS_POLLERS }, (_, index) =>
361 Effect.sleep(Duration.times(READINESS_POLLER_STAGGER, index)).pipe(Effect.andThen(poll)),
362 ),
363 );
364 };
365
366 const pollHttpReady = (
367 options: BackendProcessContext & { readonly timeout: Duration.Duration },
368 ): Effect.Effect<void, BackendReadinessTimeoutError, HttpClient.HttpClient> => {
369 const readinessUrl = new URL(BACKEND_READINESS_PATH, options.httpBaseUrl);
370 return waitForHttpReadyShared({
371 baseUrl: options.httpBaseUrl.href,
372 path: BACKEND_READINESS_PATH,
373 timeoutMs: Duration.toMillis(options.timeout),
374 intervalMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_INTERVAL),
375 probeTimeoutMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT),
376 makeError: ({ cause }) =>
377 new BackendReadinessTimeoutError({
378 executablePath: options.executablePath,
379 entryPath: options.entryPath,
380 cwd: options.cwd,
381 httpBaseUrl: options.httpBaseUrl,
382 readinessUrl,
383 timeoutMs: Duration.toMillis(options.timeout),
384 cause,
385 }),
386 });
387 };
388
389 function drainBackendOutput(
390 context: BackendProcessContext & { readonly pid: number },
391 streamName: BackendProcessOutputStream,
392 stream: Stream.Stream<Uint8Array, PlatformError.PlatformError>,
393 onOutput: (
394 streamName: BackendProcessOutputStream,
395 chunk: Uint8Array,
396 ) => Effect.Effect<void, Error>,
397 onOutputFailure: (error: BackendProcessOutputError) => Effect.Effect<void>,
398 ): Effect.Effect<void> {
399 return stream.pipe(
400 Stream.mapError(
401 (cause) =>
402 new BackendProcessOutputReadError({
403 ...context,
404 streamName,
405 cause,
406 }),
407 ),
408 Stream.runForEach((chunk) =>
409 onOutput(streamName, chunk).pipe(
410 Effect.mapError(
411 (cause) =>
412 new BackendProcessOutputHandlingError({
413 ...context,
414 streamName,
415 chunkByteLength: chunk.byteLength,
416 cause,
417 }),
418 ),
419 Effect.catchTags({ BackendProcessOutputHandlingError: onOutputFailure }),
420 ),
421 ),
422 Effect.catchTags({
423 BackendProcessOutputReadError: onOutputFailure,
424 }),
425 );
426 }
427
428 const encodeBootstrapJson = Schema.encodeEffect(Schema.fromJsonString(DesktopBackendBootstrap));
429 const decodeDesktopTelemetryControlLine = Schema.decodeUnknownEffect(
430 Schema.fromJsonString(DesktopTelemetryControlMessage),
431 );
432
433 export const runBackendProcess = Effect.fn("runBackendProcess")(function* (
434 options: RunBackendProcessOptions,
435 ): Effect.fn.Return<BackendProcessExit, BackendProcessError, BackendProcessRunRequirements> {
436 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
437 const bootstrapJson = yield* encodeBootstrapJson(options.bootstrap).pipe(
438 Effect.mapError(
439 (cause) =>
440 new BackendProcessBootstrapEncodeError({
441 executablePath: options.executablePath,
442 entryPath: options.entryPath,
443 cwd: options.cwd,
444 httpBaseUrl: options.httpBaseUrl,
445 cause,
446 }),
447 ),
448 );
449 const onOutput = options.onOutput ?? (() => Effect.void);
450 const bootstrapStream = Stream.encodeText(Stream.make(`${bootstrapJson}\n`));
451 const additionalFds: Record<`fd${number}`, ChildProcess.AdditionalFdConfig> = {};
452 if (options.bootstrapDelivery === "fd3") {
453 additionalFds.fd3 = {
454 type: "input",
455 stream: bootstrapStream,
456 };
457 if (options.bootstrap.desktopTelemetryFd !== undefined) {
458 additionalFds[`fd${options.bootstrap.desktopTelemetryFd}`] = {
459 type: "input",
460 stream: options.desktopTelemetryStream,
461 };
462 }
463 if (options.bootstrap.desktopTelemetryControlFd !== undefined) {
464 additionalFds[`fd${options.bootstrap.desktopTelemetryControlFd}`] = {
465 type: "output",
466 };
467 }
468 if (
469 options.bootstrap.desktopBrowserFd !== undefined &&
470 options.desktopBrowserStream !== undefined
471 ) {
472 additionalFds[`fd${options.bootstrap.desktopBrowserFd}`] = {
473 type: "input",
474 stream: options.desktopBrowserStream,
475 };
476 }
477 if (options.bootstrap.desktopBrowserControlFd !== undefined) {
478 additionalFds[`fd${options.bootstrap.desktopBrowserControlFd}`] = { type: "output" };
479 }
480 }
481 const command = ChildProcess.make(options.executablePath, options.args, {
482 cwd: options.cwd,
483 env: options.env,
484 extendEnv: options.extendEnv,
485 stdin: options.bootstrapDelivery === "stdin" ? bootstrapStream : "ignore",
486 stdout: options.captureOutput ? "pipe" : "inherit",
487 stderr: options.captureOutput ? "pipe" : "inherit",
488 killSignal: "SIGTERM",
489 forceKillAfter: DEFAULT_BACKEND_TERMINATE_GRACE,
490 // wsl.exe drops extra file descriptors; deliver its bootstrap via stdin.
491 ...(options.bootstrapDelivery === "fd3" ? { additionalFds } : {}),
492 });
493
494 const handle = yield* spawner.spawn(command).pipe(
495 Effect.mapError(
496 (cause) =>
497 new BackendProcessSpawnError({
498 executablePath: options.executablePath,
499 entryPath: options.entryPath,
500 cwd: options.cwd,
501 httpBaseUrl: options.httpBaseUrl,
502 cause,
503 }),
504 ),
505 );
506 const outputFibers: Array<Fiber.Fiber<void, never>> = [];
507 const serverReady = yield* Deferred.make<void>();
508
509 yield* options.onStarted?.(handle.pid) ?? Effect.void;
510 if (options.bootstrap.desktopTelemetryControlFd !== undefined) {
511 const controlFd = options.bootstrap.desktopTelemetryControlFd;
512 const handleControl = options.onDesktopTelemetryControl ?? (() => Effect.void);
513 yield* handle.getOutputFd(controlFd).pipe(
514 Stream.decodeText(),
515 Stream.splitLines,
516 Stream.filter((line) => line.trim().length > 0),
517 Stream.runForEach((line) =>
518 decodeDesktopTelemetryControlLine(line).pipe(
519 Effect.flatMap((message) =>
520 message.type === "serverReady"
521 ? Deferred.succeed(serverReady, undefined)
522 : handleControl(message),
523 ),
524 Effect.catchCause((cause) =>
525 logBackendProcessWarning("ignored invalid desktop telemetry control message", {
526 fd: controlFd,
527 cause: Cause.pretty(cause),
528 }),
529 ),
530 ),
531 ),
532 Effect.catchCause((cause) =>
533 logBackendProcessWarning("desktop telemetry control stream stopped", {
534 fd: controlFd,
535 cause: Cause.pretty(cause),
536 }),
537 ),
538 Effect.ensuring(
539 handleControl({
540 version: 1,
541 type: "setDiagnosticsDemand",
542 enabled: false,
543 }),
544 ),
545 Effect.forkScoped,
546 );
547 }
548 if (
549 options.bootstrap.desktopBrowserControlFd !== undefined &&
550 options.onDesktopBrowserCommand !== undefined
551 ) {
552 const browserFd = options.bootstrap.desktopBrowserControlFd;
553 const handleCommand = options.onDesktopBrowserCommand;
554 yield* handle.getOutputFd(browserFd).pipe(
555 Stream.decodeText(),
556 Stream.splitLines,
557 Stream.filter((line) => line.length > 0),
558 Stream.runForEach(handleCommand),
559 Effect.catchCause((cause) =>
560 logBackendProcessWarning("desktop browser command stream stopped", {
561 fd: browserFd,
562 cause: Cause.pretty(cause),
563 }),
564 ),
565 Effect.forkScoped,
566 );
567 }
568 if (options.captureOutput) {
569 const outputContext = {
570 executablePath: options.executablePath,
571 entryPath: options.entryPath,
572 cwd: options.cwd,
573 httpBaseUrl: options.httpBaseUrl,
574 pid: Number(handle.pid),
575 };
576 const onOutputFailure = options.onOutputFailure ?? (() => Effect.void);
577 outputFibers.push(
578 yield* drainBackendOutput(
579 outputContext,
580 "stdout",
581 handle.stdout,
582 onOutput,
583 onOutputFailure,
584 ).pipe(Effect.forkScoped),
585 yield* drainBackendOutput(
586 outputContext,
587 "stderr",
588 handle.stderr,
589 onOutput,
590 onOutputFailure,
591 ).pipe(Effect.forkScoped),
592 );
593 }
594 // Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive.
595 // HTTP polling stays as the fallback for backends without the control fd, such as WSL.
596 const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() =>
597 Effect.raceFirst(
598 waitForHttpReady({
599 executablePath: options.executablePath,
600 entryPath: options.entryPath,
601 cwd: options.cwd,
602 httpBaseUrl: options.httpBaseUrl,
603 timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT,
604 }),
605 Deferred.await(serverReady),
606 ).pipe(
607 Effect.flatMap(() => options.onReady?.(handle.pid) ?? Effect.void),
608 Effect.as(true),
609 Effect.catchTags({
610 BackendReadinessTimeoutError: (error) =>
611 (options.onReadinessFailure?.(error) ?? Effect.void).pipe(Effect.as(false)),
612 }),
613 ),
614 );
615
616 yield* probeReadiness().pipe(Effect.repeat({ while: (ready) => !ready }), Effect.forkScoped);
617
618 const exit = yield* handle.exitCode.pipe(
619 Effect.withSpan("desktop.backendProcess.awaitExit"),
620 Effect.map((exitCode) => ({
621 code: Option.some(exitCode),
622 reason: `code=${exitCode}`,
623 })),
624 Effect.catch((cause) => {
625 // Effect's Node spawner exposes signal exits only through this cause message.
626 const signal =
627 cause.cause instanceof Error
628 ? /^Process interrupted due to receipt of signal: '(SIG[A-Z0-9]+)'$/.exec(
629 cause.cause.message,
630 )?.[1]
631 : undefined;
632 if (signal !== undefined) {
633 return Effect.succeed({ code: Option.none(), reason: `killed by ${signal}` });
634 }
635 return Effect.fail(
636 new BackendProcessExitStatusError({
637 executablePath: options.executablePath,
638 entryPath: options.entryPath,
639 cwd: options.cwd,
640 httpBaseUrl: options.httpBaseUrl,
641 pid: Number(handle.pid),
642 cause,
643 }),
644 );
645 }),
646 Effect.exit,
647 );
648 yield* options.onExitObserved?.() ?? Effect.void;
649 yield* Effect.forEach(outputFibers, Fiber.await, {
650 concurrency: "unbounded",
651 discard: true,
652 }).pipe(
653 Effect.timeout(options.outputDrainTimeout ?? DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT),
654 Effect.ignore,
655 Effect.withSpan("desktop.backendProcess.drainOutput"),
656 );
657 if (Exit.isFailure(exit)) {
658 return yield* Effect.failCause(exit.cause);
659 }
660 return exit.value;
661 });
662
663 export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
664 spec: BackendInstanceSpec,
665 ): Effect.fn.Return<
666 DesktopBackendInstance,
667 never,
668 | FileSystem.FileSystem
669 | ChildProcessSpawner.ChildProcessSpawner
670 | HttpClient.HttpClient
671 | DesktopObservability.DesktopBackendOutputLogFactory
672 | DesktopTelemetryPublisher.DesktopTelemetryPublisher
673 | DesktopBrowserHost.DesktopBrowserHost
674 | DesktopWslEnvironment.DesktopWslEnvironment
675 | Scope.Scope
676 > {
677 const parentScope = yield* Scope.Scope;
678 const fileSystem = yield* FileSystem.FileSystem;
679 const backendOutputLogFactory = yield* DesktopObservability.DesktopBackendOutputLogFactory;
680 const backendOutputLog = yield* backendOutputLogFactory.forInstance(spec.id);
681 const desktopTelemetryPublisher = yield* DesktopTelemetryPublisher.DesktopTelemetryPublisher;
682 const desktopBrowserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
683 const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment;
684 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
685 const httpClient = yield* HttpClient.HttpClient;
686 const state = yield* SubscriptionRef.make(initialState);
687 const mutex = yield* Semaphore.make(1);
688 const outputTail = yield* Ref.make("");
689 // One streaming decoder per stream keeps UTF-8 characters split across chunks intact.
690 const makeOutputDecoders = () => ({ stdout: new TextDecoder(), stderr: new TextDecoder() });
691 let outputDecoders = makeOutputDecoders();
692 const reportStartupFailure = (reason: string, managedPolicy = false) =>
693 Ref.get(outputTail).pipe(
694 Effect.flatMap(
695 (output) => spec.onStartupFailure?.({ reason, managedPolicy, output }) ?? Effect.void,
696 ),
697 );
698
699 const { logWarning: logInstanceWarning, logError: logInstanceError } =
700 DesktopObservability.makeComponentLogger(`desktop-backend-instance:${spec.id}`);
701
702 const updateActiveRun = (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
703 SubscriptionRef.update(state, withActiveRun(runId, f));
704
705 const snapshot = SubscriptionRef.get(state).pipe(
706 Effect.map((current): DesktopBackendSnapshot => ({
707 desiredRunning: current.desiredRunning,
708 ready: current.ready,
709 activePid: activePid(current.active),
710 restartAttempt: current.restartAttempt,
711 restartScheduled: Option.isSome(current.restartFiber),
712 })),
713 );
714 const currentConfig = SubscriptionRef.get(state).pipe(Effect.map((current) => current.config));
715
716 const cancelRestart = Effect.gen(function* () {
717 const restartFiber = yield* SubscriptionRef.modify(state, (current) => [
718 current.restartFiber,
719 {
720 ...current,
721 restartFiber: Option.none(),
722 },
723 ]);
724
725 yield* Option.match(restartFiber, {
726 onNone: () => Effect.void,
727 onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
728 });
729 });
730
731 const start: Effect.Effect<void> = Effect.suspend(() =>
732 mutex.withPermits(1)(
733 Effect.gen(function* () {
734 const current = yield* SubscriptionRef.get(state);
735 if (Option.isSome(current.active)) {
736 if (!current.desiredRunning) {
737 yield* SubscriptionRef.update(state, (latest) => ({
738 ...latest,
739 desiredRunning: true,
740 }));
741 }
742 return;
743 }
744
745 if (current.ready) {
746 yield* spec.onShutdown?.() ?? Effect.void;
747 yield* SubscriptionRef.update(state, (latest) =>
748 latest.ready ? { ...latest, ready: false } : latest,
749 );
750 }
751 const config = yield* spec.configResolve.pipe(
752 Effect.tapError((error) =>
753 logInstanceError("failed to generate desktop backend configuration", {
754 cause: error.message,
755 }),
756 ),
757 Effect.option,
758 );
759 if (Option.isNone(config)) {
760 if (current.desiredRunning) {
761 yield* scheduleRestart("failed to generate desktop backend configuration");
762 }
763 return;
764 }
765 const entryExists = yield* fileSystem
766 .exists(config.value.entryPath)
767 .pipe(Effect.orElseSucceed(() => false));
768
769 const resetFatalPreflightCounter =
770 !current.desiredRunning && current.preflightFailureAttempt > 0;
771 yield* cancelRestart;
772 yield* SubscriptionRef.update(state, (latest) => ({
773 ...latest,
774 desiredRunning: true,
775 ready: false,
776 config: Option.some(config.value),
777 preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt,
778 }));
779
780 const preflightFailure = config.value.preflightFailure;
781 if (Option.isSome(preflightFailure)) {
782 const { reason, fatal, retryLimit } = preflightFailure.value;
783 if (!fatal && retryLimit === undefined) {
784 yield* SubscriptionRef.update(state, (latest) =>
785 latest.preflightFailureAttempt === 0
786 ? latest
787 : { ...latest, preflightFailureAttempt: 0 },
788 );
789 yield* scheduleRestart(reason);
790 return;
791 }
792 const attemptLimit = retryLimit ?? MAX_PREFLIGHT_FAILURE_ATTEMPTS;
793 const attempt = yield* SubscriptionRef.modify(state, (latest) => {
794 const next = latest.preflightFailureAttempt + 1;
795 return [next, { ...latest, preflightFailureAttempt: next }] as const;
796 });
797 if (attempt > attemptLimit) {
798 yield* logInstanceError("backend preflight still failing after fallback; stopping", {
799 reason,
800 attempt,
801 });
802 yield* SubscriptionRef.update(state, (latest) => ({
803 ...latest,
804 desiredRunning: false,
805 ready: false,
806 }));
807 return;
808 }
809 if (attempt === attemptLimit) {
810 yield* logInstanceError(
811 "backend preflight failed repeatedly; surfacing and falling back",
812 { reason, attempt },
813 );
814 const shouldRestart = yield* (
815 spec.onPreflightFailed?.(preflightFailure.value) ?? Effect.succeed(false)
816 );
817 if (shouldRestart) {
818 yield* scheduleRestart(reason);
819 } else {
820 yield* SubscriptionRef.update(state, (latest) => ({
821 ...latest,
822 desiredRunning: false,
823 ready: false,
824 }));
825 }
826 return;
827 }
828 yield* scheduleRestart(reason);
829 return;
830 }
831 yield* SubscriptionRef.update(state, (latest) =>
832 latest.preflightFailureAttempt === 0 ? latest : { ...latest, preflightFailureAttempt: 0 },
833 );
834
835 if (!entryExists) {
836 yield* scheduleRestart(`missing server entry at ${config.value.entryPath}`);
837 return;
838 }
839
840 const runScope = yield* Scope.make("sequential");
841 const runId = yield* SubscriptionRef.modify(state, (latest) => [
842 latest.nextRunId,
843 {
844 ...latest,
845 active: Option.some({
846 id: latest.nextRunId,
847 scope: runScope,
848 fiber: Option.none(),
849 pid: Option.none(),
850 exitObserved: false,
851 stopRequested: false,
852 } satisfies ActiveBackendRun),
853 nextRunId: latest.nextRunId + 1,
854 },
855 ]);
856
857 const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* (
858 reason: string,
859 exitCode?: number,
860 ) {
861 const stateDirOwned = exitCode === SERVER_EXIT_CODE_STATE_DIR_OWNED;
862 yield* mutex.withPermits(1)(
863 Effect.gen(function* () {
864 const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
865 yield* SubscriptionRef.modify(
866 state,
867 (
868 latest,
869 ): readonly [
870 {
871 readonly isCurrentRun: boolean;
872 readonly nextState: BackendManagerState;
873 readonly pid: Option.Option<number>;
874 readonly exitObserved: boolean;
875 readonly stopRequested: boolean;
876 readonly wasReady: boolean;
877 },
878 BackendManagerState,
879 ] => {
880 const currentRun = Option.getOrUndefined(latest.active);
881 if (currentRun?.id !== runId) {
882 return [
883 {
884 isCurrentRun: false,
885 nextState: latest,
886 pid: Option.none<number>(),
887 exitObserved: false,
888 stopRequested: false,
889 wasReady: false,
890 },
891 latest,
892 ] as const;
893 }
894
895 const next = {
896 ...latest,
897 active: Option.none<ActiveBackendRun>(),
898 ready: false,
899 };
900 return [
901 {
902 isCurrentRun: true,
903 nextState: next,
904 pid: currentRun.pid,
905 exitObserved: currentRun.exitObserved,
906 stopRequested: currentRun.stopRequested,
907 wasReady: latest.ready,
908 },
909 next,
910 ] as const;
911 },
912 );
913
914 if (isCurrentRun) {
915 yield* desktopTelemetryPublisher.removeControlSource(spec.id);
916 if (Option.isSome(pid)) {
917 if (exitObserved && !stopRequested) {
918 yield* backendOutputLog.persistFailure({
919 details: `pid=${pid.value} ${reason}`,
920 });
921 } else {
922 yield* backendOutputLog.discardSession;
923 }
924 }
925 if (wasReady) {
926 yield* spec.onShutdown?.() ?? Effect.void;
927 }
928 }
929
930 if (isCurrentRun && nextState.desiredRunning) {
931 if (stateDirOwned && !stopRequested) {
932 yield* SubscriptionRef.update(state, (latest) => ({
933 ...latest,
934 desiredRunning: false,
935 }));
936 yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore);
937 return;
938 }
939 yield* scheduleRestart(reason, exitCode);
940 }
941 }),
942 );
943 });
944
945 const program = runBackendProcess({
946 ...config.value,
947 desktopTelemetryStream: desktopTelemetryPublisher.encoded,
948 desktopBrowserStream: desktopBrowserHost.events,
949 onDesktopBrowserCommand: desktopBrowserHost.handleCommandLine,
950 onDesktopTelemetryControl: (message) =>
951 desktopTelemetryPublisher.handleControlForSource(spec.id, message),
952 onStarted: Effect.fn("desktop.backendInstance.onStarted")(function* (pid) {
953 yield* updateActiveRun(runId, (run) => ({
954 ...run,
955 pid: Option.some(pid),
956 }));
957 yield* Ref.set(outputTail, "");
958 outputDecoders = makeOutputDecoders();
959 yield* backendOutputLog.beginSession({
960 details: `pid=${pid} port=${config.value.bootstrap.port} cwd=${config.value.cwd}`,
961 });
962 }),
963 onExitObserved: () =>
964 updateActiveRun(runId, (run) => ({
965 ...run,
966 exitObserved: true,
967 })),
968 onReady: Effect.fn("desktop.backendInstance.onReady")(function* (pid) {
969 const isCurrentRun = yield* SubscriptionRef.modify(state, (latest) => {
970 const activeRun = Option.getOrUndefined(latest.active);
971 if (activeRun?.id !== runId) {
972 return [false, latest] as const;
973 }
974
975 return [
976 true,
977 {
978 ...latest,
979 restartAttempt: 0,
980 ready: true,
981 },
982 ] as const;
983 });
984 if (!isCurrentRun) {
985 return;
986 }
987
988 yield* spec.onReady?.(config.value.httpBaseUrl, pid) ?? Effect.void;
989 if (
990 config.value.runningDistro !== undefined &&
991 config.value.wslRuntimeId !== undefined
992 ) {
993 yield* wslEnvironment.pruneRuntimes(
994 config.value.runningDistro,
995 config.value.wslRuntimeId,
996 config.value.wslT3Home ?? null,
997 );
998 }
999 }),
1000 onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")(
1001 function* (error) {
1002 if (isRunningDatabaseMigrations(yield* Ref.get(outputTail))) {
1003 yield* logInstanceWarning("backend is still migrating its database; waiting", {
1004 error: error.message,
1005 });
1006 return;
1007 }
1008 yield* logInstanceWarning("backend readiness check failed during bootstrap", {
1009 error: error.message,
1010 });
1011 yield* backendOutputLog.persistFailureSnapshot({
1012 details: error.message,
1013 });
1014 yield* reportStartupFailure("The server started but did not become ready in time.");
1015 },
1016 ),
1017 onOutput: (streamName, chunk) =>
1018 Ref.update(outputTail, (tail) =>
1019 // A cut through a surrogate pair would make encodeURIComponent throw when rendering.
1020 (tail + outputDecoders[streamName].decode(chunk, { stream: true }))
1021 .slice(-STARTUP_FAILURE_OUTPUT_CHARS)
1022 .toWellFormed(),
1023 ).pipe(Effect.andThen(backendOutputLog.writeOutputChunk(streamName, chunk))),
1024 }).pipe(
1025 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
1026 Effect.provideService(HttpClient.HttpClient, httpClient),
1027 Scope.provide(runScope),
1028 Effect.matchEffect({
1029 onFailure: (error) => finalizeRun(error.message),
1030 onSuccess: (exit) => finalizeRun(exit.reason, Option.getOrUndefined(exit.code)),
1031 }),
1032 Effect.ensuring(Scope.close(runScope, Exit.void).pipe(Effect.ignore)),
1033 );
1034
1035 const fiber = yield* Effect.forkIn(program, parentScope);
1036 yield* updateActiveRun(runId, (run) => ({
1037 ...run,
1038 fiber: Option.some(fiber),
1039 }));
1040 }),
1041 ),
1042 ).pipe(Effect.withSpan("desktop.backendInstance.start", { attributes: { id: spec.id } }));
1043
1044 const scheduleRestart = Effect.fn("desktop.backendInstance.scheduleRestart")(function* (
1045 reason: string,
1046 exitCode?: number,
1047 ) {
1048 const scheduled = yield* SubscriptionRef.modify(state, (latest) => {
1049 if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) {
1050 return [Option.none<{ delay: Duration.Duration; attempt: number }>(), latest] as const;
1051 }
1052
1053 const attempt = latest.restartAttempt + 1;
1054 return [
1055 Option.some({ delay: calculateRestartDelay(latest.restartAttempt), attempt }),
1056 { ...latest, restartAttempt: attempt },
1057 ] as const;
1058 });
1059
1060 yield* Option.match(scheduled, {
1061 onNone: () => Effect.void,
1062 onSome: Effect.fn("desktop.backendInstance.scheduleRestartFiber")(function* ({
1063 delay,
1064 attempt,
1065 }) {
1066 yield* logInstanceError("backend exited unexpectedly; restart scheduled", {
1067 reason,
1068 delayMs: Duration.toMillis(delay),
1069 });
1070 // Retrying cannot fix a policy refusal, so say so at once; retries still pick up a fix.
1071 const managedPolicy = exitCode === SERVER_EXIT_CODE_MANAGED_POLICY;
1072 if (managedPolicy || attempt >= STARTUP_FAILURE_RESTART_ATTEMPTS) {
1073 yield* reportStartupFailure(`The server could not start (${reason}).`, managedPolicy);
1074 }
1075 const restartFiber = yield* Effect.forkIn(
1076 Effect.sleep(delay).pipe(
1077 Effect.andThen(
1078 SubscriptionRef.modify(state, (latest) => {
1079 const shouldRestart = latest.desiredRunning;
1080 return [
1081 shouldRestart,
1082 {
1083 ...latest,
1084 restartFiber: Option.none(),
1085 },
1086 ] as const;
1087 }),
1088 ),
1089 Effect.flatMap((shouldRestart) => (shouldRestart ? start : Effect.void)),
1090 Effect.catchCause((cause) =>
1091 logInstanceError("desktop backend restart fiber failed", {
1092 cause: Cause.pretty(cause),
1093 }),
1094 ),
1095 ),
1096 parentScope,
1097 );
1098 yield* SubscriptionRef.update(state, (latest) =>
1099 Option.isNone(latest.restartFiber)
1100 ? {
1101 ...latest,
1102 restartFiber: Option.some(restartFiber),
1103 }
1104 : latest,
1105 );
1106 }),
1107 });
1108 });
1109
1110 const stop = Effect.fn("desktop.backendInstance.stop")(function* (options?: {
1111 readonly timeout?: Duration.Duration;
1112 }) {
1113 const { active, restartFiber, notifyShutdown } = yield* mutex.withPermits(1)(
1114 Effect.gen(function* () {
1115 const result = yield* SubscriptionRef.modify(state, (latest) => {
1116 const active = Option.map(latest.active, (run) =>
1117 run.exitObserved ? run : { ...run, stopRequested: true },
1118 );
1119 return [
1120 {
1121 active,
1122 restartFiber: latest.restartFiber,
1123 notifyShutdown: latest.ready,
1124 },
1125 {
1126 ...latest,
1127 desiredRunning: false,
1128 ready: false,
1129 active,
1130 restartFiber: Option.none<Fiber.Fiber<void, never>>(),
1131 },
1132 ] as const;
1133 });
1134 return result;
1135 }),
1136 );
1137
1138 if (notifyShutdown) {
1139 yield* (spec.onShutdown?.() ?? Effect.void).pipe(Effect.ignore);
1140 }
1141 yield* Option.match(restartFiber, {
1142 onNone: () => Effect.void,
1143 onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
1144 });
1145 yield* Option.match(active, {
1146 onNone: () => Effect.void,
1147 // The enclosing operation owns the span for this inline handler.
1148 onSome: Effect.fnUntraced(function* (run) {
1149 yield* closeRun(run, parentScope, options);
1150 const cleanup = yield* mutex.withPermits(1)(
1151 SubscriptionRef.modify(
1152 state,
1153 (
1154 latest,
1155 ): readonly [
1156 {
1157 readonly needsCleanup: boolean;
1158 readonly shouldStart: boolean;
1159 },
1160 BackendManagerState,
1161 ] => {
1162 const current = Option.getOrUndefined(latest.active);
1163 if (current?.id !== run.id) {
1164 return [
1165 {
1166 needsCleanup: false,
1167 shouldStart:
1168 latest.desiredRunning &&
1169 Option.isNone(latest.active) &&
1170 Option.isNone(latest.restartFiber),
1171 },
1172 latest,
1173 ];
1174 }
1175 return [
1176 {
1177 needsCleanup: true,
1178 shouldStart: latest.desiredRunning,
1179 },
1180 {
1181 ...latest,
1182 active: Option.none<ActiveBackendRun>(),
1183 },
1184 ];
1185 },
1186 ),
1187 );
1188 if (cleanup.needsCleanup) {
1189 yield* desktopTelemetryPublisher.removeControlSource(spec.id);
1190 yield* backendOutputLog.discardSession;
1191 }
1192 if (cleanup.shouldStart) {
1193 yield* start;
1194 }
1195 }),
1196 });
1197 });
1198
1199 const waitForReady = Effect.fn("DesktopBackendManager.waitForReady")(
1200 (timeout: Duration.Duration) =>
1201 SubscriptionRef.changes(state).pipe(
1202 Stream.filter((current) => current.ready || !current.desiredRunning),
1203 Stream.map((current) => current.ready),
1204 Stream.runHead,
1205 Effect.timeoutOption(timeout),
1206 Effect.map(
1207 (result) => Option.isSome(result) && Option.getOrElse(result.value, () => false),
1208 ),
1209 ),
1210 );
1211
1212 yield* Effect.addFinalizer(() => stop());
1213
1214 return {
1215 id: spec.id,
1216 label: spec.label,
1217 start,
1218 stop,
1219 currentConfig,
1220 snapshot,
1221 waitForReady,
1222 } satisfies DesktopBackendInstance;
1223 });