The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,656 lines 62 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 // @effect-diagnostics globalTimers:off
3 // The launcher must survive unloadable server versions; use Node built-ins without Effect.
4 import * as NodeChildProcess from "node:child_process";
5 import * as NodeCrypto from "node:crypto";
6 import * as NodeFS from "node:fs";
7 import * as NodeFSP from "node:fs/promises";
8 import * as NodePath from "node:path";
9 import * as NodeUtil from "node:util";
10 import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11 import { isNoCopyOnWrite, markNoCopyOnWrite } from "./noCopyOnWrite.ts";
12
13 import type {
14 PendingServiceUpdate,
15 ServiceLauncherChildMessage,
16 ServiceLauncherContext,
17 ServiceLauncherParentMessage,
18 ServiceState,
19 ServiceUpdateRecord,
20 } from "./cloud/serviceProtocol.ts";
21 import {
22 compareExactServiceVersions,
23 decodeServiceLauncherChildMessage,
24 isExactServiceVersion,
25 parseLegacyServiceState,
26 decodeServiceUnit,
27 disabledServiceTask,
28 encodeServiceTask,
29 parseServiceState,
30 serviceTaskIsDisabled,
31 serviceUnitBaseDirOf,
32 serviceUnitLogPathOf,
33 SERVICE_ENV_FILE,
34 SERVICE_LAUNCHER_LOCK_FILE,
35 SERVICE_SERVING_LOCK_FILE,
36 SERVICE_LAUNCHER_CONTEXT_ENV,
37 SERVICE_LAUNCHER_PROTOCOL,
38 SERVICE_STATE_FILE,
39 SERVICE_RESTART_PENDING_FILE,
40 SERVICE_STOP_MARKER_FILE,
41 SERVICE_UNIT_ENV,
42 systemdUnitDirectoryOf,
43 } from "./cloud/serviceProtocol.ts";
44 import {
45 legacyOwnerStartedBeforeRecord,
46 legacyOwnerStartTimeCommand,
47 } from "./legacyServerOwner.ts";
48
49 const HANDOFF_DELAY_MS = 2_000;
50 /**
51 * Migrations count toward it and can scale with the database: tens of GB on a cold page cache
52 * take minutes. A missed window rolls back the database, so the next attempt pays the same cost.
53 */
54 const PREPARED_TIMEOUT_MS = 30 * 60_000;
55 const TERMINATE_GRACE_MS = 5_000;
56 // Keep SQLite startup and journaling from turning a full filesystem into a restart loop.
57 const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2;
58 const DISK_SPACE_RETRY_MS = 30_000;
59 // A replacement started by a handoff waits up to two minutes for the stopping launcher.
60 const LOCK_RETRY_MS = 500;
61 const LOCK_ATTEMPTS = 240;
62 // Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect.
63 const STATE_DIR_OWNED_EXIT_CODE = 78;
64 const STATE_DIR_OWNER_POLL_MS = 30_000;
65
66 type TerminalStatus = "committed" | "rolled-back" | "failed";
67 type ChildRole = "active" | "trial";
68
69 interface ManagedChild {
70 readonly version: string;
71 role: ChildRole;
72 readonly process: NodeChildProcess.ChildProcess;
73 }
74
75 const runtimePaths = (baseDir: string, version: string) => {
76 const versionDir = NodePath.join(baseDir, "runtime", "versions", version);
77 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
78 const executableName = process.platform === "win32" ? "t3.exe" : "t3";
79 return {
80 versionDir,
81 entryPath: NodePath.join(versionDir, executableName),
82 sentinelPath: NodePath.join(versionDir, ".install-complete"),
83 };
84 };
85
86 const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
87 command: paths.entryPath,
88 args: ["serve"],
89 });
90
91 function isRegularFile(fd: number): boolean {
92 try {
93 return NodeFS.fstatSync(fd).isFile();
94 } catch {
95 return false;
96 }
97 }
98
99 const MAX_PENDING_OUTPUT_BYTES = 8 * 1024 * 1024;
100
101 /**
102 * Node writes to a regular file synchronously, so a server whose output is the log file freezes
103 * while the filesystem stalls one append. It gets a pipe instead, drained here without backpressure;
104 * output beyond the cap is dropped while a write is stuck.
105 */
106 function appendOutput(source: NodeJS.ReadableStream, fd: number): void {
107 let pending: Array<Buffer> = [];
108 let pendingBytes = 0;
109 let writing = false;
110 const flush = (): void => {
111 writing = pending.length > 0;
112 if (!writing) return;
113 const chunk = Buffer.concat(pending, pendingBytes);
114 pending = [];
115 pendingBytes = 0;
116 // Best effort, like the launcher's own diagnostics: a failed append drops that output.
117 NodeFS.write(fd, chunk, flush);
118 };
119 source.on("data", (chunk: Buffer) => {
120 if (pendingBytes + chunk.length > MAX_PENDING_OUTPUT_BYTES) return;
121 pending.push(chunk);
122 pendingBytes += chunk.length;
123 if (!writing) flush();
124 });
125 }
126
127 /**
128 * The unit or launch agent that `t3 service install` wrote. Its program names
129 * the runtime that ran the install, and a remote update never replaces it, so
130 * after a commit the launcher points it at the committed runtime. Otherwise the
131 * next start lands on a runtime that nothing in the service state records.
132 */
133 export interface ServiceUnit {
134 readonly path: string;
135 /** systemd and Task Scheduler keep a loaded copy; launchd re-reads the plist at bootstrap. */
136 readonly reload: () => Promise<void>;
137 }
138
139 const runSchtasks = (...args: ReadonlyArray<string>) =>
140 NodeUtil.promisify(NodeChildProcess.execFile)("schtasks", args, {
141 windowsHide: true,
142 timeout: 30_000,
143 });
144
145 /**
146 * Registering replaces the task, so one the CLI disabled to keep the service stopped is
147 * registered disabled. The caller holds the unit lease, as the CLI does to disable it.
148 */
149 export async function registerServiceTask(
150 unitPath: string,
151 taskName: string,
152 schtasks: (...args: ReadonlyArray<string>) => Promise<{ readonly stdout: string }> = runSchtasks,
153 ): Promise<void> {
154 const register = (xmlPath: string) => schtasks("/Create", "/XML", xmlPath, "/TN", taskName, "/F");
155 const registered = await schtasks("/Query", "/TN", taskName, "/XML");
156 if (!serviceTaskIsDisabled(registered.stdout)) {
157 await register(unitPath);
158 return;
159 }
160 const disabledPath = `${unitPath}.${process.pid}.disabled`;
161 try {
162 const unit = decodeServiceUnit(await NodeFSP.readFile(unitPath));
163 await NodeFSP.writeFile(disabledPath, encodeServiceTask(disabledServiceTask(unit)));
164 await register(disabledPath);
165 } finally {
166 await NodeFSP.rm(disabledPath, { force: true });
167 }
168 }
169
170 // The manager's environment can differ from the one it gives services (environment.d).
171 async function readSystemdUnitDirectory(): Promise<string | undefined> {
172 try {
173 const { stdout } = await NodeUtil.promisify(NodeChildProcess.execFile)(
174 "busctl",
175 [
176 "--user",
177 "--json=short",
178 "get-property",
179 "org.freedesktop.systemd1",
180 "/org/freedesktop/systemd1",
181 "org.freedesktop.systemd1.Manager",
182 "UnitPath",
183 ],
184 { timeout: 5_000 },
185 );
186 const { data } = JSON.parse(stdout) as { readonly data?: unknown };
187 return Array.isArray(data) && data.every((entry) => typeof entry === "string")
188 ? systemdUnitDirectoryOf(data)
189 : undefined;
190 } catch {
191 return undefined;
192 }
193 }
194
195 async function serviceUnitFromEnvironment(
196 env: NodeJS.ProcessEnv = process.env,
197 ): Promise<ServiceUnit | undefined> {
198 const unitName = env[SERVICE_UNIT_ENV]?.trim();
199 if (!unitName) return undefined;
200 if (unitName.endsWith(".xml")) {
201 const localAppData = env.LOCALAPPDATA?.trim();
202 if (!localAppData) return undefined;
203 const path = NodePath.win32.join(localAppData, "t3code", unitName);
204 const taskName = serviceTaskName(env.USERDOMAIN ?? "", env.USERNAME ?? "");
205 return { path, reload: () => registerServiceTask(path, taskName) };
206 }
207 const home = env.HOME?.trim();
208 if (!home) return undefined;
209 if (unitName.endsWith(".plist")) {
210 return {
211 path: NodePath.join(home, "Library", "LaunchAgents", unitName),
212 reload: () => Promise.resolve(),
213 };
214 }
215 const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
216 const unitDir =
217 (await readSystemdUnitDirectory()) ??
218 NodePath.join(
219 xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
220 ? xdgConfigHome
221 : NodePath.join(home, ".config"),
222 "systemd",
223 "user",
224 );
225 return {
226 path: NodePath.join(unitDir, unitName),
227 reload: () =>
228 new Promise((resolve, reject) => {
229 NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
230 stdio: "ignore",
231 timeout: 30_000,
232 })
233 .once("error", reject)
234 .once("exit", (code, signal) =>
235 code === 0
236 ? resolve()
237 : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
238 );
239 }),
240 };
241 }
242
243 // systemd's ExecStart value, or the plist string or task argument before __service-launcher.
244 // Only the version segment changes, so quoted or escaped paths stay as rendered.
245 // The launch agent's PATH names a versions directory without the executable; leave it.
246 const SERVICE_UNIT_PROGRAMS = [
247 /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
248 /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
249 /(<Arguments>--headless "[^"<]*?[\\/]runtime[\\/]versions[\\/])([^\\/"<]+)([\\/]t3\.exe" "__service-launcher")/,
250 ] as const;
251
252 /**
253 * The unit contents with their program pointed at `version`'s runtime. A unit
254 * already at that version or a newer one is returned unchanged: `t3 update`
255 * may have repointed it to a later version while this commit was in flight.
256 */
257 function repointServiceUnit(contents: string, version: string): string {
258 for (const program of SERVICE_UNIT_PROGRAMS) {
259 const match = program.exec(contents);
260 if (match === null) continue;
261 const [, prefix = "", current = "", suffix = ""] = match;
262 if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
263 return contents;
264 }
265 return contents.replace(program, () => `${prefix}${version}${suffix}`);
266 }
267 return contents;
268 }
269
270 const SERVICE_LOG_MAX_BYTES = 10 * 1024 * 1024;
271 const SERVICE_LOG_CHECK_INTERVAL_MS = 10 * 60_000;
272
273 /**
274 * The service manager holds the log open for appending, so it is copied aside and emptied in
275 * place rather than renamed. The copy is best effort: on a full disk the truncate frees space.
276 */
277 async function boundServiceLog(logPath: string): Promise<void> {
278 const size = await NodeFSP.stat(logPath).then(
279 (stats) => stats.size,
280 () => 0,
281 );
282 if (size <= SERVICE_LOG_MAX_BYTES) return;
283 const archivePath = `${logPath}.1`;
284 const pendingArchivePath = `${archivePath}.tmp`;
285 await NodeFSP.copyFile(logPath, pendingArchivePath)
286 .then(() => NodeFSP.rename(pendingArchivePath, archivePath))
287 .catch(() => NodeFSP.rm(pendingArchivePath, { force: true }).catch(() => undefined));
288 await NodeFSP.truncate(logPath, 0).catch(() => undefined);
289 }
290
291 async function serviceLogPath(
292 options: LauncherOptions,
293 serviceUnit: ServiceUnit | undefined,
294 ): Promise<string | undefined> {
295 if (options.logFile !== undefined) return options.logFile;
296 if (serviceUnit === undefined) return undefined;
297 const contents = await NodeFSP.readFile(serviceUnit.path)
298 .then(decodeServiceUnit)
299 .catch(() => undefined);
300 return contents === undefined ? undefined : serviceUnitLogPathOf(contents);
301 }
302
303 const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
304 const RESTORE_MARKER = ".restore-pending";
305
306 const databaseBackupDir = (baseDir: string, updateId: string) =>
307 NodePath.join(baseDir, "runtime", "db-backup", updateId);
308
309 const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) =>
310 NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`);
311
312 async function pathExists(target: string): Promise<boolean> {
313 try {
314 await NodeFSP.access(target);
315 return true;
316 } catch (cause) {
317 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false;
318 throw cause;
319 }
320 }
321
322 const MANAGED_SERVICE_ENV_NAMES = new Set([
323 "PATH",
324 "T3CODE_HOME",
325 "T3_BOOT_SERVICE_UNIT",
326 SERVICE_LAUNCHER_CONTEXT_ENV,
327 ]);
328
329 export async function readServiceEnvironment(
330 baseDir: string,
331 inherited: NodeJS.ProcessEnv = process.env,
332 ): Promise<NodeJS.ProcessEnv> {
333 const environment = { ...inherited };
334 let contents: string;
335 try {
336 contents = await NodeFSP.readFile(NodePath.join(baseDir, SERVICE_ENV_FILE), "utf8");
337 } catch (cause) {
338 if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return environment;
339 throw cause;
340 }
341 for (const [name, value] of Object.entries(NodeUtil.parseEnv(contents))) {
342 if (
343 /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
344 !MANAGED_SERVICE_ENV_NAMES.has(name.toUpperCase())
345 ) {
346 environment[name] = value;
347 }
348 }
349 return environment;
350 }
351
352 // Opened read-write: Windows refuses to flush a handle without write access.
353 async function syncFile(filePath: string): Promise<void> {
354 const handle = await NodeFSP.open(filePath, "r+");
355 try {
356 await handle.sync();
357 } finally {
358 await handle.close();
359 }
360 }
361
362 // Fsync makes renames durable; Windows returns EPERM and NTFS journals them itself.
363 async function syncDirectory(directory: string): Promise<void> {
364 const handle = await NodeFSP.open(directory, "r");
365 try {
366 await handle.sync();
367 } catch (error) {
368 if ((error as NodeJS.ErrnoException).code !== "EPERM") throw error;
369 } finally {
370 await handle.close();
371 }
372 }
373
374 function runCp(args: ReadonlyArray<string>, signal?: AbortSignal): Promise<boolean> {
375 return new Promise((resolve) => {
376 NodeChildProcess.execFile("cp", args, { signal }, (error) => resolve(error === null));
377 });
378 }
379
380 // macOS fs.copyFile clone flags fall back to full copies.
381 async function copyDatabaseFile(
382 source: string,
383 destination: string,
384 signal?: AbortSignal,
385 ): Promise<void> {
386 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
387 const platform = process.platform;
388 // cp -c replaces files and breaks hard links; copy into hard-linked destinations instead.
389 const hardLinked =
390 platform === "darwin" &&
391 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
392 const args =
393 platform === "darwin" && !hardLinked
394 ? ["-c", source, destination]
395 : platform === "linux"
396 ? ["--reflink=auto", source, destination]
397 : undefined;
398 if (args !== undefined && (await runCp(args, signal))) return;
399 signal?.throwIfAborted();
400 await NodeFSP.copyFile(source, destination);
401 }
402
403 const GIB = 1024 ** 3;
404 // Cloned backups still need headroom for trial writes and migrations to diverge.
405 const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
406 const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
407 // While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
408 const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
409
410 async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
411 const stats: Array<NodeFS.Stats> = [];
412 for (const suffix of DB_FILE_SUFFIXES) {
413 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
414 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
415 throw cause;
416 });
417 if (stat !== undefined) stats.push(stat);
418 }
419 return stats;
420 }
421
422 // cp -c silently copies across volumes; require one device matching the APFS system volume's filesystem type.
423 async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
424 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
425 const platform = process.platform;
426 if (platform === "darwin") {
427 const [backupDirectory, backupFs, systemFs] = await Promise.all([
428 NodeFSP.stat(backupRoot),
429 NodeFSP.statfs(backupRoot),
430 NodeFSP.statfs("/"),
431 ]);
432 if (backupFs.type !== systemFs.type) return false;
433 return (await statDatabaseFiles(dbPath)).every(
434 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
435 );
436 }
437 if (platform !== "linux") return false;
438 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
439 try {
440 for (const suffix of DB_FILE_SUFFIXES) {
441 const source = `${dbPath}${suffix}`;
442 if (suffix !== "" && !(await pathExists(source))) continue;
443 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
444 }
445 return true;
446 } finally {
447 for (const suffix of DB_FILE_SUFFIXES) {
448 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
449 }
450 }
451 }
452
453 export interface BackupSpaceProbe {
454 readonly readFreeBytes: (directory: string) => Promise<number>;
455 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
456 }
457
458 const filesystemBackupSpaceProbe: BackupSpaceProbe = {
459 readFreeBytes: async (directory) => {
460 const stats = await NodeFSP.statfs(directory);
461 return stats.bavail * stats.bsize;
462 },
463 canClone: backupCanClone,
464 };
465
466 // A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
467 // Update requests are refused while one is pending, so no staging directory is still in use here.
468 async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
469 for (const entry of await NodeFSP.readdir(backupRoot)) {
470 if (!entry.endsWith(".staging")) continue;
471 await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
472 () => undefined,
473 );
474 }
475 }
476
477 /** Check before stopping the server so full disks refuse updates without an outage. */
478 export async function databaseBackupBlocker(
479 baseDir: string,
480 dbPath: string,
481 probe: Partial<BackupSpaceProbe> = {},
482 ): Promise<string | undefined> {
483 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
484 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
485 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
486 if (await isNoCopyOnWrite(dbPath)) await markNoCopyOnWrite(backupRoot);
487 await removeAbandonedBackupCopies(backupRoot);
488 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
489 const size = files.reduce((total, file) => total + file.size, 0);
490 const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
491 if (free >= fullCopyBytes) return undefined;
492 const clonedBytes = Math.max(
493 CLONED_BACKUP_MIN_HEADROOM_BYTES,
494 size * CLONED_BACKUP_HEADROOM_FRACTION,
495 );
496 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
497 if (free >= needed) return undefined;
498 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
499 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
500 }
501
502 // A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
503 async function copyWithoutOpening(source: string, destination: string): Promise<void> {
504 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
505 if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
506 if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
507 }
508
509 // Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
510 // which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
511 // `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
512 // a write-heavy server's new writes wait, uncommitted, until the server stops.
513 async function startOnlineDatabaseBackup(
514 baseDir: string,
515 pending: PendingServiceUpdate,
516 probe: Partial<BackupSpaceProbe> = {},
517 ): Promise<(() => Promise<void>) | undefined> {
518 const backupDir = databaseBackupDir(baseDir, pending.id);
519 if (await pathExists(backupDir)) return undefined;
520 const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
521 if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
522 return undefined;
523 const NodeSqlite = await import("node:sqlite").catch(() => undefined);
524 if (NodeSqlite === undefined) return undefined;
525
526 const stagingDir = `${backupDir}.staging`;
527 const discardStaging = () =>
528 NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
529 const fallBack = async (cause: unknown) => {
530 await discardStaging();
531 process.stderr.write(
532 `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
533 );
534 return undefined;
535 };
536 let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
537 // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
538 let dbPath = pending.dbPath;
539 const release = () => {
540 try {
541 snapshot?.close();
542 } catch {}
543 snapshot = undefined;
544 };
545 try {
546 await discardStaging();
547 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
548 dbPath = await NodeFSP.realpath(pending.dbPath);
549 snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
550 const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
551 if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
552 snapshot.exec("BEGIN");
553 snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
554 const mainCopy = databaseBackupFile(stagingDir, "");
555 await copyWithoutOpening(dbPath, mainCopy);
556 await syncFile(mainCopy);
557 } catch (cause) {
558 release();
559 return fallBack(cause);
560 }
561 return async () => {
562 try {
563 const wal = `${dbPath}-wal`;
564 if (!(await pathExists(wal)))
565 throw new Error("the WAL disappeared while the snapshot was open.");
566 const walCopy = databaseBackupFile(stagingDir, "-wal");
567 await copyWithoutOpening(wal, walCopy);
568 await syncFile(walCopy);
569 release();
570 await NodeFSP.rename(stagingDir, backupDir);
571 await syncDirectory(NodePath.dirname(backupDir));
572 } catch (cause) {
573 release();
574 await fallBack(cause);
575 }
576 };
577 }
578
579 // Preserve completed backups; restarted trials may already have changed the live database.
580 async function backupDatabaseOnce(
581 baseDir: string,
582 pending: PendingServiceUpdate,
583 signal: AbortSignal,
584 ): Promise<void> {
585 const backupDir = databaseBackupDir(baseDir, pending.id);
586 if (await pathExists(backupDir)) return;
587
588 const stagingDir = `${backupDir}.staging`;
589 await NodeFSP.rm(stagingDir, { recursive: true, force: true });
590 await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
591 try {
592 for (const suffix of DB_FILE_SUFFIXES) {
593 signal.throwIfAborted();
594 const source = `${pending.dbPath}${suffix}`;
595 if (suffix !== "" && !(await pathExists(source))) continue;
596 const destination = databaseBackupFile(stagingDir, suffix);
597 await copyDatabaseFile(source, destination, signal);
598 await syncFile(destination);
599 }
600 await NodeFSP.rename(stagingDir, backupDir);
601 await syncDirectory(NodePath.dirname(backupDir));
602 } catch (cause) {
603 await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
604 throw cause;
605 }
606 }
607
608 const restoreMarkerPath = (baseDir: string, updateId: string) =>
609 NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER);
610
611 const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) =>
612 pathExists(restoreMarkerPath(baseDir, pending.id));
613
614 // Mark before live writes so recovery cannot boot a partial restore.
615 async function markDatabaseRestorePending(backupDir: string): Promise<void> {
616 const markerPath = NodePath.join(backupDir, RESTORE_MARKER);
617 if (!(await pathExists(markerPath))) {
618 const handle = await NodeFSP.open(markerPath, "wx", 0o600);
619 try {
620 await handle.sync();
621 } finally {
622 await handle.close();
623 }
624 await syncDirectory(backupDir);
625 }
626 }
627
628 // Resolve symlink targets even if a crashed cp -c left the final target missing.
629 async function restoreTarget(path: string): Promise<string> {
630 let current = path;
631 for (let hops = 0; hops < 40; hops++) {
632 try {
633 return await NodeFSP.realpath(current);
634 } catch (cause) {
635 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
636 }
637 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
638 const code = (cause as NodeJS.ErrnoException).code;
639 if (code === "ENOENT" || code === "EINVAL") return undefined;
640 throw cause;
641 });
642 if (link === undefined) return current;
643 // Do not normalize .. before following directory symlinks; match filesystem resolution.
644 current = NodePath.isAbsolute(link)
645 ? link
646 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
647 }
648 throw new Error(`Too many symbolic links resolving ${path}.`);
649 }
650
651 async function restoreDatabaseBackup(
652 baseDir: string,
653 pending: PendingServiceUpdate,
654 ): Promise<void> {
655 const backupDir = databaseBackupDir(baseDir, pending.id);
656 if (!(await pathExists(backupDir))) return;
657
658 await markDatabaseRestorePending(backupDir);
659 const directories = new Set([NodePath.dirname(pending.dbPath)]);
660 // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
661 const mainTarget = await restoreTarget(pending.dbPath);
662 for (const suffix of DB_FILE_SUFFIXES) {
663 const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
664 if (livePath !== `${pending.dbPath}${suffix}`) {
665 await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
666 }
667 const source = databaseBackupFile(backupDir, suffix);
668 if (await pathExists(source)) {
669 // Restore into the symlink target; cloning would replace the link.
670 const target = await restoreTarget(livePath);
671 await copyDatabaseFile(source, target);
672 await syncFile(target);
673 directories.add(NodePath.dirname(target));
674 } else {
675 await NodeFSP.rm(livePath, { force: true });
676 }
677 }
678 for (const directory of directories) await syncDirectory(directory);
679 }
680
681 async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
682 const backupDir = databaseBackupDir(baseDir, updateId);
683 if (!(await pathExists(backupDir))) return;
684 await NodeFSP.rm(backupDir, { recursive: true, force: true });
685 await syncDirectory(NodePath.dirname(backupDir));
686 }
687
688 export async function readServiceState(filePath: string): Promise<ServiceState> {
689 const contents = await NodeFSP.readFile(filePath, "utf8");
690 const state = parseServiceState(contents) ?? adoptLegacyServiceState(contents);
691 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
692 return state;
693 }
694
695 // An older coordinator can write protocol 2 beside a newer runtime. Adopt
696 // only its install record, in memory: rewriting it outside the selection lock
697 // could replace state a concurrent install wrote after this read.
698 function adoptLegacyServiceState(contents: string): ServiceState | undefined {
699 const legacy = parseLegacyServiceState(contents);
700 return legacy && { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: legacy.activeVersion };
701 }
702
703 export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
704 await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
705 await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
706 }
707
708 async function replaceFileDurably(filePath: string, contents: string | Uint8Array): Promise<void> {
709 const directory = NodePath.dirname(filePath);
710 const tempPath = NodePath.join(
711 directory,
712 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
713 );
714 let handle: NodeFSP.FileHandle | undefined;
715 try {
716 handle = await NodeFSP.open(tempPath, "wx", 0o600);
717 await handle.writeFile(contents);
718 await handle.sync();
719 await handle.close();
720 handle = undefined;
721 await NodeFSP.rename(tempPath, filePath);
722 await syncDirectory(directory);
723 } finally {
724 await handle?.close().catch(() => undefined);
725 await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined);
726 }
727 }
728
729 async function runtimeExists(baseDir: string, version: string): Promise<boolean> {
730 const paths = runtimePaths(baseDir, version);
731 try {
732 const [entry, sentinel] = await Promise.all([
733 NodeFSP.stat(paths.entryPath),
734 NodeFSP.readFile(paths.sentinelPath, "utf8"),
735 ]);
736 return entry.isFile() && sentinel.trim() === version;
737 } catch {
738 return false;
739 }
740 }
741
742 function terminalUpdate<S extends TerminalStatus>(input: {
743 readonly pending: PendingServiceUpdate;
744 readonly status: S;
745 readonly reason?: string;
746 }): Exclude<ServiceUpdateRecord, PendingServiceUpdate> & { readonly status: S } {
747 return {
748 id: input.pending.id,
749 fromVersion: input.pending.fromVersion,
750 targetVersion: input.pending.targetVersion,
751 status: input.status,
752 ...(input.reason === undefined ? {} : { reason: input.reason }),
753 };
754 }
755
756 function sendMessage(
757 child: NodeChildProcess.ChildProcess,
758 message: ServiceLauncherParentMessage,
759 ): Promise<void> {
760 return new Promise((resolve, reject) => {
761 if (!child.connected || child.send === undefined) {
762 reject(new Error("service child IPC is disconnected."));
763 return;
764 }
765 child.send(message, (error) => (error === null ? resolve() : reject(error)));
766 });
767 }
768
769 function waitForExit(child: NodeChildProcess.ChildProcess): Promise<void> {
770 if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
771 return new Promise((resolve) => child.once("exit", () => resolve()));
772 }
773
774 async function terminateChild(
775 child: NodeChildProcess.ChildProcess,
776 signal: NodeJS.Signals,
777 overIpc: boolean,
778 ): Promise<void> {
779 if (child.exitCode !== null || child.signalCode !== null) return;
780 if (overIpc) {
781 await sendMessage(child, { type: "stop" }).catch(() => child.kill(signal));
782 } else {
783 child.kill(signal);
784 }
785 const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS);
786 try {
787 await waitForExit(child);
788 } finally {
789 clearTimeout(force);
790 }
791 }
792
793 /** Task Scheduler names are machine-wide, so each Windows account runs its own task. */
794 export const serviceTaskName = (domain: string, userName: string) => `t3code-${domain}-${userName}`;
795
796 export const serviceLauncherLockPath = (baseDir: string) =>
797 NodePath.join(baseDir, "runtime", SERVICE_LAUNCHER_LOCK_FILE);
798
799 // An exclusive SQLite transaction, like the server's owner lock: the OS drops it when the holder dies.
800 async function tryAcquireSqliteLock(
801 lockPath: string,
802 busyTimeoutMs = 0,
803 ): Promise<{ readonly close: () => void } | undefined> {
804 const { DatabaseSync } = await import("node:sqlite");
805 const db = new DatabaseSync(lockPath);
806 try {
807 db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; BEGIN EXCLUSIVE;`);
808 return db;
809 } catch (cause) {
810 db.close();
811 if (cause instanceof Error && "errcode" in cause && cause.errcode === 5) return undefined;
812 throw cause;
813 }
814 }
815
816 export async function serviceLauncherIsRunning(lockPath: string): Promise<boolean> {
817 if (!NodeFS.existsSync(lockPath)) return false;
818 const lock = await tryAcquireSqliteLock(lockPath);
819 lock?.close();
820 return lock === undefined;
821 }
822
823 const servingLockPath = (baseDir: string) =>
824 NodePath.join(baseDir, "runtime", SERVICE_SERVING_LOCK_FILE);
825
826 /** Whether a launcher has, or is about to start, a server for this home. An idle one does not. */
827 export const serviceLauncherIsServing = (baseDir: string): Promise<boolean> =>
828 serviceLauncherIsRunning(servingLockPath(baseDir)).catch(() => false);
829
830 /** Resolves once no launcher holds the lock; the caller bounds the wait through `signal`. */
831 export async function waitForServiceLauncherExit(
832 lockPath: string,
833 signal: AbortSignal,
834 ): Promise<void> {
835 while (await serviceLauncherIsRunning(lockPath)) {
836 signal.throwIfAborted();
837 await new Promise((resolve) => setTimeout(resolve, 100));
838 }
839 }
840
841 // Mirror acquireServerOwnership: the lock, then a live pre-lock server-runtime.json owner;
842 // leave other failures for server startup to report.
843 export async function stateDirOwned(baseDir: string): Promise<boolean> {
844 const stateDir = NodePath.join(baseDir, "userdata");
845 const lock = await tryAcquireSqliteLock(NodePath.join(stateDir, "server-owner.sqlite")).catch(
846 () => null,
847 );
848 if (lock === null) return false;
849 if (lock === undefined) return true;
850 lock.close();
851 let record: unknown;
852 try {
853 record = JSON.parse(
854 await NodeFSP.readFile(NodePath.join(stateDir, "server-runtime.json"), "utf8"),
855 );
856 } catch {
857 return false;
858 }
859 if (typeof record !== "object" || record === null || "ownerId" in record) return false;
860 const pid = "pid" in record ? record.pid : undefined;
861 if (typeof pid !== "number" || !Number.isInteger(pid) || pid <= 0) return false;
862 try {
863 process.kill(pid, 0);
864 } catch (error) {
865 if (!(error instanceof Error && "code" in error && error.code === "EPERM")) return false;
866 }
867 const recordedAt = Date.parse(
868 "startedAt" in record && typeof record.startedAt === "string" ? record.startedAt : "",
869 );
870 if (!Number.isFinite(recordedAt)) return true;
871 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
872 const platform = process.platform;
873 const { command, args, env } = legacyOwnerStartTimeCommand(pid, platform);
874 const output = await new Promise<string | undefined>((resolve) =>
875 NodeChildProcess.execFile(
876 command,
877 args,
878 { env: { ...process.env, ...env }, timeout: 2_000, maxBuffer: 16_384 },
879 (error, stdout) => resolve(error === null ? stdout : undefined),
880 ),
881 );
882 return legacyOwnerStartedBeforeRecord(output, recordedAt, platform);
883 }
884
885 const stopMarkerPath = (baseDir: string) =>
886 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
887 const restartPendingPath = (baseDir: string) =>
888 NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE);
889
890 export class Launcher {
891 readonly #baseDir: string;
892 readonly #statePath: string;
893 #state: ServiceState;
894 #child: ManagedChild | null = null;
895 #timer: NodeJS.Timeout | undefined;
896 #transitions: Promise<void> = Promise.resolve();
897 #stopRequested = false;
898 readonly #stopSignal = new AbortController();
899 #stopping = false;
900 #done = false;
901 readonly #completion = Promise.withResolvers<void>();
902 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
903 readonly #serviceUnit: ServiceUnit | undefined;
904 readonly #environment: Readonly<NodeJS.ProcessEnv>;
905 readonly #stateDirOwnerPollMs: number;
906 #stateDirFreeProbes = 0;
907 readonly #output: number | undefined;
908 readonly #lockPath: string | undefined;
909 #servingLock: { readonly close: () => void } | undefined;
910 readonly #stopChildOverIpc: boolean;
911
912 constructor(
913 baseDir: string,
914 state: ServiceState,
915 options: {
916 readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
917 readonly serviceUnit?: ServiceUnit;
918 readonly environment?: Readonly<NodeJS.ProcessEnv>;
919 readonly stateDirOwnerPollMs?: number;
920 /** Launcher and server output, for managers that cannot redirect it. */
921 readonly output?: number;
922 /** Makes this the only launcher for its home; a replacement waits for the one stopping. */
923 readonly lockPath?: string;
924 /** Windows `kill()` is TerminateProcess, which skips the server's shutdown finalizers. */
925 readonly stopChildOverIpc?: boolean;
926 } = {},
927 ) {
928 this.#baseDir = baseDir;
929 this.#stateDirOwnerPollMs = options.stateDirOwnerPollMs ?? STATE_DIR_OWNER_POLL_MS;
930 this.#backupSpaceProbe = options.backupSpaceProbe;
931 this.#serviceUnit = options.serviceUnit;
932 this.#environment = { ...(options.environment ?? process.env) };
933 this.#output = options.output;
934 this.#lockPath = options.lockPath;
935 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
936 this.#stopChildOverIpc = options.stopChildOverIpc ?? process.platform === "win32";
937 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
938 this.#state = state;
939 }
940
941 async run(): Promise<void> {
942 const onSigterm = () => void this.stop("SIGTERM");
943 const onSigint = () => void this.stop("SIGINT");
944 process.once("SIGTERM", onSigterm);
945 process.once("SIGINT", onSigint);
946 try {
947 const lock =
948 this.#lockPath === undefined ? undefined : await this.#acquireLock(this.#lockPath);
949 try {
950 // The launcher that held the lock may have moved the state since main read it.
951 if (lock !== undefined) this.#state = await readServiceState(this.#statePath);
952 // A stop that arrived before this point leaves nothing to start or recover.
953 if (!this.#stopRequested) this.#enqueue(() => this.#recover());
954 await this.#completion.promise;
955 } finally {
956 if (lock !== undefined) {
957 // Releasing the lock says this launcher is done with the home.
958 await this.#transitions.catch(() => undefined);
959 lock.close();
960 }
961 }
962 } finally {
963 this.#releaseServing();
964 process.off("SIGTERM", onSigterm);
965 process.off("SIGINT", onSigint);
966 }
967 }
968
969 async #acquireLock(lockPath: string): Promise<{ readonly close: () => void } | undefined> {
970 for (let attempt = 1; !this.#stopRequested; attempt++) {
971 const lock = await tryAcquireSqliteLock(lockPath);
972 if (lock !== undefined) return lock;
973 if (attempt >= LOCK_ATTEMPTS) {
974 throw new Error("Another T3 Code service launcher still runs for this T3 home.");
975 }
976 await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_MS));
977 }
978 return undefined;
979 }
980
981 // Advisory: without it an SSH launch may start its own server, as it does beside no service.
982 async #claimServing(): Promise<void> {
983 // An SSH launch probes this lock for a moment; wait out that probe.
984 this.#servingLock ??= await tryAcquireSqliteLock(servingLockPath(this.#baseDir), 250).catch(
985 () => undefined,
986 );
987 }
988
989 #releaseServing(): void {
990 this.#servingLock?.close();
991 this.#servingLock = undefined;
992 }
993
994 #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") {
995 return terminateChild(child, signal, this.#stopChildOverIpc);
996 }
997
998 // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space.
999 #log(line: string): void {
1000 try {
1001 if (this.#output === undefined) process.stderr.write(line);
1002 else NodeFS.writeSync(this.#output, line);
1003 } catch {}
1004 }
1005
1006 #enqueue(transition: () => Promise<void>): void {
1007 this.#transitions = this.#transitions
1008 .then(transition, transition)
1009 .catch((cause: unknown) =>
1010 this.#fatal(cause instanceof Error ? cause : new Error(String(cause))),
1011 );
1012 }
1013
1014 async #fatal(error: Error): Promise<void> {
1015 if (this.#done) return;
1016 this.#done = true;
1017 this.#stopping = true;
1018 this.#clearTimer();
1019 const child = this.#child?.process;
1020 this.#child = null;
1021 if (child !== undefined) await this.#terminate(child);
1022 this.#completion.reject(error);
1023 }
1024
1025 async stop(signal: NodeJS.Signals): Promise<void> {
1026 // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer.
1027 try {
1028 NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 });
1029 } catch {}
1030 if (this.#stopRequested || this.#stopping) {
1031 await this.#completion.promise.catch(() => undefined);
1032 return;
1033 }
1034 this.#stopRequested = true;
1035 this.#stopSignal.abort();
1036 this.#clearTimer();
1037 this.#enqueue(async () => {
1038 // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup.
1039 this.#stopping = true;
1040 // An exit-78 transition queued before this one may have started the owner poll.
1041 this.#clearTimer();
1042 const child = this.#child?.process;
1043 this.#child = null;
1044 if (child !== undefined) await this.#terminate(child, signal);
1045 this.#done = true;
1046 this.#completion.resolve();
1047 });
1048 await this.#completion.promise.catch(() => undefined);
1049 }
1050
1051 #clearTimer(): void {
1052 clearTimeout(this.#timer);
1053 this.#timer = undefined;
1054 }
1055
1056 async #serviceStartDiskSpaceBlocker(): Promise<string | undefined> {
1057 const databasePath =
1058 this.#state.update?.status === "pending"
1059 ? this.#state.update.dbPath
1060 : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite");
1061 const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath);
1062 const freeBytes = async (directory: string): Promise<number> => {
1063 try {
1064 return await filesystemBackupSpaceProbe.readFreeBytes(directory);
1065 } catch (cause) {
1066 const parent = NodePath.dirname(directory);
1067 if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) {
1068 return freeBytes(parent);
1069 }
1070 return Number.POSITIVE_INFINITY;
1071 }
1072 };
1073 const available = await Promise.all([
1074 freeBytes(this.#baseDir),
1075 freeBytes(NodePath.dirname(resolvedDatabasePath)),
1076 ]);
1077 const free = Math.min(...available);
1078 return free < MIN_SERVICE_START_FREE_BYTES
1079 ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.`
1080 : undefined;
1081 }
1082
1083 async #deferForDiskSpace(retry: () => Promise<void>): Promise<boolean> {
1084 const reason = await this.#serviceStartDiskSpaceBlocker();
1085 if (reason === undefined) return false;
1086 // A stop during the probe must not arm a retry that outlives it.
1087 if (this.#stopRequested) return true;
1088 this.#clearTimer();
1089 this.#releaseServing();
1090 this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`);
1091 this.#timer = setInterval(
1092 () =>
1093 this.#enqueue(async () => {
1094 if (this.#child !== null || this.#stopping) return;
1095 if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return;
1096 this.#clearTimer();
1097 await this.#claimServing();
1098 await retry();
1099 }),
1100 DISK_SPACE_RETRY_MS,
1101 );
1102 return true;
1103 }
1104
1105 async #selectState(next: ServiceState): Promise<void> {
1106 const release = await acquireServiceSelectionLock(this.#statePath);
1107 try {
1108 const selected = await readServiceState(this.#statePath);
1109 // Local installs replace the selection and clear its update record. An old
1110 // launcher must not restore its own selection when asked to update again.
1111 if (
1112 selected.activeVersion !== this.#state.activeVersion ||
1113 selected.update?.id !== this.#state.update?.id ||
1114 selected.update?.status !== this.#state.update?.status
1115 ) {
1116 throw new Error("Service selection changed. Restart the service before updating.");
1117 }
1118 await writeServiceState(this.#statePath, next);
1119 this.#state = next;
1120 } finally {
1121 release();
1122 }
1123 }
1124
1125 async #recover(): Promise<void> {
1126 await this.#claimServing();
1127 // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
1128 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
1129 const restartPending = restartPendingPath(this.#baseDir);
1130 try {
1131 const release = await acquireServiceSelectionLock(this.#statePath);
1132 try {
1133 const selected = await readServiceState(this.#statePath);
1134 const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
1135 () => undefined,
1136 );
1137 const awaited = awaitedVersion?.trim();
1138 if (
1139 selected.activeVersion === this.#state.activeVersion &&
1140 selected.update?.id === this.#state.update?.id &&
1141 selected.update?.status === this.#state.update?.status &&
1142 awaited !== undefined &&
1143 isExactServiceVersion(awaited) &&
1144 compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
1145 ) {
1146 await NodeFSP.rm(restartPending, { force: true });
1147 }
1148 } finally {
1149 release();
1150 }
1151 } catch (error) {
1152 process.stderr.write(
1153 `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
1154 );
1155 }
1156 const update = this.#state.update;
1157 if (update?.status !== "pending") {
1158 if (update !== undefined) {
1159 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
1160 }
1161 try {
1162 await this.#startChild(this.#state.activeVersion, "active", update);
1163 } finally {
1164 if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
1165 }
1166 return;
1167 }
1168 if (await databaseRestorePending(this.#baseDir, update)) {
1169 await this.#returnToPrevious(update, "failed", "rollback-interrupted");
1170 return;
1171 }
1172 if (!(await runtimeExists(this.#baseDir, update.targetVersion))) {
1173 await this.#returnToPrevious(update, "failed", "target-runtime-missing");
1174 return;
1175 }
1176 await this.#startTrial(update);
1177 }
1178
1179 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
1180 if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return;
1181 try {
1182 await backupDatabaseOnce(this.#baseDir, pending, this.#stopSignal.signal);
1183 } catch {
1184 // A stop aborts the copy and leaves the update pending for the next start.
1185 if (this.#stopRequested) return;
1186 await this.#returnToPrevious(pending, "failed", "db-backup-failed");
1187 return;
1188 }
1189 if (this.#stopRequested) return;
1190 try {
1191 await this.#startChild(pending.targetVersion, "trial", pending);
1192 } catch {
1193 await this.#returnToPrevious(pending, "failed", "candidate-start-failed");
1194 }
1195 }
1196
1197 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
1198 if (this.#stopping) return;
1199 if (
1200 await this.#deferForDiskSpace(() =>
1201 role === "trial" && update?.status === "pending"
1202 ? this.#startTrial(update)
1203 : this.#startChild(version, role, update),
1204 )
1205 )
1206 return;
1207 if (!(await runtimeExists(this.#baseDir, version))) {
1208 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
1209 }
1210 if (this.#stopping) return;
1211 const paths = runtimePaths(this.#baseDir, version);
1212 const context: ServiceLauncherContext = {
1213 protocol: SERVICE_LAUNCHER_PROTOCOL,
1214 childVersion: version,
1215 ...(update === undefined ? {} : { update }),
1216 };
1217 const spawnArguments = runtimeSpawnArguments(paths);
1218 const appendStdout = this.#output === undefined && isRegularFile(1);
1219 const appendStderr = this.#output === undefined && isRegularFile(2);
1220 const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, {
1221 env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) },
1222 stdio:
1223 this.#output === undefined
1224 ? ["inherit", appendStdout ? "pipe" : "inherit", appendStderr ? "pipe" : "inherit", "ipc"]
1225 : ["ignore", this.#output, this.#output, "ipc"],
1226 });
1227 if (child.stdout !== null) appendOutput(child.stdout, 1);
1228 if (child.stderr !== null) appendOutput(child.stderr, 2);
1229 await new Promise<void>((resolve, reject) => {
1230 const onError = (error: Error) => reject(error);
1231 child.once("error", onError);
1232 child.once("spawn", () => {
1233 child.removeListener("error", onError);
1234 child.on("error", (error) => this.#enqueue(() => Promise.reject(error)));
1235 resolve();
1236 });
1237 });
1238 if (this.#stopping) {
1239 await this.#terminate(child);
1240 return;
1241 }
1242
1243 const managed: ManagedChild = {
1244 version,
1245 role,
1246 process: child,
1247 };
1248 this.#child = managed;
1249 child.on("message", (value) => {
1250 const message = decodeServiceLauncherChildMessage(value);
1251 if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message));
1252 });
1253 child.once("exit", (code, signal) =>
1254 this.#enqueue(() => this.#handleExit(managed, code, signal)),
1255 );
1256
1257 if (role === "trial") {
1258 this.#timer = setTimeout(
1259 () => this.#enqueue(() => this.#handlePreparedTimeout(managed)),
1260 PREPARED_TIMEOUT_MS,
1261 );
1262 }
1263 }
1264
1265 async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise<void> {
1266 if (this.#child !== child || this.#stopping) return;
1267 if (message.type === "request-update") {
1268 await this.#handleUpdateRequest(child, message);
1269 return;
1270 }
1271 await this.#handlePrepared(child, message.updateId);
1272 }
1273
1274 async #handleUpdateRequest(
1275 child: ManagedChild,
1276 message: Extract<ServiceLauncherChildMessage, { readonly type: "request-update" }>,
1277 ): Promise<void> {
1278 const reject = (reason: string) =>
1279 sendMessage(child.process, { type: "update-rejected", reason });
1280 if (child.role !== "active") {
1281 await reject("Only the active server can request an update.");
1282 return;
1283 }
1284 if (child.version !== this.#state.activeVersion) {
1285 await reject("The requesting server is not the selected active version.");
1286 return;
1287 }
1288 if (this.#state.update?.status === "pending") {
1289 await reject("Another server update is already pending.");
1290 return;
1291 }
1292 if (!isExactServiceVersion(message.targetVersion)) {
1293 await reject("The requested target is not an exact version.");
1294 return;
1295 }
1296 if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) {
1297 await reject("Remote updates must select a newer server version.");
1298 return;
1299 }
1300 if (!NodePath.isAbsolute(message.dbPath)) {
1301 await reject("The requested database path is not absolute.");
1302 return;
1303 }
1304 if (!(await runtimeExists(this.#baseDir, message.targetVersion))) {
1305 await reject("The requested target runtime is missing or incomplete.");
1306 return;
1307 }
1308 // Failed checks must refuse updates rather than stop the server with unknown backup capacity.
1309 const backupBlocker = await databaseBackupBlocker(
1310 this.#baseDir,
1311 message.dbPath,
1312 this.#backupSpaceProbe,
1313 ).catch(
1314 (error: unknown) =>
1315 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
1316 );
1317 if (!child.process.connected) return;
1318 if (backupBlocker !== undefined) {
1319 await reject(backupBlocker);
1320 return;
1321 }
1322
1323 const pending: PendingServiceUpdate = {
1324 id: NodeCrypto.randomUUID(),
1325 fromVersion: child.version,
1326 targetVersion: message.targetVersion,
1327 dbPath: message.dbPath,
1328 status: "pending",
1329 };
1330 const next: ServiceState = { ...this.#state, update: pending };
1331 try {
1332 await this.#selectState(next);
1333 } catch (error) {
1334 await reject(error instanceof Error ? error.message : String(error));
1335 return;
1336 }
1337 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
1338 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
1339 }
1340
1341 async #beginTrial(child: ManagedChild): Promise<void> {
1342 const pending = this.#state.update;
1343 if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") {
1344 return;
1345 }
1346 this.#timer = undefined;
1347 // Anything thrown outside its own fallback still leaves the offline backup to run.
1348 const finishOnlineBackup = await startOnlineDatabaseBackup(
1349 this.#baseDir,
1350 pending,
1351 this.#backupSpaceProbe,
1352 ).catch(() => undefined);
1353 this.#child = null;
1354 await this.#terminate(child.process);
1355 await finishOnlineBackup?.();
1356 await this.#startTrial(pending);
1357 }
1358
1359 async #handlePrepared(child: ManagedChild, updateId: string): Promise<void> {
1360 const pending = this.#state.update;
1361 if (
1362 child.role !== "trial" ||
1363 pending?.status !== "pending" ||
1364 pending.id !== updateId ||
1365 pending.targetVersion !== child.version
1366 ) {
1367 if (child.role === "trial" && pending?.status === "pending") {
1368 await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child);
1369 return;
1370 }
1371 throw new Error("Trial child reported prepared for an unexpected update.");
1372 }
1373 this.#clearTimer();
1374 const committed = terminalUpdate({ pending, status: "committed" });
1375 const next: ServiceState = {
1376 ...this.#state,
1377 activeVersion: pending.targetVersion,
1378 update: committed,
1379 };
1380 await this.#selectState(next);
1381 child.role = "active";
1382 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
1383 try {
1384 await sendMessage(child.process, { type: "committed", updateId: committed.id });
1385 } finally {
1386 await this.#repointServiceUnit(committed.targetVersion);
1387 }
1388 }
1389
1390 // Selection is durable before this best-effort unit reconciliation.
1391 async #repointServiceUnit(version: string): Promise<void> {
1392 const unit = this.#serviceUnit;
1393 if (unit === undefined) return;
1394 try {
1395 const release = await acquireServiceSelectionLock(unit.path);
1396 try {
1397 // The installer writes state before the unit under this lease. Do not
1398 // acquire its state lease here: the installer takes that lease first.
1399 const selected = await readServiceState(this.#statePath);
1400 if (
1401 selected.activeVersion !== this.#state.activeVersion ||
1402 selected.update?.id !== this.#state.update?.id ||
1403 selected.update?.status !== this.#state.update?.status
1404 ) {
1405 return;
1406 }
1407 const contents = decodeServiceUnit(await NodeFSP.readFile(unit.path));
1408 const installedBaseDir = serviceUnitBaseDirOf(contents);
1409 if (
1410 installedBaseDir === undefined ||
1411 NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
1412 ) {
1413 return;
1414 }
1415 const repointed = repointServiceUnit(contents, version);
1416 if (repointed !== contents) {
1417 await replaceFileDurably(
1418 unit.path,
1419 unit.path.endsWith(".xml") ? encodeServiceTask(repointed) : repointed,
1420 );
1421 }
1422 // A previous rename may have succeeded before reload failed.
1423 await unit.reload();
1424 } finally {
1425 release();
1426 }
1427 } catch (error) {
1428 process.stderr.write(
1429 `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1430 );
1431 }
1432 }
1433
1434 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
1435 const pending = this.#state.update;
1436 if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") {
1437 return;
1438 }
1439 this.#timer = undefined;
1440 await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child);
1441 }
1442
1443 async #handleExit(
1444 child: ManagedChild,
1445 code: number | null,
1446 signal: NodeJS.Signals | null,
1447 ): Promise<void> {
1448 if (this.#child !== child || this.#stopping) return;
1449 this.#child = null;
1450 if (code === STATE_DIR_OWNED_EXIT_CODE) {
1451 await this.#idleWhileStateDirOwned();
1452 return;
1453 }
1454 if (child.role === "trial") {
1455 this.#clearTimer();
1456 const pending = this.#state.update;
1457 if (pending?.status !== "pending") {
1458 throw new Error("Trial child exited without matching pending state.");
1459 }
1460 await this.#returnToPrevious(
1461 pending,
1462 "rolled-back",
1463 `candidate-exited:${String(code ?? signal ?? "unknown")}`,
1464 );
1465 return;
1466 }
1467
1468 this.#clearTimer();
1469 const pending = this.#state.update;
1470 if (pending?.status === "pending") {
1471 await this.#startTrial(pending);
1472 return;
1473 }
1474 if (
1475 await this.#deferForDiskSpace(() =>
1476 this.#startChild(child.version, "active", this.#state.update),
1477 )
1478 )
1479 return;
1480 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1481 }
1482
1483 // Stay idle until the lock frees to avoid supervisor restart loops. A pending update resumes its
1484 // trial, since the other owner may have migrated the database past fromVersion. It stays pending
1485 // only once its backup is gone, so no later trial, even after a launcher restart, restores it over
1486 // the other owner's writes; exit 78 comes before the child opens the database.
1487 async #idleWhileStateDirOwned(): Promise<void> {
1488 this.#clearTimer();
1489 const pending = this.#state.update;
1490 if (pending?.status === "pending") {
1491 const discarded = await discardDatabaseBackup(this.#baseDir, pending.id).then(
1492 () => true,
1493 () => false,
1494 );
1495 if (!discarded) {
1496 const next: ServiceState = {
1497 ...this.#state,
1498 activeVersion: pending.fromVersion,
1499 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1500 };
1501 await this.#selectState(next);
1502 await this.#repointServiceUnit(next.activeVersion);
1503 }
1504 }
1505 this.#releaseServing();
1506 this.#log(
1507 "[service-launcher] Another T3 Code server owns this T3 home. This service starts again once it stops.\n",
1508 );
1509 this.#stateDirFreeProbes = 0;
1510 // The interval also keeps Node alive: signal listeners alone do not.
1511 this.#timer = setInterval(
1512 () => this.#enqueue(() => this.#reclaimStateDir()),
1513 this.#stateDirOwnerPollMs,
1514 );
1515 }
1516
1517 async #reclaimStateDir(): Promise<void> {
1518 if (this.#stopRequested) {
1519 this.#clearTimer();
1520 return;
1521 }
1522 if (this.#timer === undefined || this.#child !== null) return;
1523 const owned = await stateDirOwned(this.#baseDir);
1524 // A stop during the probe clears the timer.
1525 if (this.#timer === undefined) return;
1526 // A relaunching desktop app frees the home for a few seconds; leave it that gap.
1527 this.#stateDirFreeProbes = owned ? 0 : this.#stateDirFreeProbes + 1;
1528 if (this.#stateDirFreeProbes < 2) return;
1529 this.#clearTimer();
1530 await this.#claimServing();
1531 this.#log("[service-launcher] The T3 home is free again. Starting the server.\n");
1532 const update = this.#state.update;
1533 if (update?.status === "pending") await this.#startTrial(update);
1534 else await this.#startChild(this.#state.activeVersion, "active", update);
1535 }
1536
1537 async #returnToPrevious(
1538 pending: PendingServiceUpdate,
1539 status: "rolled-back" | "failed",
1540 reason: string,
1541 child?: ManagedChild,
1542 ): Promise<void> {
1543 if (child !== undefined) {
1544 this.#child = null;
1545 await this.#terminate(child.process);
1546 }
1547 if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason)))
1548 return;
1549 await restoreDatabaseBackup(this.#baseDir, pending);
1550 const outcome = terminalUpdate({ pending, status, reason });
1551 const next: ServiceState = {
1552 ...this.#state,
1553 activeVersion: pending.fromVersion,
1554 update: outcome,
1555 };
1556 await this.#selectState(next);
1557 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1558 try {
1559 await this.#startChild(next.activeVersion, "active", outcome);
1560 } finally {
1561 await this.#repointServiceUnit(next.activeVersion);
1562 }
1563 }
1564 }
1565
1566 /** Task Scheduler cannot set variables or redirect output, so a Windows task passes them as flags. */
1567 export interface LauncherOptions {
1568 readonly baseDir?: string | undefined;
1569 readonly logFile?: string | undefined;
1570 readonly serviceUnit?: string | undefined;
1571 }
1572
1573 export async function main(options: LauncherOptions = {}): Promise<void> {
1574 const baseDir = options.baseDir ?? process.env.T3CODE_HOME;
1575 if (baseDir === undefined || baseDir.trim() === "") {
1576 throw new Error("T3CODE_HOME is required by the T3 Code service launcher.");
1577 }
1578 const output =
1579 options.logFile === undefined ? undefined : NodeFS.openSync(options.logFile, "a", 0o600);
1580 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1581 const windows = process.platform === "win32";
1582 let logTimer: NodeJS.Timeout | undefined;
1583 // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal.
1584 process.stderr.on("error", () => undefined);
1585 try {
1586 // What the service manager started the launcher with. It locates the unit; service.env
1587 // changes only what the server gets.
1588 const managerEnvironment = {
1589 ...process.env,
1590 T3CODE_HOME: baseDir,
1591 ...(options.serviceUnit === undefined ? {} : { [SERVICE_UNIT_ENV]: options.serviceUnit }),
1592 };
1593 const environment = await readServiceEnvironment(baseDir, managerEnvironment);
1594 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1595 const contents = await NodeFSP.readFile(statePath, "utf8");
1596 let state = parseServiceState(contents);
1597 if (state === undefined) {
1598 state = adoptLegacyServiceState(contents);
1599 if (state === undefined) throw new Error("Service state is invalid or unsupported.");
1600 if (!(await runtimeExists(baseDir, state.activeVersion))) {
1601 throw new Error(`Selected t3@${state.activeVersion} runtime is missing or incomplete.`);
1602 }
1603 }
1604 const serviceUnit = await serviceUnitFromEnvironment(managerEnvironment);
1605 const logPath = await serviceLogPath(options, serviceUnit);
1606 if (logPath !== undefined) {
1607 await boundServiceLog(logPath);
1608 logTimer = setInterval(
1609 () => void boundServiceLog(logPath),
1610 SERVICE_LOG_CHECK_INTERVAL_MS,
1611 ).unref();
1612 }
1613 const launcher = new Launcher(baseDir, state, {
1614 environment,
1615 ...(serviceUnit === undefined ? {} : { serviceUnit }),
1616 ...(output === undefined ? {} : { output }),
1617 ...(windows ? { lockPath: serviceLauncherLockPath(baseDir) } : {}),
1618 });
1619 // Ending a scheduled task kills its console host without a signal; the console read fails instead.
1620 const watchConsole = windows && process.stdin.isTTY === true;
1621 const onConsoleLost = () => void launcher.stop("SIGTERM");
1622 if (watchConsole) {
1623 process.stdin.on("error", () => undefined);
1624 process.stdin.once("close", onConsoleLost);
1625 process.stdin.resume();
1626 }
1627 try {
1628 await launcher.run();
1629 } finally {
1630 if (watchConsole) {
1631 process.stdin.off("close", onConsoleLost);
1632 process.stdin.destroy();
1633 }
1634 }
1635 } catch (cause) {
1636 if (output !== undefined) {
1637 NodeFS.writeSync(
1638 output,
1639 `[service-launcher] ${cause instanceof Error ? cause.message : String(cause)}\n`,
1640 );
1641 }
1642 // Agents the server left behind keep the console host, and so the task, running; ending it
1643 // lets the next repeating trigger start the service again.
1644 if (windows && options.serviceUnit !== undefined) {
1645 const taskName = serviceTaskName(process.env.USERDOMAIN ?? "", process.env.USERNAME ?? "");
1646 NodeChildProcess.spawnSync("schtasks", ["/End", "/TN", taskName], {
1647 stdio: "ignore",
1648 windowsHide: true,
1649 });
1650 }
1651 throw cause;
1652 } finally {
1653 clearInterval(logTimer);
1654 if (output !== undefined) NodeFS.closeSync(output);
1655 }
1656 }