The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,963 lines 76.6 KB
Raw
1 // @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
2 import type * as NodeChildProcess from "node:child_process";
3 // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4 import * as NodeFS from "node:fs";
5 // @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback.
6 import * as NodeFSP from "node:fs/promises";
7 import * as NodeSqlite from "node:sqlite";
8
9 import * as NodeServices from "@effect/platform-node/NodeServices";
10 import { assert, expect, it } from "@effect/vitest";
11 import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";
12 import * as HostProcess from "@t3tools/shared/HostProcess";
13 import * as Effect from "effect/Effect";
14 import * as FileSystem from "effect/FileSystem";
15 import * as Path from "effect/Path";
16 import { vi } from "vite-plus/test";
17
18 import {
19 renderBootServicePlist,
20 renderBootServiceTask,
21 renderBootServiceUnit,
22 } from "./cloud/bootService.ts";
23 import {
24 databaseBackupBlocker,
25 Launcher,
26 main,
27 readServiceEnvironment,
28 type BackupSpaceProbe,
29 readServiceState,
30 registerServiceTask,
31 serviceLauncherIsRunning,
32 serviceLauncherIsServing,
33 serviceLauncherLockPath,
34 stateDirOwned,
35 writeServiceState,
36 } from "./serviceLauncher.ts";
37 import {
38 compareExactServiceVersions,
39 decodeServiceState,
40 isExactServiceVersion,
41 decodeServiceUnit,
42 disabledServiceTask,
43 SERVICE_ENV_FILE,
44 SERVICE_LAUNCHER_PROTOCOL,
45 SERVICE_RESTART_PENDING_FILE,
46 SERVICE_STOP_MARKER_FILE,
47 } from "./cloud/serviceProtocol.ts";
48
49 const spawnedChildren = vi.hoisted((): Array<NodeChildProcess.ChildProcess> => []);
50 const spawnHook = vi.hoisted(() => ({
51 current: undefined as ((child: NodeChildProcess.ChildProcess) => void) | undefined,
52 }));
53
54 const failRm = vi.hoisted(() => ({ path: undefined as string | undefined }));
55
56 vi.mock("node:fs/promises", async (importOriginal) => {
57 const actual = await importOriginal<typeof NodeFSP>();
58 const rm: typeof actual.rm = (path, options) =>
59 path === failRm.path
60 ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))
61 : actual.rm(path, options);
62 return { ...actual, rm };
63 });
64
65 vi.mock("node:child_process", async (importOriginal) => {
66 const actual = await importOriginal<typeof NodeChildProcess>();
67 const spawn = (...args: Parameters<typeof actual.spawn>) => {
68 const child = actual.spawn(...args);
69 spawnedChildren.push(child);
70 spawnHook.current?.(child);
71 return child;
72 };
73 return { ...actual, spawn: spawn as typeof actual.spawn };
74 });
75
76 const writeSparseFile = (filePath: string, size: number) =>
77 Effect.gen(function* () {
78 const fileSystem = yield* FileSystem.FileSystem;
79 yield* fileSystem.writeFileString(filePath, "");
80 yield* fileSystem.truncate(filePath, size);
81 });
82
83 it("accepts only exact semantic versions", () => {
84 for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) {
85 assert.isTrue(isExactServiceVersion(version), version);
86 }
87 for (const version of ["latest", "01.2.3", "1.2.3-01", "1.2.3-alpha..1", "1.2.3+."]) {
88 assert.isFalse(isExactServiceVersion(version), version);
89 }
90 });
91
92 it("orders exact semantic versions without treating build metadata as precedence", () => {
93 assert.equal(compareExactServiceVersions("1.2.3", "1.2.3"), 0);
94 assert.equal(compareExactServiceVersions("1.2.4", "1.2.3"), 1);
95 assert.equal(compareExactServiceVersions("2.0.0-alpha.1", "2.0.0-alpha.2"), -1);
96 assert.equal(compareExactServiceVersions("2.0.0-alpha.2", "2.0.0-alpha.beta"), -1);
97 assert.equal(compareExactServiceVersions("2.0.0-alpha-beta", "2.0.0-alpha-alpha"), 1);
98 assert.equal(compareExactServiceVersions("2.0.0", "2.0.0-rc.1"), 1);
99 assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0);
100 });
101
102 it("rejects contradictory service state", () => {
103 assert.isUndefined(
104 decodeServiceState({
105 protocol: SERVICE_LAUNCHER_PROTOCOL,
106 activeVersion: "0.0.31",
107 update: {
108 id: "update-1",
109 fromVersion: "0.0.30",
110 targetVersion: "0.0.32",
111 dbPath: "/tmp/state.sqlite",
112 status: "pending",
113 },
114 }),
115 );
116
117 assert.isUndefined(
118 decodeServiceState({
119 protocol: SERVICE_LAUNCHER_PROTOCOL,
120 activeVersion: "1.0.0",
121 update: {
122 id: "update-3",
123 fromVersion: "1.0.0",
124 targetVersion: "1.1.0",
125 status: "pending",
126 },
127 }),
128 );
129
130 assert.isUndefined(
131 decodeServiceState({
132 protocol: SERVICE_LAUNCHER_PROTOCOL,
133 activeVersion: "1.0.0",
134 update: {
135 id: "update-2",
136 fromVersion: "1.0.0",
137 targetVersion: "0.9.0",
138 dbPath: "/tmp/state.sqlite",
139 status: "pending",
140 },
141 }),
142 );
143 });
144
145 const unitPlan = (root: string, version: string) => ({
146 program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"],
147 baseDir: root,
148 logPath: `${root}/logs/boot-service.log`,
149 unitPath: `${root}/unit/t3code.service`,
150 });
151
152 const writeFakeRuntime = (
153 fs: FileSystem.FileSystem,
154 path: Path.Path,
155 versionDir: string,
156 childSource: string,
157 ) =>
158 Effect.gen(function* () {
159 const entryPath = path.join(versionDir, "t3");
160 yield* fs.makeDirectory(versionDir, { recursive: true });
161 yield* fs.writeFileString(entryPath, `#!${process.execPath}\n${childSource}`);
162 yield* fs.chmod(entryPath, 0o755);
163 yield* fs.writeFileString(
164 path.join(versionDir, ".install-complete"),
165 `${path.basename(versionDir)}\n`,
166 );
167 return entryPath;
168 });
169
170 const runLauncher = async (root: string) => {
171 vi.stubEnv("T3CODE_HOME", root);
172 try {
173 await main();
174 } finally {
175 vi.unstubAllEnvs();
176 }
177 };
178
179 const fakeServiceUnit = (
180 fs: FileSystem.FileSystem,
181 path: Path.Path,
182 root: string,
183 version: string,
184 render = renderBootServiceUnit,
185 ) =>
186 Effect.gen(function* () {
187 const plan = unitPlan(root, version);
188 yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true });
189 const contents = render(plan);
190 yield* fs.writeFileString(plan.unitPath, contents);
191 const unit = {
192 path: plan.unitPath,
193 reloads: 0,
194 loadedContents: contents,
195 failReload: false,
196 reload: async () => {
197 unit.reloads += 1;
198 if (unit.failReload) throw new Error("daemon-reload failed");
199 unit.loadedContents = await NodeFSP.readFile(unit.path, "utf8");
200 },
201 };
202 return unit;
203 });
204
205 it.layer(NodeServices.layer)("service state persistence", (it) => {
206 it.effect.each([
207 "startup",
208 "log write fails",
209 "home probe error",
210 "database probe error",
211 "pending update",
212 "update handoff",
213 "trial exit",
214 "failed trial start",
215 "stop",
216 ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) =>
217 Effect.gen(function* () {
218 const fs = yield* FileSystem.FileSystem;
219 const path = yield* Path.Path;
220 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" });
221 const statePath = path.join(root, "runtime", "service-state.json");
222 const databasePath = path.join(root, "userdata", "statev2.sqlite");
223 let backupPath = path.join(root, "runtime", "db-backup", "disk-update");
224 const pending = {
225 id: "disk-update",
226 fromVersion: "1.0.0",
227 targetVersion: "1.1.0",
228 dbPath: databasePath,
229 status: "pending" as const,
230 };
231 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
232 yield* fs.writeFileString(databasePath, "before trial");
233 const childSource = `
234 const fs = require("node:fs");
235 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
236 if (context.update?.status === "pending") {
237 if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version");
238 if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") {
239 throw new Error("missing original backup");
240 }
241 if (${JSON.stringify(scenario)} === "trial exit") {
242 fs.writeFileSync(context.update.dbPath, "migrated");
243 process.on("message", () => process.exit(1));
244 process.send("migrated");
245 } else {
246 process.send({ type: "prepared", updateId: context.update.id });
247 process.on("message", (message) => {
248 if (message.type === "committed") process.send("ready");
249 });
250 }
251 } else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) {
252 process.on("message", (message) => {
253 if (message.type === "update-accepted") process.send("exiting");
254 if (message === "exit") process.exit(0);
255 });
256 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
257 } else {
258 process.send("ready");
259 }
260 setInterval(() => {}, 1_000);
261 `;
262 for (const version of ["1.0.0", "1.1.0"]) {
263 yield* writeFakeRuntime(
264 fs,
265 path,
266 path.join(root, "runtime", "versions", version),
267 childSource,
268 );
269 }
270 yield* Effect.promise(() =>
271 writeServiceState(statePath, {
272 protocol: SERVICE_LAUNCHER_PROTOCOL,
273 activeVersion: "1.0.0",
274 ...([
275 "startup",
276 "log write fails",
277 "home probe error",
278 "database probe error",
279 "update handoff",
280 "stop",
281 ].includes(scenario)
282 ? {}
283 : { update: pending }),
284 }),
285 );
286 let low = [
287 "startup",
288 "log write fails",
289 "home probe error",
290 "database probe error",
291 "pending update",
292 "stop",
293 ].includes(scenario);
294 let recovered = false;
295 const blocked = Promise.withResolvers<void>();
296 const ready = Promise.withResolvers<void>();
297 const probing = Promise.withResolvers<void>();
298 const capacityResult = Promise.withResolvers<void>();
299 const actualStatfs = NodeFSP.statfs;
300 const stats = yield* Effect.promise(() => actualStatfs(root));
301 yield* Effect.acquireRelease(
302 Effect.sync(() => {
303 vi.useFakeTimers({
304 toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"],
305 });
306 const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => {
307 if (scenario === "stop" && directory === root) {
308 probing.resolve();
309 await capacityResult.promise;
310 }
311 if (
312 (scenario === "home probe error" && directory === root) ||
313 (scenario === "database probe error" && directory === path.dirname(databasePath))
314 ) {
315 throw new Error("statfs unavailable");
316 }
317 if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath))
318 low = true;
319 return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 };
320 });
321 const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
322 if (String(chunk).includes("Waiting for free space")) {
323 blocked.resolve();
324 if (scenario === "log write fails") {
325 throw Object.assign(new Error("no space left"), { code: "ENOSPC" });
326 }
327 }
328 return true;
329 });
330 spawnHook.current = (child) =>
331 child.on("message", (message) => {
332 if (message === "ready") ready.resolve();
333 if (message === "migrated" || message === "exiting") {
334 low = true;
335 child.send("exit");
336 }
337 });
338 return { capacity, log };
339 }),
340 ({ capacity, log }) =>
341 Effect.sync(() => {
342 capacity.mockRestore();
343 log.mockRestore();
344 spawnHook.current = undefined;
345 vi.useRealTimers();
346 }),
347 );
348 const beforeSpawns = spawnedChildren.length;
349 const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
350 const running = launcher.run();
351 yield* Effect.addFinalizer(() =>
352 Effect.promise(async () => {
353 await launcher.stop("SIGTERM");
354 await running;
355 }),
356 );
357 if (scenario === "stop") {
358 yield* Effect.promise(() => probing.promise);
359 const stopping = launcher.stop("SIGTERM");
360 capacityResult.resolve();
361 yield* Effect.promise(() => stopping);
362 yield* Effect.promise(() => running);
363 assert.equal(spawnedChildren.length, beforeSpawns);
364 assert.equal(vi.getTimerCount(), 0);
365 return;
366 }
367 yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running]));
368 assert.equal(
369 spawnedChildren.length - beforeSpawns,
370 ["trial exit", "update handoff"].includes(scenario) ? 1 : 0,
371 );
372 if (scenario === "update handoff") {
373 const state = yield* Effect.promise(() => readServiceState(statePath));
374 backupPath = path.join(root, "runtime", "db-backup", state.update!.id);
375 }
376 assert.equal(
377 yield* fs.exists(backupPath),
378 scenario === "trial exit" || scenario === "failed trial start",
379 );
380 if (scenario === "trial exit")
381 assert.equal(yield* fs.readFileString(databasePath), "migrated");
382 if (scenario === "failed trial start") {
383 yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true });
384 }
385 low = false;
386 recovered = true;
387 yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000));
388 yield* Effect.promise(() => Promise.race([ready.promise, running]));
389 const state = yield* Effect.promise(() => readServiceState(statePath));
390 assert.equal(
391 state.activeVersion,
392 ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0",
393 );
394 assert.equal(
395 state.update?.status,
396 {
397 startup: undefined,
398 "log write fails": undefined,
399 "home probe error": undefined,
400 "database probe error": undefined,
401 "pending update": "committed",
402 "update handoff": "committed",
403 "trial exit": "rolled-back",
404 "failed trial start": "failed",
405 }[scenario],
406 );
407 assert.equal(yield* fs.readFileString(databasePath), "before trial");
408 assert.isFalse(yield* fs.exists(backupPath));
409 }),
410 );
411
412 it.effect(
413 "loads optional service.env literally without changing inherited or managed variables",
414 () =>
415 Effect.gen(function* () {
416 const fs = yield* FileSystem.FileSystem;
417 const path = yield* Path.Path;
418 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-env-" });
419 const inherited = Object.freeze({
420 PATH: "inherited-path",
421 T3CODE_HOME: root,
422 T3_BOOT_SERVICE_UNIT: "managed-unit",
423 T3_SERVICE_LAUNCHER_CONTEXT: "managed-context",
424 T3CODE_PORT: "ambient",
425 });
426 assert.deepEqual(
427 yield* Effect.promise(() => readServiceEnvironment(root, inherited)),
428 inherited,
429 );
430 yield* fs.writeFileString(
431 path.join(root, SERVICE_ENV_FILE),
432 [
433 "# service configuration",
434 "export T3CODE_PORT=1234 # inline comment",
435 'T3CODE_TOKEN="literal $HOME ${USER} $(whoami) # token"',
436 'MULTILINE="first\nsecond"',
437 "EMPTY=",
438 "PATH=/bad",
439 "Path=/bad",
440 "T3CODE_HOME=/bad",
441 "t3code_home=/bad",
442 "T3_BOOT_SERVICE_UNIT=bad",
443 "t3_boot_service_unit=bad",
444 "T3_SERVICE_LAUNCHER_CONTEXT=bad",
445 "t3_service_launcher_context=bad",
446 ].join("\n"),
447 );
448 assert.deepEqual(yield* Effect.promise(() => readServiceEnvironment(root, inherited)), {
449 ...inherited,
450 T3CODE_PORT: "1234",
451 T3CODE_TOKEN: "literal $HOME ${USER} $(whoami) # token",
452 MULTILINE: "first\nsecond",
453 EMPTY: "",
454 });
455 }),
456 );
457
458 it.effect("registers its task again without enabling one the CLI disabled", () =>
459 Effect.gen(function* () {
460 const fs = yield* FileSystem.FileSystem;
461 const path = yield* Path.Path;
462 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-task-" });
463 const unitPath = path.join(root, "t3code-task.xml");
464 const unit = renderBootServiceTask(unitPlan(root, "1.1.0"), {
465 userId: "PC\\theo",
466 homeDir: "C:\\Users\\theo",
467 conhostPath: "conhost.exe",
468 });
469 yield* fs.writeFileString(unitPath, unit);
470 // Task Scheduler's copy: `t3 doctor` disabled it to keep the service stopped.
471 let task = disabledServiceTask(unit.replace("1.1.0", "1.0.0"));
472 const schtasks = async (...args: ReadonlyArray<string>) => {
473 if (args[0] === "/Create") task = decodeServiceUnit(await NodeFSP.readFile(args[2] ?? ""));
474 return { stdout: task };
475 };
476
477 yield* Effect.promise(() => registerServiceTask(unitPath, "t3code-PC-theo", schtasks));
478 assert.equal(task, disabledServiceTask(unit));
479 assert.deepEqual(yield* fs.readDirectory(root), ["t3code-task.xml"]);
480
481 task = unit.replace("1.1.0", "1.0.0");
482 yield* Effect.promise(() => registerServiceTask(unitPath, "t3code-PC-theo", schtasks));
483 assert.equal(task, unit);
484 }),
485 );
486
487 it.effect("finds its unit through the manager's environment, not service.env", () =>
488 Effect.gen(function* () {
489 const fs = yield* FileSystem.FileSystem;
490 const path = yield* Path.Path;
491 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-unit-" });
492 const home = path.join(root, "home");
493 const unitPath = path.join(home, "Library", "LaunchAgents", "t3code.plist");
494 const render = (version: string) =>
495 renderBootServicePlist(unitPlan(root, version), { homeDir: home, environmentPath: "" });
496 yield* fs.makeDirectory(path.dirname(unitPath), { recursive: true });
497 yield* fs.writeFileString(unitPath, render("1.0.0"));
498 // The server may run with another HOME; the launch agent still lives in the manager's.
499 yield* fs.writeFileString(path.join(root, SERVICE_ENV_FILE), "HOME=/elsewhere\n");
500 yield* writeFakeRuntime(
501 fs,
502 path,
503 path.join(root, "runtime", "versions", "1.1.0"),
504 "process.exit(0);\n",
505 );
506 yield* Effect.promise(() =>
507 writeServiceState(path.join(root, "runtime", "service-state.json"), {
508 protocol: SERVICE_LAUNCHER_PROTOCOL,
509 activeVersion: "1.1.0",
510 update: {
511 id: "update-1",
512 fromVersion: "1.0.0",
513 targetVersion: "1.1.0",
514 status: "committed",
515 },
516 }),
517 );
518 vi.stubEnv("HOME", home);
519 vi.stubEnv("T3_BOOT_SERVICE_UNIT", "t3code.plist");
520
521 yield* Effect.promise(() =>
522 expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."),
523 );
524
525 assert.equal(yield* fs.readFileString(unitPath), render("1.1.0"));
526 }),
527 );
528
529 it.effect(
530 "repoints the systemd unit the manager loads when environment.d moves XDG_CONFIG_HOME",
531 () =>
532 Effect.gen(function* () {
533 const fs = yield* FileSystem.FileSystem;
534 const path = yield* Path.Path;
535 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-xdg-" });
536 const home = path.join(root, "home");
537 const managerUnitDir = path.join(home, ".config", "systemd", "user");
538 const unitPath = path.join(managerUnitDir, "t3code.service");
539 const render = (version: string) =>
540 renderBootServiceUnit({ ...unitPlan(root, version), unitPath });
541 yield* fs.makeDirectory(managerUnitDir, { recursive: true });
542 yield* fs.writeFileString(unitPath, render("1.0.0"));
543 // Fake manager: busctl reports the unit search path, systemctl reloads.
544 const bin = path.join(root, "bin");
545 yield* fs.makeDirectory(bin, { recursive: true });
546 const unitPathJson = JSON.stringify({
547 type: "as",
548 data: [`${managerUnitDir}.control`, managerUnitDir, "/usr/lib/systemd/user"],
549 });
550 yield* fs.writeFileString(
551 path.join(bin, "busctl"),
552 `#!/bin/sh\nprintf '%s\\n' '${unitPathJson}'\n`,
553 );
554 yield* fs.writeFileString(
555 path.join(bin, "systemctl"),
556 `#!/bin/sh\necho reload >> '${path.join(root, "reloads")}'\n`,
557 );
558 yield* fs.chmod(path.join(bin, "busctl"), 0o755);
559 yield* fs.chmod(path.join(bin, "systemctl"), 0o755);
560 yield* writeFakeRuntime(
561 fs,
562 path,
563 path.join(root, "runtime", "versions", "1.1.0"),
564 "process.exit(0);\n",
565 );
566 yield* Effect.promise(() =>
567 writeServiceState(path.join(root, "runtime", "service-state.json"), {
568 protocol: SERVICE_LAUNCHER_PROTOCOL,
569 activeVersion: "1.1.0",
570 update: {
571 id: "update-1",
572 fromVersion: "1.0.0",
573 targetVersion: "1.1.0",
574 status: "committed",
575 },
576 }),
577 );
578 vi.stubEnv("PATH", bin);
579 vi.stubEnv("HOME", home);
580 // environment.d changes what services inherit, not where the manager reads units.
581 vi.stubEnv("XDG_CONFIG_HOME", path.join(home, "service-config"));
582 vi.stubEnv("T3_BOOT_SERVICE_UNIT", "t3code.service");
583
584 yield* Effect.promise(() =>
585 expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."),
586 );
587
588 assert.equal(yield* fs.readFileString(unitPath), render("1.1.0"));
589 assert.equal(yield* fs.readFileString(path.join(root, "reloads")), "reload\n");
590 }),
591 );
592
593 it.effect("keeps the service manager's append log bounded when it starts", () =>
594 Effect.gen(function* () {
595 const fs = yield* FileSystem.FileSystem;
596 const path = yield* Path.Path;
597 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-log-" });
598 const home = path.join(root, "home");
599 const unitPath = path.join(home, "Library", "LaunchAgents", "t3code.plist");
600 const plan = unitPlan(root, "1.0.0");
601 yield* fs.makeDirectory(path.dirname(unitPath), { recursive: true });
602 yield* fs.writeFileString(
603 unitPath,
604 renderBootServicePlist(plan, { homeDir: home, environmentPath: "" }),
605 );
606 const oversized = 64 * 1024 * 1024;
607 yield* fs.makeDirectory(path.dirname(plan.logPath), { recursive: true });
608 yield* fs.writeFileString(plan.logPath, "");
609 yield* fs.truncate(plan.logPath, oversized);
610 yield* writeFakeRuntime(
611 fs,
612 path,
613 path.join(root, "runtime", "versions", "1.0.0"),
614 "process.exit(0);\n",
615 );
616 yield* Effect.promise(() =>
617 writeServiceState(path.join(root, "runtime", "service-state.json"), {
618 protocol: SERVICE_LAUNCHER_PROTOCOL,
619 activeVersion: "1.0.0",
620 }),
621 );
622 vi.stubEnv("HOME", home);
623 vi.stubEnv("T3_BOOT_SERVICE_UNIT", "t3code.plist");
624
625 yield* Effect.promise(() =>
626 expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."),
627 );
628
629 const { size } = yield* fs.stat(plan.logPath);
630 expect(Number(size)).toBeLessThan(oversized);
631 }),
632 );
633
634 it.effect("adopts a legacy install record without rewriting it", () =>
635 Effect.gen(function* () {
636 const fs = yield* FileSystem.FileSystem;
637 const path = yield* Path.Path;
638 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-migrate-" });
639 const statePath = path.join(root, "runtime", "service-state.json");
640 const activeVersion = "0.0.43-nightly.20260919.1962+build.1";
641 const versionDir = path.join(root, "runtime", "versions", activeVersion);
642 yield* writeFakeRuntime(
643 fs,
644 path,
645 versionDir,
646 `
647 import { writeFileSync } from "node:fs";
648 writeFileSync(new URL("./started", import.meta.url), "");
649 process.exit(0);
650 `,
651 );
652 const legacyContents = `{ "protocol": 2, "activeVersion": "${activeVersion}" }`;
653 yield* fs.writeFileString(statePath, legacyContents);
654 const restartPending = path.join(root, "runtime", ".restart-pending");
655 yield* fs.writeFileString(restartPending, `${activeVersion}\n`);
656
657 yield* Effect.promise(() =>
658 expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."),
659 );
660
661 assert.isTrue(yield* fs.exists(path.join(versionDir, "started")));
662 assert.isFalse(yield* fs.exists(restartPending));
663 assert.equal(yield* fs.readFileString(statePath), legacyContents);
664 }),
665 );
666
667 it.effect("starts the runtime in a T3 home whose name ends in a space", () =>
668 Effect.gen(function* () {
669 const fs = yield* FileSystem.FileSystem;
670 const path = yield* Path.Path;
671 const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" });
672 const root = path.join(parent, "T3 data ");
673 const versionDir = path.join(root, "runtime", "versions", "1.0.0");
674 yield* writeFakeRuntime(
675 fs,
676 path,
677 versionDir,
678 `
679 import { writeFileSync } from "node:fs";
680 writeFileSync(new URL("./started", import.meta.url), "");
681 process.exit(0);
682 `,
683 );
684 yield* fs.writeFileString(
685 path.join(root, "runtime", "service-state.json"),
686 JSON.stringify({ protocol: 4, activeVersion: "1.0.0" }),
687 );
688
689 yield* Effect.promise(() =>
690 expect(runLauncher(root)).rejects.toThrow("Active child exited unexpectedly (0)."),
691 );
692
693 assert.isTrue(yield* fs.exists(path.join(versionDir, "started")));
694 }),
695 );
696
697 it.effect("refuses non-install legacy documents without rewriting or spawning", () =>
698 Effect.gen(function* () {
699 const fs = yield* FileSystem.FileSystem;
700 const path = yield* Path.Path;
701 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-reject-" });
702 const statePath = path.join(root, "runtime", "service-state.json");
703 yield* writeFakeRuntime(
704 fs,
705 path,
706 path.join(root, "runtime", "versions", "1.0.0"),
707 "process.exit(0);\n",
708 );
709 const legacy = { protocol: 2, activeVersion: "1.0.0" };
710 const childCount = spawnedChildren.length;
711 for (const contents of [
712 JSON.stringify({
713 ...legacy,
714 update: {
715 id: "u",
716 fromVersion: "1.0.0",
717 targetVersion: "1.1.0",
718 dbPath: "/tmp/state.sqlite",
719 status: "pending",
720 },
721 }),
722 JSON.stringify({ ...legacy, update: null }),
723 JSON.stringify({ ...legacy, extra: true }),
724 JSON.stringify({ ...legacy, protocol: 1 }),
725 JSON.stringify({ ...legacy, protocol: 5 }),
726 JSON.stringify({ ...legacy, activeVersion: "latest" }),
727 "{ invalid JSON",
728 ]) {
729 yield* fs.writeFileString(statePath, contents);
730 yield* Effect.promise(() =>
731 expect(runLauncher(root)).rejects.toThrow("Service state is invalid or unsupported."),
732 );
733 assert.equal(yield* fs.readFileString(statePath), contents);
734 assert.equal(spawnedChildren.length, childCount);
735 }
736 }),
737 );
738
739 it.effect("refuses a legacy state whose runtime is missing without rewriting or spawning", () =>
740 Effect.gen(function* () {
741 const fs = yield* FileSystem.FileSystem;
742 const path = yield* Path.Path;
743 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-layout-" });
744 const statePath = path.join(root, "runtime", "service-state.json");
745 const contents = '{ "protocol": 2, "activeVersion": "1.0.0" }';
746 yield* fs.makeDirectory(path.dirname(statePath), { recursive: true });
747 yield* fs.writeFileString(statePath, contents);
748 const childCount = spawnedChildren.length;
749
750 yield* Effect.promise(() =>
751 expect(runLauncher(root)).rejects.toThrow(
752 "Selected t3@1.0.0 runtime is missing or incomplete.",
753 ),
754 );
755 assert.equal(yield* fs.readFileString(statePath), contents);
756 assert.equal(spawnedChildren.length, childCount);
757 }),
758 );
759
760 it.effect("a fresh launcher clears a restart deferred by t3 update", () =>
761 Effect.gen(function* () {
762 const fs = yield* FileSystem.FileSystem;
763 const path = yield* Path.Path;
764 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-restart-" });
765 const statePath = path.join(root, "runtime", "service-state.json");
766 const restartPending = path.join(root, "runtime", SERVICE_RESTART_PENDING_FILE);
767 yield* writeFakeRuntime(
768 fs,
769 path,
770 path.join(root, "runtime", "versions", "1.0.0"),
771 "setInterval(() => {}, 1_000);\n",
772 );
773 yield* Effect.promise(() =>
774 writeServiceState(statePath, {
775 protocol: SERVICE_LAUNCHER_PROTOCOL,
776 activeVersion: "1.0.0",
777 }),
778 );
779 const run = () =>
780 Effect.gen(function* () {
781 const launcher = new Launcher(
782 root,
783 yield* Effect.promise(() => readServiceState(statePath)),
784 );
785 const running = launcher.run();
786 yield* Effect.promise(() => launcher.stop("SIGTERM"));
787 yield* Effect.promise(() => running);
788 });
789
790 yield* fs.writeFileString(restartPending, "1.0.1\n");
791 yield* run();
792 assert.isTrue(yield* fs.exists(restartPending));
793
794 yield* fs.writeFileString(restartPending, "1.0.0\n");
795 yield* run();
796 assert.isFalse(yield* fs.exists(restartPending));
797 }),
798 );
799
800 it.effect.each([
801 { unitVersion: "1.0.0", replacement: "an intentional downgrade" },
802 { unitVersion: "1.2.0", replacement: "another T3 home" },
803 ])(
804 "recovers a rolled-back update with unit $unitVersion and preserves $replacement",
805 ({ unitVersion, replacement }) =>
806 Effect.gen(function* () {
807 const fs = yield* FileSystem.FileSystem;
808 const path = yield* Path.Path;
809 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" });
810 const statePath = path.join(root, "runtime", "service-state.json");
811 yield* writeFakeRuntime(
812 fs,
813 path,
814 path.join(root, "runtime", "versions", "1.1.0"),
815 "setInterval(() => {}, 1_000);\n",
816 );
817 yield* Effect.promise(() =>
818 writeServiceState(statePath, {
819 protocol: SERVICE_LAUNCHER_PROTOCOL,
820 activeVersion: "1.1.0",
821 update: {
822 id: "update-1",
823 fromVersion: "1.1.0",
824 targetVersion: "1.2.0",
825 status: "rolled-back",
826 },
827 }),
828 );
829 const unit = yield* fakeServiceUnit(fs, path, root, unitVersion);
830 const expected = renderBootServiceUnit(
831 unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion),
832 );
833 const createLauncher = Effect.gen(function* () {
834 return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), {
835 serviceUnit: unit,
836 });
837 });
838 const run = (launcher: Launcher) =>
839 Effect.gen(function* () {
840 const running = launcher.run();
841 yield* Effect.promise(() => launcher.stop("SIGTERM"));
842 yield* Effect.promise(() => running);
843 });
844
845 yield* run(yield* createLauncher);
846 assert.equal(yield* fs.readFileString(unit.path), expected);
847 assert.equal(unit.loadedContents, expected);
848
849 yield* run(yield* createLauncher);
850 assert.equal(yield* fs.readFileString(unit.path), expected);
851 assert.equal(unit.loadedContents, expected);
852
853 yield* Effect.promise(() =>
854 writeServiceState(statePath, {
855 protocol: SERVICE_LAUNCHER_PROTOCOL,
856 activeVersion: "1.1.0",
857 update: {
858 id: "update-2",
859 fromVersion: "1.0.0",
860 targetVersion: "1.1.0",
861 status: "committed",
862 },
863 }),
864 );
865 const staleLauncher = yield* createLauncher;
866 // A local install can replace the selection or service owner without stopping this launcher.
867 const selectedRoot =
868 replacement === "another T3 home" ? path.join(root, "other-home") : root;
869 yield* writeFakeRuntime(
870 fs,
871 path,
872 path.join(selectedRoot, "runtime", "versions", "1.0.0"),
873 "setInterval(() => {}, 1_000);\n",
874 );
875 const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json");
876 yield* Effect.promise(() =>
877 writeServiceState(selectedStatePath, {
878 protocol: SERVICE_LAUNCHER_PROTOCOL,
879 activeVersion: "1.0.0",
880 }),
881 );
882 const selectedUnit = renderBootServiceUnit({
883 ...unitPlan(selectedRoot, "1.0.0"),
884 unitPath: unit.path,
885 });
886 yield* fs.writeFileString(unit.path, selectedUnit);
887 yield* Effect.promise(() => unit.reload());
888 const oldHomeState = yield* fs.readFileString(statePath);
889 const selectedState = yield* fs.readFileString(selectedStatePath);
890
891 yield* run(staleLauncher);
892 assert.equal(yield* fs.readFileString(statePath), oldHomeState);
893 assert.equal(yield* fs.readFileString(selectedStatePath), selectedState);
894 assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement);
895 assert.equal(unit.loadedContents, selectedUnit, replacement);
896 }),
897 );
898
899 it.effect("waits without exiting and restarts once the other owner releases the T3 home", () =>
900 Effect.gen(function* () {
901 const fs = yield* FileSystem.FileSystem;
902 const path = yield* Path.Path;
903 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-" });
904 const statePath = path.join(root, "runtime", "service-state.json");
905 const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
906 yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
907 yield* writeFakeRuntime(
908 fs,
909 path,
910 path.join(root, "runtime", "versions", "1.0.0"),
911 [
912 `const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`,
913 `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`,
914 `catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`,
915 'process.send("owner");',
916 "setInterval(() => {}, 1_000);",
917 ].join("\n"),
918 );
919 yield* Effect.promise(() =>
920 writeServiceState(statePath, {
921 protocol: SERVICE_LAUNCHER_PROTOCOL,
922 activeVersion: "1.0.0",
923 }),
924 );
925 using otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
926 otherOwner.exec("BEGIN EXCLUSIVE;");
927 const refused = Promise.withResolvers<void>();
928 const owned = Promise.withResolvers<void>();
929 yield* Effect.acquireRelease(
930 Effect.sync(() =>
931 vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
932 if (String(chunk).includes("owns this T3 home")) refused.resolve();
933 return true;
934 }),
935 ),
936 (spy) => Effect.sync(() => spy.mockRestore()),
937 );
938 yield* Effect.acquireRelease(
939 Effect.sync(() => {
940 spawnHook.current = (child) => child.once("message", () => owned.resolve());
941 }),
942 () =>
943 Effect.sync(() => {
944 spawnHook.current = undefined;
945 }),
946 );
947 const spawnsBefore = spawnedChildren.length;
948 const launcher = new Launcher(
949 root,
950 yield* Effect.promise(() => readServiceState(statePath)),
951 { stateDirOwnerPollMs: 10 },
952 );
953 let completed = false;
954 const running = launcher.run().finally(() => {
955 completed = true;
956 });
957 yield* Effect.promise(() => refused.promise);
958 assert.isFalse(completed);
959 // An idle launcher must not make an SSH launch wait for a server it is not starting.
960 assert.isFalse(yield* Effect.promise(() => serviceLauncherIsServing(root)));
961 otherOwner.close();
962 yield* Effect.promise(() => owned.promise);
963 assert.isTrue(yield* Effect.promise(() => serviceLauncherIsServing(root)));
964 yield* Effect.promise(() => launcher.stop("SIGTERM"));
965 yield* Effect.promise(() => running);
966 assert.strictEqual(spawnedChildren.length - spawnsBefore, 2);
967 }),
968 );
969
970 it.effect(
971 "keeps a pending update while owned and rolls a failed resumed trial back to the other owner's writes",
972 () =>
973 Effect.gen(function* () {
974 const fs = yield* FileSystem.FileSystem;
975 const path = yield* Path.Path;
976 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-reclaim-update-" });
977 const statePath = path.join(root, "runtime", "service-state.json");
978 const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
979 const databasePath = path.join(root, "userdata", "state.sqlite");
980 const startsPath = path.join(root, "starts.log");
981 yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
982 yield* fs.writeFileString(databasePath, "before update");
983 for (const version of ["1.0.0", "1.1.0"]) {
984 yield* writeFakeRuntime(
985 fs,
986 path,
987 path.join(root, "runtime", "versions", version),
988 [
989 `const db = new (require("node:sqlite").DatabaseSync)(${JSON.stringify(ownerPath)});`,
990 `try { db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); }`,
991 `catch { process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED}); }`,
992 "const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);",
993 `require("node:fs").appendFileSync(${JSON.stringify(startsPath)}, context.childVersion + "\\n");`,
994 'if (context.update?.status === "pending") {',
995 ` require("node:fs").writeFileSync(${JSON.stringify(databasePath)}, "migrated by failed trial");`,
996 " process.exit(1);",
997 "}",
998 'process.send("active");',
999 "setInterval(() => {}, 1_000);",
1000 ].join("\n"),
1001 );
1002 }
1003 yield* Effect.promise(() =>
1004 writeServiceState(statePath, {
1005 protocol: SERVICE_LAUNCHER_PROTOCOL,
1006 activeVersion: "1.0.0",
1007 update: {
1008 id: "update-1",
1009 fromVersion: "1.0.0",
1010 targetVersion: "1.1.0",
1011 dbPath: databasePath,
1012 status: "pending",
1013 },
1014 }),
1015 );
1016 using otherOwner = new NodeSqlite.DatabaseSync(ownerPath);
1017 otherOwner.exec("BEGIN EXCLUSIVE;");
1018 const refused = Promise.withResolvers<void>();
1019 const active = Promise.withResolvers<void>();
1020 yield* Effect.acquireRelease(
1021 Effect.sync(() =>
1022 vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
1023 if (String(chunk).includes("owns this T3 home")) refused.resolve();
1024 return true;
1025 }),
1026 ),
1027 (spy) => Effect.sync(() => spy.mockRestore()),
1028 );
1029 yield* Effect.acquireRelease(
1030 Effect.sync(() => {
1031 spawnHook.current = (child) =>
1032 child.on("message", (message) => message === "active" && active.resolve());
1033 }),
1034 () =>
1035 Effect.sync(() => {
1036 spawnHook.current = undefined;
1037 }),
1038 );
1039 const launcher = new Launcher(
1040 root,
1041 yield* Effect.promise(() => readServiceState(statePath)),
1042 { stateDirOwnerPollMs: 10 },
1043 );
1044 const running = launcher.run();
1045 yield* Effect.promise(() => refused.promise);
1046 const idle = yield* Effect.promise(() => readServiceState(statePath));
1047 assert.strictEqual(idle.activeVersion, "1.0.0");
1048 assert.strictEqual(idle.update?.status, "pending");
1049 assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", "update-1")));
1050
1051 yield* fs.writeFileString(databasePath, "written by the other owner");
1052 otherOwner.close();
1053 yield* Effect.promise(() => active.promise);
1054 yield* Effect.promise(() => launcher.stop("SIGTERM"));
1055 yield* Effect.promise(() => running);
1056
1057 const state = yield* Effect.promise(() => readServiceState(statePath));
1058 assert.strictEqual(state.activeVersion, "1.0.0");
1059 assert.strictEqual(state.update?.status, "rolled-back");
1060 assert.strictEqual(yield* fs.readFileString(startsPath), "1.1.0\n1.0.0\n");
1061 assert.strictEqual(yield* fs.readFileString(databasePath), "written by the other owner");
1062 }),
1063 );
1064
1065 it.effect("fails a pending update when its backup cannot be dropped while owned", () =>
1066 Effect.gen(function* () {
1067 const fs = yield* FileSystem.FileSystem;
1068 const path = yield* Path.Path;
1069 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-stale-backup-" });
1070 const statePath = path.join(root, "runtime", "service-state.json");
1071 const ownerPath = path.join(root, "userdata", "server-owner.sqlite");
1072 const databasePath = path.join(root, "userdata", "state.sqlite");
1073 yield* fs.makeDirectory(path.dirname(ownerPath), { recursive: true });
1074 yield* fs.writeFileString(databasePath, "before update");
1075 yield* writeFakeRuntime(
1076 fs,
1077 path,
1078 path.join(root, "runtime", "versions", "1.1.0"),
1079 `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`,
1080 );
1081 yield* Effect.promise(() =>
1082 writeServiceState(statePath, {
1083 protocol: SERVICE_LAUNCHER_PROTOCOL,
1084 activeVersion: "1.0.0",
1085 update: {
1086 id: "update-1",
1087 fromVersion: "1.0.0",
1088 targetVersion: "1.1.0",
1089 dbPath: databasePath,
1090 status: "pending",
1091 },
1092 }),
1093 );
1094 const refused = Promise.withResolvers<void>();
1095 yield* Effect.acquireRelease(
1096 Effect.sync(() =>
1097 vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
1098 if (String(chunk).includes("owns this T3 home")) refused.resolve();
1099 return true;
1100 }),
1101 ),
1102 (spy) => Effect.sync(() => spy.mockRestore()),
1103 );
1104 yield* Effect.acquireRelease(
1105 Effect.sync(() => {
1106 failRm.path = path.join(root, "runtime", "db-backup", "update-1");
1107 }),
1108 () =>
1109 Effect.sync(() => {
1110 failRm.path = undefined;
1111 }),
1112 );
1113 const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
1114 const running = launcher.run();
1115 yield* Effect.promise(() => refused.promise);
1116 yield* Effect.promise(() => launcher.stop("SIGTERM"));
1117 yield* Effect.promise(() => running);
1118
1119 const state = yield* Effect.promise(() => readServiceState(statePath));
1120 assert.strictEqual(state.activeVersion, "1.0.0");
1121 assert.strictEqual(state.update?.status, "failed");
1122 }),
1123 );
1124
1125 it.effect("treats the T3 home as owned while the lock or a pre-lock server holds it", () =>
1126 Effect.gen(function* () {
1127 const fs = yield* FileSystem.FileSystem;
1128 const path = yield* Path.Path;
1129 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-owned-" });
1130 const stateDir = path.join(root, "userdata");
1131 const recordPath = path.join(stateDir, "server-runtime.json");
1132 yield* fs.makeDirectory(stateDir, { recursive: true });
1133 assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
1134
1135 using owner = new NodeSqlite.DatabaseSync(path.join(stateDir, "server-owner.sqlite"));
1136 owner.exec("BEGIN EXCLUSIVE;");
1137 assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
1138 owner.close();
1139
1140 const record = { version: 1, pid: process.pid, port: 1, origin: "", startedAt: "" };
1141 yield* fs.writeFileString(recordPath, JSON.stringify(record));
1142 assert.isTrue(yield* Effect.promise(() => stateDirOwned(root)));
1143 yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, ownerId: "lock-aware" }));
1144 assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
1145 // This process started after 1970, so its PID was reused since that record.
1146 yield* fs.writeFileString(
1147 recordPath,
1148 JSON.stringify({ ...record, startedAt: "1970-01-01T00:00:00Z" }),
1149 );
1150 assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
1151 yield* fs.writeFileString(recordPath, JSON.stringify({ ...record, pid: 2 ** 22 + 1 }));
1152 assert.isFalse(yield* Effect.promise(() => stateDirOwned(root)));
1153 }),
1154 );
1155
1156 const commitUpdate = (
1157 render = renderBootServiceUnit,
1158 missingUnit = false,
1159 legacyInstall = false,
1160 ) =>
1161 Effect.gen(function* () {
1162 const fs = yield* FileSystem.FileSystem;
1163 const path = yield* Path.Path;
1164 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service 100% & <test>-" });
1165 const statePath = path.join(root, "runtime", "service-state.json");
1166 const databasePath = path.join(root, "userdata", "state.sqlite");
1167 const activatedPath = path.join(root, "activated");
1168 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1169 yield* fs.writeFileString(databasePath, "before trial");
1170 const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render);
1171 if (missingUnit) yield* fs.remove(unit.path);
1172 const childSource = `
1173 import assert from "node:assert/strict";
1174 import { existsSync, readFileSync, writeFileSync } from "node:fs";
1175 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
1176 assert.equal(context.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
1177 if (context.update?.status === "pending") {
1178 const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
1179 assert.equal(state.activeVersion, "1.0.0");
1180 assert.equal(state.update.status, "pending");
1181 ${
1182 missingUnit
1183 ? `assert.equal(existsSync(${JSON.stringify(unit.path)}), false);`
1184 : `assert.equal(readFileSync(${JSON.stringify(unit.path)}, "utf8"), ${JSON.stringify(unit.loadedContents)});`
1185 }
1186 process.on("message", (message) => {
1187 if (message.type === "committed") {
1188 const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
1189 assert.equal(state.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
1190 assert.equal(state.activeVersion, "1.1.0");
1191 assert.equal(state.update.status, "committed");
1192 writeFileSync(${JSON.stringify(activatedPath)}, message.updateId);
1193 process.exit(0);
1194 }
1195 });
1196 process.send({ type: "prepared", updateId: context.update.id });
1197 } else if (context.update === undefined) {
1198 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
1199 setInterval(() => {}, 1_000);
1200 } else {
1201 process.exit(0);
1202 }
1203 `;
1204 for (const version of ["1.0.0", "1.1.0"]) {
1205 yield* writeFakeRuntime(
1206 fs,
1207 path,
1208 path.join(root, "runtime", "versions", version),
1209 childSource,
1210 );
1211 }
1212 // The old CLI installs the new runtime but writes state with its own protocol.
1213 yield* fs.writeFileString(
1214 statePath,
1215 `${JSON.stringify(
1216 {
1217 protocol: legacyInstall ? 3 : SERVICE_LAUNCHER_PROTOCOL,
1218 activeVersion: "1.0.0",
1219 },
1220 null,
1221 2,
1222 )}\n`,
1223 );
1224 const launcher = new Launcher(
1225 root,
1226 yield* Effect.promise(() => readServiceState(statePath)),
1227 { serviceUnit: unit },
1228 );
1229 const failure = yield* Effect.promise(() =>
1230 launcher.run().then(
1231 () => "launcher unexpectedly completed",
1232 (cause: unknown) => String(cause),
1233 ),
1234 );
1235 assert.include(failure, "Active child exited unexpectedly (0)");
1236 const state = yield* Effect.promise(() => readServiceState(statePath));
1237 assert.equal(state.activeVersion, "1.1.0");
1238 assert.equal(state.update?.status, "committed");
1239 assert.equal(yield* fs.readFileString(activatedPath), state.update?.id);
1240 const recover = () =>
1241 Effect.gen(function* () {
1242 const fresh = new Launcher(
1243 root,
1244 yield* Effect.promise(() => readServiceState(statePath)),
1245 { serviceUnit: unit },
1246 );
1247 const running = fresh.run();
1248 yield* Effect.promise(() => fresh.stop("SIGTERM"));
1249 yield* Effect.promise(() => running);
1250 });
1251 return { fs, root, unit, recover };
1252 });
1253
1254 it.effect.each([
1255 {
1256 name: "systemd after a protocol-3 local upgrade",
1257 render: renderBootServiceUnit,
1258 legacyInstall: true,
1259 },
1260 {
1261 name: "launchd",
1262 legacyInstall: false,
1263 render: (plan: Parameters<typeof renderBootServicePlist>[0]) =>
1264 renderBootServicePlist(plan, {
1265 homeDir: "/Users/test & <user>",
1266 environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0",
1267 }),
1268 },
1269 {
1270 name: "Task Scheduler",
1271 legacyInstall: false,
1272 render: (plan: Parameters<typeof renderBootServiceTask>[0]) =>
1273 renderBootServiceTask(
1274 { ...plan, program: [`${plan.program[0]}.exe`, "__service-launcher"] },
1275 { userId: "PC\\theo", homeDir: "C:\\Users\\theo", conhostPath: "conhost.exe" },
1276 ),
1277 },
1278 ])(
1279 "commits and repoints $name only after the trial reports prepared",
1280 ({ render, legacyInstall }) =>
1281 Effect.gen(function* () {
1282 const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall);
1283 const expected = render(unitPlan(root, "1.1.0"));
1284 assert.equal(yield* fs.readFileString(unit.path), expected);
1285 assert.equal(unit.loadedContents, expected);
1286 assert.equal(unit.reloads, 1);
1287 }),
1288 );
1289
1290 it.effect(
1291 "keeps a durable commit through unit failure and retries a failed reload on restart",
1292 () =>
1293 Effect.gen(function* () {
1294 const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true);
1295 assert.equal(unit.reloads, 0);
1296 const original = renderBootServiceUnit(unitPlan(root, "1.0.0"));
1297 const expected = renderBootServiceUnit(unitPlan(root, "1.1.0"));
1298 assert.equal(unit.loadedContents, original);
1299 yield* fs.writeFileString(unit.path, original);
1300 unit.failReload = true;
1301 yield* recover();
1302 assert.equal(yield* fs.readFileString(unit.path), expected);
1303 assert.equal(unit.reloads, 1);
1304 assert.equal(unit.loadedContents, original);
1305
1306 unit.failReload = false;
1307 yield* recover();
1308 assert.equal(yield* fs.readFileString(unit.path), expected);
1309 assert.equal(unit.loadedContents, expected);
1310 assert.equal(unit.reloads, 2);
1311 }),
1312 );
1313
1314 it.effect("rolls back invalid trials with the startup environment", () =>
1315 Effect.gen(function* () {
1316 const fs = yield* FileSystem.FileSystem;
1317 const path = yield* Path.Path;
1318 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" });
1319 const statePath = path.join(root, "runtime", "service-state.json");
1320 const databasePath = path.join(root, "userdata", "state.sqlite");
1321 const observationsPath = path.join(root, "environment-observations.jsonl");
1322 const envPath = path.join(root, SERVICE_ENV_FILE);
1323 yield* fs.writeFileString(envPath, "T3CODE_TEST_SERVICE_TOKEN=initial\n");
1324 vi.stubEnv("T3CODE_HOME", root);
1325 vi.stubEnv("T3CODE_TEST_SERVICE_TOKEN", "inherited");
1326 yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs()));
1327 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1328 yield* fs.writeFileString(databasePath, "before trial");
1329 const encodedDatabasePath = JSON.stringify(databasePath);
1330 const childSource = `
1331 import { appendFileSync, writeFileSync } from "node:fs";
1332 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
1333 appendFileSync(${JSON.stringify(observationsPath)}, JSON.stringify({
1334 token: process.env.T3CODE_TEST_SERVICE_TOKEN,
1335 update: context.update?.status ?? "active",
1336 }) + "\\n");
1337 if (context.update?.status === "pending") {
1338 process.send({ type: "prepared", updateId: "wrong-update" });
1339 } else if (context.update === undefined) {
1340 writeFileSync(${JSON.stringify(envPath)}, "T3CODE_TEST_SERVICE_TOKEN=changed\\n");
1341 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
1342 setInterval(() => {}, 1_000);
1343 } else {
1344 process.exit(0);
1345 }
1346 `;
1347 for (const version of ["1.0.0", "1.1.0"]) {
1348 yield* writeFakeRuntime(
1349 fs,
1350 path,
1351 path.join(root, "runtime", "versions", version),
1352 childSource,
1353 );
1354 }
1355 yield* Effect.promise(() =>
1356 writeServiceState(statePath, {
1357 protocol: SERVICE_LAUNCHER_PROTOCOL,
1358 activeVersion: "1.0.0",
1359 }),
1360 );
1361 const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0");
1362
1363 const launcher = new Launcher(
1364 root,
1365 yield* Effect.promise(() => readServiceState(statePath)),
1366 {
1367 serviceUnit: unit,
1368 environment: yield* Effect.promise(() => readServiceEnvironment(root)),
1369 },
1370 );
1371 yield* Effect.promise(() =>
1372 launcher.run().then(
1373 () => Promise.reject(new Error("launcher unexpectedly completed")),
1374 () => Promise.resolve(),
1375 ),
1376 );
1377
1378 const state = yield* Effect.promise(() => readServiceState(statePath));
1379 assert.equal(state.activeVersion, "1.0.0");
1380 assert.equal(state.update?.status, "rolled-back");
1381 assert.equal(
1382 yield* fs.readFileString(unit.path),
1383 renderBootServiceUnit(unitPlan(root, "1.0.0")),
1384 );
1385 assert.equal(unit.reloads, 1);
1386 assert.equal(
1387 state.update?.status === "rolled-back" ? state.update.reason : undefined,
1388 "invalid-prepared",
1389 );
1390 const observations = (yield* fs.readFileString(observationsPath))
1391 .trim()
1392 .split("\n")
1393 .map((line) => JSON.parse(line));
1394 assert.deepEqual(observations, [
1395 { token: "initial", update: "active" },
1396 { token: "initial", update: "pending" },
1397 { token: "initial", update: "rolled-back" },
1398 ]);
1399 assert.equal((yield* HostProcess.Environment).T3CODE_TEST_SERVICE_TOKEN, "inherited");
1400 assert.equal(yield* fs.readFileString(envPath), "T3CODE_TEST_SERVICE_TOKEN=changed\n");
1401 }),
1402 );
1403
1404 it.effect.each(["file", "symlink", "dangling symlink"] as const)(
1405 "restores the database when a migrating trial exits, with the database a %s",
1406 (kind) =>
1407 Effect.gen(function* () {
1408 const fs = yield* FileSystem.FileSystem;
1409 const path = yield* Path.Path;
1410 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" });
1411 const statePath = path.join(root, "runtime", "service-state.json");
1412 const databasePath = path.join(root, "userdata", "state.sqlite");
1413 const databaseFile =
1414 kind === "file" ? databasePath : path.join(root, "data", "original.sqlite");
1415 const linkTarget = kind === "dangling symlink" ? "../original.sqlite" : databaseFile;
1416 const original = "database before migration";
1417 yield* fs.makeDirectory(path.dirname(databaseFile), { recursive: true });
1418 if (kind === "dangling symlink") {
1419 yield* fs.makeDirectory(path.join(root, "data", "sub"));
1420 yield* fs.symlink(path.join(root, "data", "sub"), path.dirname(databasePath));
1421 } else {
1422 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1423 }
1424 yield* fs.writeFileString(databaseFile, original);
1425 if (kind !== "file") yield* fs.symlink(linkTarget, databasePath);
1426 const encodedDatabasePath = JSON.stringify(databasePath);
1427 const childSource = `
1428 import { realpathSync, unlinkSync, writeFileSync } from "node:fs";
1429 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
1430 if (context.update?.status === "pending") {
1431 writeFileSync(context.update.dbPath, "database after migration");
1432 writeFileSync(context.update.dbPath + "-wal", "trial wal");
1433 writeFileSync(context.update.dbPath + "-shm", "trial shm");
1434 ${kind === "dangling symlink" ? "unlinkSync(realpathSync(context.update.dbPath));" : ""}
1435 process.exit(1);
1436 } else if (context.update === undefined) {
1437 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
1438 setInterval(() => {}, 1_000);
1439 } else {
1440 process.exit(0);
1441 }
1442 `;
1443 for (const version of ["1.0.0", "1.1.0"]) {
1444 yield* writeFakeRuntime(
1445 fs,
1446 path,
1447 path.join(root, "runtime", "versions", version),
1448 childSource,
1449 );
1450 }
1451 yield* Effect.promise(() =>
1452 writeServiceState(statePath, {
1453 protocol: SERVICE_LAUNCHER_PROTOCOL,
1454 activeVersion: "1.0.0",
1455 }),
1456 );
1457
1458 const launcher = new Launcher(
1459 root,
1460 yield* Effect.promise(() => readServiceState(statePath)),
1461 );
1462 yield* Effect.promise(() =>
1463 launcher.run().then(
1464 () => Promise.reject(new Error("launcher unexpectedly completed")),
1465 () => Promise.resolve(),
1466 ),
1467 );
1468
1469 const state = yield* Effect.promise(() => readServiceState(statePath));
1470 assert.equal(state.activeVersion, "1.0.0");
1471 assert.equal(state.update?.status, "rolled-back");
1472 assert.equal(yield* fs.readFileString(databaseFile), original);
1473 if (kind !== "file") assert.equal(yield* fs.readLink(databasePath), linkTarget);
1474 assert.isFalse(yield* fs.exists(path.join(root, "original.sqlite")));
1475 assert.isFalse(yield* fs.exists(`${databasePath}-wal`));
1476 assert.isFalse(yield* fs.exists(`${databasePath}-shm`));
1477 const updateId = state.update?.id;
1478 assert.isDefined(updateId);
1479 assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId)));
1480 }),
1481 );
1482
1483 it.effect.each(["wal", "delete"] as const)(
1484 "backs up a %s-mode database (online when WAL) and restores its final state",
1485 (journalMode) =>
1486 Effect.gen(function* () {
1487 const fs = yield* FileSystem.FileSystem;
1488 const path = yield* Path.Path;
1489 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" });
1490 const statePath = path.join(root, "runtime", "service-state.json");
1491 const databasePath = path.join(root, "userdata", "state.sqlite");
1492 const observationsPath = path.join(root, "observations.json");
1493 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1494 // SQLite keeps -wal beside the symlink's target, so the backup must restore it there.
1495 yield* fs.symlink("actual.sqlite", databasePath);
1496 const seed = new NodeSqlite.DatabaseSync(databasePath);
1497 seed.exec(`
1498 PRAGMA journal_mode = ${journalMode};
1499 CREATE TABLE events (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);
1500 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 500)
1501 INSERT INTO events (payload) SELECT 'before the update ' || i FROM n;
1502 `);
1503 seed.close();
1504
1505 const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode });
1506 const childSource = `
1507 import { readdirSync, writeFileSync } from "node:fs";
1508 import { DatabaseSync } from "node:sqlite";
1509 const paths = ${encoded};
1510 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
1511 if (context.update?.status === "pending") {
1512 writeFileSync(paths.observationsPath + ".trial", JSON.stringify(
1513 readdirSync(paths.root + "/runtime/db-backup/" + context.update.id).sort(),
1514 ));
1515 const trial = new DatabaseSync(context.update.dbPath);
1516 trial.exec("PRAGMA journal_mode = " + paths.journalMode + "; INSERT INTO events (payload) VALUES ('trial');");
1517 trial.close();
1518 process.exit(1);
1519 } else if (context.update === undefined) {
1520 const db = new DatabaseSync(paths.databasePath);
1521 db.exec("PRAGMA busy_timeout = 5000;");
1522 db.exec("PRAGMA journal_mode = " + paths.journalMode + ";");
1523 const insert = db.prepare("INSERT INTO events (payload) VALUES (?)");
1524 const writer = setInterval(() => insert.run("written while serving"), 1);
1525 process.once("SIGTERM", () => {
1526 clearInterval(writer);
1527 insert.run("last write before stopping");
1528 const backupRoot = paths.root + "/runtime/db-backup";
1529 writeFileSync(paths.observationsPath, JSON.stringify({
1530 backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)),
1531 rows: db.prepare("SELECT count(*) AS n FROM events").get().n,
1532 }));
1533 db.close();
1534 process.exit(0);
1535 });
1536 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
1537 } else {
1538 process.exit(0);
1539 }
1540 `;
1541 for (const version of ["1.0.0", "1.1.0"]) {
1542 yield* writeFakeRuntime(
1543 fs,
1544 path,
1545 path.join(root, "runtime", "versions", version),
1546 childSource,
1547 );
1548 }
1549 yield* Effect.promise(() =>
1550 writeServiceState(statePath, {
1551 protocol: SERVICE_LAUNCHER_PROTOCOL,
1552 activeVersion: "1.0.0",
1553 }),
1554 );
1555
1556 const launcher = new Launcher(
1557 root,
1558 yield* Effect.promise(() => readServiceState(statePath)),
1559 {
1560 backupSpaceProbe: { canClone: async () => false },
1561 },
1562 );
1563 yield* Effect.promise(() =>
1564 launcher.run().then(
1565 () => Promise.reject(new Error("launcher unexpectedly completed")),
1566 () => Promise.resolve(),
1567 ),
1568 );
1569
1570 const state = yield* Effect.promise(() => readServiceState(statePath));
1571 assert.equal(state.update?.status, "rolled-back");
1572 const observed = JSON.parse(yield* fs.readFileString(observationsPath));
1573 // A WAL database's main file is copied before the server is asked to stop; others fall back offline.
1574 assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []);
1575 assert.isAbove(observed.rows, 501);
1576 // Only the online copy holds no -shm: the offline fallback copies the one the server left.
1577 assert.deepEqual(
1578 JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)),
1579 journalMode === "wal" ? ["database", "database-wal"] : ["database"],
1580 );
1581 const restored = new NodeSqlite.DatabaseSync(databasePath);
1582 try {
1583 assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok");
1584 assert.equal(
1585 restored.prepare("SELECT count(*) AS n FROM events").get()?.n,
1586 observed.rows,
1587 );
1588 assert.equal(
1589 restored
1590 .prepare(
1591 "SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'",
1592 )
1593 .get()?.n,
1594 1,
1595 );
1596 } finally {
1597 restored.close();
1598 }
1599 }),
1600 );
1601
1602 const requestRejectedUpdate = (backupSpaceProbe: Partial<BackupSpaceProbe>) =>
1603 Effect.gen(function* () {
1604 const fs = yield* FileSystem.FileSystem;
1605 const path = yield* Path.Path;
1606 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" });
1607 const statePath = path.join(root, "runtime", "service-state.json");
1608 const databasePath = path.join(root, "userdata", "state.sqlite");
1609 const replyPath = path.join(root, "rejection.txt");
1610 const trialMarker = path.join(root, "trial-started");
1611 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1612 yield* fs.writeFileString(databasePath, "database");
1613 const encoded = JSON.stringify({ databasePath, replyPath, trialMarker });
1614 const childSource = `
1615 import { writeFileSync } from "node:fs";
1616 const paths = ${encoded};
1617 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
1618 if (context.update?.status === "pending") {
1619 writeFileSync(paths.trialMarker, "started");
1620 process.exit(1);
1621 } else if (context.update === undefined) {
1622 process.on("message", (message) => {
1623 if (message.type === "update-rejected") {
1624 writeFileSync(paths.replyPath, message.reason);
1625 process.exit(0);
1626 }
1627 });
1628 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
1629 setInterval(() => {}, 1_000);
1630 } else {
1631 process.exit(0);
1632 }
1633 `;
1634 for (const version of ["1.0.0", "1.1.0"]) {
1635 yield* writeFakeRuntime(
1636 fs,
1637 path,
1638 path.join(root, "runtime", "versions", version),
1639 childSource,
1640 );
1641 }
1642 yield* Effect.promise(() =>
1643 writeServiceState(statePath, {
1644 protocol: SERVICE_LAUNCHER_PROTOCOL,
1645 activeVersion: "1.0.0",
1646 }),
1647 );
1648
1649 const launcher = new Launcher(
1650 root,
1651 yield* Effect.promise(() => readServiceState(statePath)),
1652 { backupSpaceProbe },
1653 );
1654 yield* Effect.promise(() =>
1655 launcher.run().then(
1656 () => Promise.reject(new Error("launcher unexpectedly completed")),
1657 () => Promise.resolve(),
1658 ),
1659 );
1660
1661 return {
1662 reason: yield* fs.readFileString(replyPath),
1663 trialStarted: yield* fs.exists(trialMarker),
1664 state: yield* Effect.promise(() => readServiceState(statePath)),
1665 };
1666 });
1667
1668 it.effect("refuses a backup that cannot fit and leaves the server running", () =>
1669 Effect.gen(function* () {
1670 const { reason, trialStarted, state } = yield* requestRejectedUpdate({
1671 readFreeBytes: () => Promise.resolve(0),
1672 canClone: () => Promise.resolve(true),
1673 });
1674
1675 assert.include(reason, "Not enough free disk space");
1676 assert.include(reason, "The server was not stopped");
1677 assert.isFalse(trialStarted);
1678 assert.equal(state.activeVersion, "1.0.0");
1679 assert.isUndefined(state.update);
1680 }),
1681 );
1682
1683 it.effect("refuses the update when free space cannot be checked", () =>
1684 Effect.gen(function* () {
1685 const { reason, trialStarted, state } = yield* requestRejectedUpdate({
1686 readFreeBytes: () => Promise.reject(new Error("statfs unavailable")),
1687 });
1688
1689 assert.include(
1690 reason,
1691 "Could not check that the database backup fits before updating: statfs unavailable",
1692 );
1693 assert.isFalse(trialStarted);
1694 assert.equal(state.activeVersion, "1.0.0");
1695 assert.isUndefined(state.update);
1696 }),
1697 );
1698
1699 it.effect("holds its lock while it serves and stops the server over IPC, marker set", () =>
1700 Effect.gen(function* () {
1701 const fs = yield* FileSystem.FileSystem;
1702 const path = yield* Path.Path;
1703 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-" });
1704 const statePath = path.join(root, "runtime", "service-state.json");
1705 const databasePath = path.join(root, "userdata", "state.sqlite");
1706 const reportPath = path.join(root, "stopped-with-marker");
1707 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1708 yield* fs.writeFileString(databasePath, "database");
1709 const encoded = JSON.stringify({
1710 databasePath,
1711 reportPath,
1712 markerPath: path.join(root, "runtime", SERVICE_STOP_MARKER_FILE),
1713 });
1714 const childSource = `
1715 const fs = require("node:fs");
1716 const paths = ${encoded};
1717 process.on("message", (message) => {
1718 if (message.type !== "stop") return;
1719 fs.writeFileSync(paths.reportPath, String(fs.existsSync(paths.markerPath)));
1720 process.exit(0);
1721 });
1722 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
1723 `;
1724 for (const version of ["1.0.0", "1.1.0"]) {
1725 yield* writeFakeRuntime(
1726 fs,
1727 path,
1728 path.join(root, "runtime", "versions", version),
1729 childSource,
1730 );
1731 }
1732 yield* Effect.promise(() =>
1733 writeServiceState(statePath, {
1734 protocol: SERVICE_LAUNCHER_PROTOCOL,
1735 activeVersion: "1.0.0",
1736 }),
1737 );
1738 const lockPath = serviceLauncherLockPath(root);
1739 const serving = Promise.withResolvers<void>();
1740 const launcher = new Launcher(
1741 root,
1742 yield* Effect.promise(() => readServiceState(statePath)),
1743 {
1744 lockPath,
1745 stopChildOverIpc: true,
1746 // The update request proves the child is up; refusing it leaves the server running.
1747 backupSpaceProbe: {
1748 readFreeBytes: () => {
1749 serving.resolve();
1750 return Promise.resolve(0);
1751 },
1752 canClone: () => Promise.resolve(false),
1753 },
1754 },
1755 );
1756 const running = launcher.run();
1757 yield* Effect.promise(() => Promise.race([serving.promise, running]));
1758
1759 assert.isTrue(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath)));
1760 // What losing the task's console does.
1761 yield* Effect.promise(() => launcher.stop("SIGTERM"));
1762 yield* Effect.promise(() => running);
1763 assert.equal(yield* fs.readFileString(reportPath), "true");
1764 assert.isFalse(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath)));
1765 }),
1766 );
1767
1768 it.effect("leaves a pending update alone when stopped while it starts up", () =>
1769 Effect.gen(function* () {
1770 const fs = yield* FileSystem.FileSystem;
1771 const path = yield* Path.Path;
1772 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-startup-stop-" });
1773 const statePath = path.join(root, "runtime", "service-state.json");
1774 // Recovery would roll this back: the target runtime is missing.
1775 const pending = {
1776 protocol: SERVICE_LAUNCHER_PROTOCOL,
1777 activeVersion: "1.0.0",
1778 update: {
1779 id: "update-1",
1780 fromVersion: "1.0.0",
1781 targetVersion: "1.1.0",
1782 dbPath: path.join(root, "userdata", "state.sqlite"),
1783 status: "pending",
1784 },
1785 } as const;
1786 yield* Effect.promise(() => writeServiceState(statePath, pending));
1787 const lockPath = serviceLauncherLockPath(root);
1788 const launcher = new Launcher(root, pending, { lockPath });
1789
1790 const running = launcher.run();
1791 yield* Effect.promise(() => launcher.stop("SIGTERM"));
1792 yield* Effect.promise(() => running);
1793
1794 // The lock is released only once the launcher is done with the home.
1795 assert.isFalse(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath)));
1796 assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), pending);
1797 }),
1798 );
1799
1800 it.effect("starts no trial when stopped during the pre-update backup", () =>
1801 Effect.gen(function* () {
1802 const fs = yield* FileSystem.FileSystem;
1803 const path = yield* Path.Path;
1804 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-backup-stop-" });
1805 const statePath = path.join(root, "runtime", "service-state.json");
1806 const childSource = "setInterval(() => {}, 1000);\n";
1807 for (const version of ["1.0.0", "1.1.0"]) {
1808 yield* writeFakeRuntime(
1809 fs,
1810 path,
1811 path.join(root, "runtime", "versions", version),
1812 childSource,
1813 );
1814 }
1815 const dbPath = path.join(root, "userdata", "state.sqlite");
1816 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
1817 yield* fs.writeFileString(dbPath, "database");
1818 const pending = {
1819 protocol: SERVICE_LAUNCHER_PROTOCOL,
1820 activeVersion: "1.0.0",
1821 update: {
1822 id: "update-1",
1823 fromVersion: "1.0.0",
1824 targetVersion: "1.1.0",
1825 dbPath,
1826 status: "pending",
1827 },
1828 } as const;
1829 yield* Effect.promise(() => writeServiceState(statePath, pending));
1830 const childCount = spawnedChildren.length;
1831 const launcher = new Launcher(root, pending);
1832
1833 // Recovery is already queued, so this stop lands while the backup runs.
1834 const running = launcher.run();
1835 yield* Effect.promise(() => launcher.stop("SIGTERM"));
1836 yield* Effect.promise(() => running);
1837
1838 // A trial started here arms its 30-minute prepared timer, which keeps the stopped launcher alive.
1839 assert.deepEqual(
1840 spawnedChildren.slice(childCount).map((child) => child.spawnargs.join(" ")),
1841 [],
1842 );
1843 assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), pending);
1844 const backupDir = path.join(root, "runtime", "db-backup", "update-1");
1845 assert.isFalse(yield* fs.exists(backupDir));
1846 assert.isFalse(yield* fs.exists(`${backupDir}.staging`));
1847 }),
1848 );
1849
1850 it.effect("records no update when the server exits during the backup check", () =>
1851 Effect.gen(function* () {
1852 const fs = yield* FileSystem.FileSystem;
1853 const path = yield* Path.Path;
1854 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-exit-" });
1855 const statePath = path.join(root, "runtime", "service-state.json");
1856 const databasePath = path.join(root, "userdata", "state.sqlite");
1857 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
1858 yield* fs.writeFileString(databasePath, "database");
1859 const encoded = JSON.stringify({ databasePath });
1860 const childSource = `
1861 const paths = ${encoded};
1862 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }, () =>
1863 process.exit(3),
1864 );
1865 setInterval(() => {}, 1_000);
1866 `;
1867 for (const version of ["1.0.0", "1.1.0"]) {
1868 yield* writeFakeRuntime(
1869 fs,
1870 path,
1871 path.join(root, "runtime", "versions", version),
1872 childSource,
1873 );
1874 }
1875 yield* Effect.promise(() =>
1876 writeServiceState(statePath, {
1877 protocol: SERVICE_LAUNCHER_PROTOCOL,
1878 activeVersion: "1.0.0",
1879 }),
1880 );
1881 const launcher = new Launcher(
1882 root,
1883 yield* Effect.promise(() => readServiceState(statePath)),
1884 {
1885 backupSpaceProbe: {
1886 readFreeBytes: () => {
1887 const child = spawnedChildren.at(-1)!;
1888 return new Promise((resolve) => {
1889 if (!child.connected) resolve(Number.MAX_SAFE_INTEGER);
1890 else child.once("disconnect", () => resolve(Number.MAX_SAFE_INTEGER));
1891 });
1892 },
1893 },
1894 },
1895 );
1896 const failure = yield* Effect.promise(() =>
1897 launcher.run().then(
1898 () => new Error("launcher unexpectedly completed"),
1899 (cause: unknown) => cause as Error,
1900 ),
1901 );
1902
1903 assert.include(failure.message, "Active child exited unexpectedly (3)");
1904 const state = yield* Effect.promise(() => readServiceState(statePath));
1905 assert.equal(state.activeVersion, "1.0.0");
1906 assert.isUndefined(state.update);
1907 }),
1908 );
1909
1910 it.effect("reclaims partial backups left by killed updates before checking space", () =>
1911 Effect.gen(function* () {
1912 const fs = yield* FileSystem.FileSystem;
1913 const path = yield* Path.Path;
1914 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" });
1915 const databasePath = path.join(root, "state.sqlite");
1916 yield* writeSparseFile(databasePath, 10 * 1024 ** 3);
1917 const backupRoot = path.join(root, "runtime", "db-backup");
1918 const orphan = path.join(backupRoot, "earlier-update.staging");
1919 const completed = path.join(backupRoot, "completed-update");
1920 yield* fs.makeDirectory(orphan, { recursive: true });
1921 yield* fs.writeFileString(path.join(orphan, "database"), "partial copy");
1922 yield* fs.makeDirectory(completed);
1923 yield* fs.writeFileString(path.join(completed, "database"), "complete copy");
1924
1925 // The orphan is what keeps this disk too full for a full copy.
1926 const blocker = yield* Effect.promise(() =>
1927 databaseBackupBlocker(root, databasePath, {
1928 readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3),
1929 canClone: () => Promise.resolve(false),
1930 }),
1931 );
1932
1933 assert.isUndefined(blocker);
1934 assert.isFalse(yield* fs.exists(orphan));
1935 assert.isTrue(yield* fs.exists(path.join(completed, "database")));
1936 }),
1937 );
1938
1939 it.effect("checks database backup space against clone or full-copy needs", () =>
1940 Effect.gen(function* () {
1941 const fs = yield* FileSystem.FileSystem;
1942 const path = yield* Path.Path;
1943 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" });
1944 const databasePath = path.join(root, "state.sqlite");
1945 yield* writeSparseFile(databasePath, 10 * 1024 ** 3);
1946 const check = (free: number, cloneable: boolean) =>
1947 Effect.promise(() =>
1948 databaseBackupBlocker(root, databasePath, {
1949 readFreeBytes: () => Promise.resolve(free),
1950 canClone: () => Promise.resolve(cloneable),
1951 }),
1952 );
1953
1954 assert.isUndefined(yield* check(11 * 1024 ** 3, false));
1955 assert.isUndefined(yield* check(1.5 * 1024 ** 3, true));
1956 assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free");
1957 const blocker = yield* check(8 * 1024 ** 3, false);
1958 assert.include(blocker ?? "", "Not enough free disk space");
1959 assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free");
1960 assert.include(blocker ?? "", "The server was not stopped");
1961 }),
1962 );
1963 });