@devries-cameron20 / t3code-1 / commits / d0d0608b6b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a remote update points the service unit at the runtime it committed (#19652)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Ibrahim Elkamali committed Nov 11, 2026 at 12:48 UTC d0d0608b6be8c50cdc38b5045de76c5d0b9407cb
9 files changed +937 -128
apps/server/src/cloud/bootService.test.ts
+194
@@ -1,3 +1,5 @@
1 +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers.
2 +import * as NodeChildProcess from "node:child_process";
3 import * as NodeServices from "@effect/platform-node/NodeServices";
4 import { expect, it } from "@effect/vitest";
5 import * as HostProcess from "@t3tools/shared/HostProcess";
@@ -5,11 +7,13 @@ import * as ConfigProvider from "effect/ConfigProvider";
7 import * as Duration from "effect/Duration";
8 import * as Effect from "effect/Effect";
9 import * as FileSystem from "effect/FileSystem";
10 +import * as Fiber from "effect/Fiber";
11 import * as Layer from "effect/Layer";
12 import * as Path from "effect/Path";
13 import { CliError } from "effect/cli";
14 import { HttpClient } from "effect/http";
15 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
16 +import { vi } from "vite-plus/test";
17
18 import * as ProcessRunner from "../processRunner.ts";
19 import * as BootService from "./bootService.ts";
@@ -22,6 +26,24 @@ import {
26 serviceStateHasPendingUpdate,
27 } from "./serviceProtocol.ts";
28
29 +const leaseObserver = vi.hoisted(() => ({ contended: (_destination: string) => {} }));
30 +vi.mock("node:fs", async (importOriginal) => {
31 + const actual = await importOriginal<typeof import("node:fs")>();
32 + return {
33 + ...actual,
34 + renameSync: (...args: Parameters<typeof actual.renameSync>) => {
35 + try {
36 + return actual.renameSync(...args);
37 + } catch (error) {
38 + if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) {
39 + leaseObserver.contended(String(args[1]));
40 + }
41 + throw error;
42 + }
43 + },
44 + };
45 +});
46 +
47 it("reads the served T3 home back out of a rendered unit or plist", () => {
48 const plan = (baseDir: string) => ({
49 program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"],
@@ -442,6 +464,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
464 current: false,
465 installedVersion: "1.2.4-nightly.1",
466 });
467 + yield* service.install({ allowDowngrade: true });
468 + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
469 + protocol: SERVICE_LAUNCHER_PROTOCOL,
470 + activeVersion: "1.2.3",
471 + });
472 }
473 }),
474 );
@@ -575,6 +602,168 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
602 expect(yield* newer.restart).toBe("restarted");
603 expect((yield* newer.status).problems).not.toContain("restart-pending");
604 expect((yield* newer.status).current).toBe(true);
605 +
606 + const latest = yield* makeService(undefined, "1.2.5");
607 + yield* latest.install({ start: true });
608 + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
609 + protocol: SERVICE_LAUNCHER_PROTOCOL,
610 + activeVersion: "1.2.5",
611 + });
612 + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.5/t3");
613 + expect((yield* latest.status).current).toBe(true);
614 + expect(
615 + yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)),
616 + ).toBe(false);
617 + }),
618 + );
619 +
620 + it.effect(
621 + "serializes a local install with another process reconciling an older unit snapshot",
622 + () =>
623 + Effect.gen(function* () {
624 + const { service, fs, statePath, makeService, runtime } = yield* makeHarness();
625 + const path = yield* Path.Path;
626 + const plan = yield* service.install();
627 + const unit = yield* fs.readFileString(plan.unitPath);
628 + yield* fs.writeFileString(
629 + plan.unitPath,
630 + unit.replace("versions/1.2.3/t3", "versions/1.2.2/t3"),
631 + );
632 + yield* fs.writeFileString(
633 + statePath,
634 + JSON.stringify({
635 + protocol: SERVICE_LAUNCHER_PROTOCOL,
636 + activeVersion: "1.2.3",
637 + update: {
638 + id: "committed",
639 + fromVersion: "1.2.2",
640 + targetVersion: "1.2.3",
641 + status: "committed",
642 + },
643 + }),
644 + );
645 + const rejection = path.join(plan.baseDir, "rejected-update");
646 + const databasePath = path.join(plan.baseDir, "state.sqlite");
647 + yield* fs.writeFileString(databasePath, "database");
648 + yield* fs.writeFileString(
649 + runtime.entryPath,
650 + `#!${process.execPath}
651 +import { writeFileSync } from "node:fs";
652 +process.on("message", message => {
653 + if (message === "request-update") process.send({ type: "request-update", targetVersion: "1.2.5", dbPath: ${JSON.stringify(databasePath)} });
654 + if (message.type === "update-rejected") {
655 + writeFileSync(${JSON.stringify(rejection)}, message.reason);
656 + process.exit(0);
657 + }
658 +});
659 +`,
660 + );
661 + yield* fs.chmod(runtime.entryPath, 0o755);
662 + const newer = yield* makeService(undefined, "1.2.4");
663 + yield* makeService(undefined, "1.2.5");
664 + const snapshot = Promise.withResolvers<void>();
665 + const milestone = Promise.withResolvers<void>();
666 + const script = path.join(plan.baseDir, "launcher-contention.mjs");
667 + yield* fs.writeFileString(
668 + script,
669 + `
670 +import fs from "node:fs/promises";
671 +import cp from "node:child_process";
672 +import { syncBuiltinESMExports } from "node:module";
673 +const server = Promise.withResolvers();
674 +const originalSpawn = cp.spawn;
675 +cp.spawn = (...args) => { const child = originalSpawn(...args); server.resolve(child); return child; };
676 +process.on("message", async message => {
677 + if (message === "installed") (await server.promise).send("request-update");
678 +});
679 +const originalRead = fs.readFile;
680 +fs.readFile = async (...args) => {
681 + const contents = await originalRead(...args);
682 + if (args[0] === ${JSON.stringify(plan.unitPath)}) {
683 + fs.readFile = originalRead;
684 + syncBuiltinESMExports();
685 + const resume = new Promise(resolve => process.once("message", resolve));
686 + process.send("snapshot");
687 + await resume;
688 + }
689 + return contents;
690 +};
691 +syncBuiltinESMExports();
692 +const { Launcher, readServiceState } = await import(${JSON.stringify(import.meta.resolve("../serviceLauncher.ts"))});
693 +const launcher = new Launcher(${JSON.stringify(plan.baseDir)}, await readServiceState(${JSON.stringify(statePath)}), {
694 + serviceUnit: { path: ${JSON.stringify(plan.unitPath)}, reload: async () => {} },
695 +});
696 +try { await launcher.run(); throw Error("unexpected completion"); }
697 +catch (error) {
698 + if (!error.message.includes("Active child exited unexpectedly (0)")) throw error;
699 +}
700 +const { acquireServiceSelectionLock } = await import(${JSON.stringify(import.meta.resolve("./serviceSelectionLock.ts"))});
701 +// Exit owning the real lease, as a crashed process would.
702 +await acquireServiceSelectionLock(${JSON.stringify(plan.unitPath)});
703 +process.disconnect();
704 +`,
705 + );
706 + const child = yield* Effect.acquireRelease(
707 + Effect.sync(() =>
708 + NodeChildProcess.fork(script, [], {
709 + execArgv: [],
710 + stdio: ["ignore", "inherit", "inherit", "ipc"],
711 + }),
712 + ),
713 + (child) =>
714 + Effect.sync(() => {
715 + if (child.exitCode === null && child.signalCode === null) child.kill();
716 + }),
717 + );
718 + const exited = new Promise<void>((resolve, reject) => {
719 + child.once("error", reject);
720 + child.once("exit", (code) =>
721 + code === 0 ? resolve() : reject(new Error(`launcher exited ${code}`)),
722 + );
723 + });
724 + child.on("message", (message) => {
725 + if (message === "snapshot") snapshot.resolve();
726 + });
727 + yield* Effect.promise(() =>
728 + Promise.race([
729 + snapshot.promise,
730 + exited.then(() => {
731 + throw new Error("launcher exited before snapshot");
732 + }),
733 + ]),
734 + );
735 + leaseObserver.contended = (destination) => {
736 + if (destination === plan.unitPath + ".selection-lock") milestone.resolve();
737 + };
738 + yield* Effect.addFinalizer(() =>
739 + Effect.sync(() => {
740 + leaseObserver.contended = () => {};
741 + }),
742 + );
743 + const installing = yield* newer
744 + .install({ start: false })
745 + .pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild);
746 + // Resume the launcher once the installer contends for the unit lease, or once an
747 + // uncoordinated installer has finished, which makes the stale snapshot win.
748 + yield* Effect.promise(() => milestone.promise);
749 + child.send("resume");
750 + yield* Fiber.join(installing);
751 + child.send("installed");
752 + yield* Effect.promise(() => exited);
753 + expect(yield* fs.readFileString(rejection)).toContain("Service selection changed");
754 + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
755 + protocol: SERVICE_LAUNCHER_PROTOCOL,
756 + activeVersion: "1.2.4",
757 + });
758 + expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3");
759 + expect(
760 + yield* fs.readFileString(
761 + path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE),
762 + ),
763 + ).toBe("1.2.4\n");
764 + // The next installer reclaims the dead owner and releases its own lease.
765 + yield* newer.install({ start: false });
766 + expect(yield* fs.exists(plan.unitPath + ".selection-lock")).toBe(false);
767 }),
768 );
769
@@ -617,6 +806,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
806 const error = yield* service.install({ start: false }).pipe(Effect.flip);
807 expect(error._tag).toBe("BootServiceUpdatePendingError");
808 expect(yield* fs.readFileString(statePath)).toBe(pendingState);
809 + yield* fs.remove((yield* service.status).unitPath);
810 + expect((yield* service.install({ start: false }).pipe(Effect.flip))._tag).toBe(
811 + "BootServiceUpdatePendingError",
812 + );
813 + expect(yield* fs.readFileString(statePath)).toBe(pendingState);
814 }),
815 );
816
apps/server/src/cloud/bootService.ts
+102 -84
@@ -16,6 +16,7 @@ import * as Schema from "effect/Schema";
16 import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease";
17
18 import * as ProcessRunner from "../processRunner.ts";
19 +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts";
20 import {
21 ensurePinnedRuntimeInstalled,
22 launcherOwnsVersionsDir,
@@ -28,20 +29,26 @@ import {
29 SERVICE_LAUNCHER_PROTOCOL,
30 SERVICE_RESTART_PENDING_FILE,
31 SERVICE_STATE_FILE,
32 + SERVICE_UNIT_ENV,
33 compareExactServiceVersions,
34 + serviceUnitBaseDirOf as bootServiceBaseDirOf,
35 parseServiceState,
36 serviceStateActiveVersion,
37 serviceStateHasPendingUpdate,
38 type ServiceState,
39 } from "./serviceProtocol.ts";
40
41 +export {
42 + serviceUnitBaseDirOf as bootServiceBaseDirOf,
43 + serviceUnitProgramOf as bootServiceProgramOf,
44 +} from "./serviceProtocol.ts";
45 +
46 const BOOT_SERVICE_NAME = "t3code";
47 export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`;
48 // `.service` suffix keeps the label distinct from the desktop app's bundle id
49 // (com.t3tools.t3code), so launchd and TCC records never collide.
50 export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service";
51 export const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`;
44 -const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT";
52 export const BOOT_SERVICE_LOG_FILE = "boot-service.log";
53
54 /** systemd expands `%` specifiers, including in unquoted append-log paths. */
@@ -56,32 +63,6 @@ function quoteSystemdValue(value: string): string {
63 : escaped;
64 }
65
59 -const unquoteSystemdValue = (raw: string) =>
60 - (raw.startsWith('"') && raw.endsWith('"')
61 - ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\")
62 - : raw
63 - ).replaceAll("%%", "%");
64 -
65 -const unescapeXmlText = (value: string) =>
66 - value.replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&amp;", "&");
67 -
68 -export function bootServiceBaseDirOf(contents: string): string | undefined {
69 - const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1];
70 - if (systemd !== undefined) return unquoteSystemdValue(systemd.trim());
71 - const plist = /<key>T3CODE_HOME<\/key>\s*<string>([^<]*)<\/string>/.exec(contents)?.[1];
72 - return plist === undefined ? undefined : unescapeXmlText(plist);
73 -}
74 -
75 -/** The executable an installed unit or plist starts: the launcher's runtime. */
76 -export function bootServiceProgramOf(contents: string): string | undefined {
77 - const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1];
78 - if (systemd !== undefined) return unquoteSystemdValue(systemd);
79 - const plist = /<key>ProgramArguments<\/key>\s*<array>\s*<string>([^<]*)<\/string>/.exec(
80 - contents,
81 - )?.[1];
82 - return plist === undefined ? undefined : unescapeXmlText(plist);
83 -}
84 -
66 export interface BootServicePlan {
67 readonly program: ReadonlyArray<string>;
68 readonly baseDir: string;
@@ -104,7 +85,7 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
85 "Type=simple",
86 "WorkingDirectory=%h",
87 `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`,
107 - `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`,
88 + `Environment=${SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`,
89 `ExecStart=${plan.program.map(quoteSystemdValue).join(" ")}`,
90 // Let the launcher mark an explicit stop before it signals the server.
91 // systemd still SIGKILLs the whole cgroup if graceful shutdown times out.
@@ -162,7 +143,7 @@ export function renderBootServicePlist(
143 ` <string>${escapeXmlText(options.environmentPath)}</string>`,
144 ` <key>T3CODE_HOME</key>`,
145 ` <string>${escapeXmlText(plan.baseDir)}</string>`,
165 - ` <key>${BOOT_SERVICE_UNIT_ENV}</key>`,
146 + ` <key>${SERVICE_UNIT_ENV}</key>`,
147 ` <string>${BOOT_SERVICE_PLIST_FILE}</string>`,
148 ` </dict>`,
149 ` <key>WorkingDirectory</key>`,
@@ -496,9 +477,12 @@ export class BootServicePrerequisiteError extends Schema.TaggedError<BootService
477
478 export class BootServiceUpdatePendingError extends Schema.TaggedError<BootServiceUpdatePendingError>()(
479 "BootServiceUpdatePendingError",
499 - {},
480 + { recoveryCommand: Schema.optional(Schema.String) },
481 ) {
482 override get message(): string {
483 + if (this.recoveryCommand !== undefined) {
484 + return `A remote server update is still pending, but its service definition is missing. With the service stopped, run this on the server machine: ${this.recoveryCommand}\nWait for the update to finish or roll back, stop the foreground launcher with Ctrl-C, then retry the install.`;
485 + }
486 return "A remote server update is still pending. Wait for it to finish, then retry.";
487 }
488 }
@@ -606,7 +590,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
590 const xdgConfigHome = yield* Config.String("XDG_CONFIG_HOME").pipe(Config.withDefault(""));
591 const installerPath = yield* Config.String("PATH").pipe(Config.withDefault(""));
592 const runningInsideService =
609 - (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) ===
593 + (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) ===
594 BOOT_SERVICE_UNIT_FILE;
595 const fs = yield* FileSystem.FileSystem;
596 const path = yield* Path.Path;
@@ -799,6 +783,16 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
783 if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] });
784 });
785
786 + const withSelectionLock = <A, E, R>(selectionPath: string, operation: Effect.Effect<A, E, R>) =>
787 + Effect.acquireUseRelease(
788 + Effect.tryPromise({
789 + try: () => acquireServiceSelectionLock(selectionPath),
790 + catch: (cause) => new BootServiceInstallError({ cause }),
791 + }),
792 + () => operation,
793 + (release) => Effect.sync(release),
794 + );
795 +
796 const install = Effect.fn("cloud.boot_service.install")(function* (options?: {
797 readonly allowDowngrade?: boolean;
798 readonly start?: boolean;
@@ -876,8 +870,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
870 const installed = yield* fs
871 .exists(unitPath)
872 .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
879 - // With start=false, changing startup-only files is safe except during a remote update,
880 - // when the launcher also writes its state file.
873 + // Stop outside the selection lock: the launcher may be waiting to acquire it.
874 const start = options?.start !== false;
875 if (installed && start && runningInsideService && manager.kind === "systemd") {
876 // Stopping the unit from inside it kills this process before the start.
@@ -890,7 +883,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
883 // Transient properties reach systemd literally, unlike unit-file specifiers.
884 `--property=StandardOutput=append:${logPath}`,
885 `--property=StandardError=append:${logPath}`,
893 - `--setenv=${BOOT_SERVICE_UNIT_ENV}=`,
886 + `--setenv=${SERVICE_UNIT_ENV}=`,
887 runtimePaths.entryPath,
888 "service",
889 "install",
@@ -905,58 +898,83 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
898 yield* runSteps(manager.stop);
899 }
900
908 - yield* Effect.gen(function* () {
909 - if (installed) {
910 - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);
911 - if (Option.isSome(previousStateText)) {
912 - if (serviceStateHasPendingUpdate(previousStateText.value)) {
913 - return yield* new BootServiceUpdatePendingError();
901 + // State first, then unit. The launcher acquires only one at a time, and a
902 + // failure to lock the unit must never prevent its durable update commit.
903 + yield* withSelectionLock(
904 + statePath,
905 + withSelectionLock(
906 + unitPath,
907 + Effect.gen(function* () {
908 + const currentlyInstalled = yield* fs.exists(unitPath);
909 + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);
910 + if (Option.isSome(previousStateText)) {
911 + const installedVersion = serviceStateActiveVersion(previousStateText.value);
912 + if (serviceStateHasPendingUpdate(previousStateText.value)) {
913 + const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`;
914 + return yield* new BootServiceUpdatePendingError(
915 + !currentlyInstalled && installedVersion !== undefined
916 + ? {
917 + recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`,
918 + }
919 + : {},
920 + );
921 + }
922 + // Read the final version after stopping the launcher; a remote update may finish after the status check.
923 + if (
924 + installedVersion !== undefined &&
925 + options?.allowDowngrade !== true &&
926 + compareExactServiceVersions(input.cliVersion, installedVersion) < 0
927 + ) {
928 + return yield* new BootServiceDowngradeRefusedError({
929 + installedVersion,
930 + targetVersion: input.cliVersion,
931 + });
932 + }
933 }
915 - // Read the final version after stopping the launcher; a remote update may finish after the status check.
916 - const installedVersion = serviceStateActiveVersion(previousStateText.value);
917 - if (
918 - installedVersion !== undefined &&
919 - options?.allowDowngrade !== true &&
920 - compareExactServiceVersions(input.cliVersion, installedVersion) < 0
921 - ) {
922 - return yield* new BootServiceDowngradeRefusedError({
923 - installedVersion,
924 - targetVersion: input.cliVersion,
925 - });
934 + yield* fs
935 + .makeDirectory(path.dirname(unitPath), { recursive: true })
936 + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
937 + // Files can match the new version even when activation fails; keep that install retryable.
938 + yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
939 + yield* writeDurably(
940 + statePath,
941 + // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document.
942 + `${JSON.stringify(
943 + {
944 + protocol: SERVICE_LAUNCHER_PROTOCOL,
945 + activeVersion: input.cliVersion,
946 + } satisfies ServiceState,
947 + null,
948 + 2,
949 + )}\n`,
950 + );
951 + if (!start && (currentlyInstalled || Option.isSome(previousStateText))) {
952 + const written = yield* fs.readFileString(statePath);
953 + if (serviceStateActiveVersion(written) !== input.cliVersion) {
954 + return yield* new BootServiceUpdatePendingError();
955 + }
956 }
927 - }
928 - }
929 - yield* fs
930 - .makeDirectory(path.dirname(unitPath), { recursive: true })
931 - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
932 - // Files can match the new version even when activation fails; keep that install retryable.
933 - yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
934 - yield* writeDurably(
935 - statePath,
936 - // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document.
937 - `${JSON.stringify(
938 - {
939 - protocol: SERVICE_LAUNCHER_PROTOCOL,
940 - activeVersion: input.cliVersion,
941 - } satisfies ServiceState,
942 - null,
943 - 2,
944 - )}\n`,
945 - );
946 - if (!start && installed) {
947 - // A concurrent remote update can overwrite launcher state; detect it before repointing the unit.
948 - const written = yield* fs.readFileString(statePath);
949 - if (serviceStateActiveVersion(written) !== input.cliVersion) {
950 - return yield* new BootServiceUpdatePendingError();
951 - }
952 - }
953 - yield* writeDurably(unitPath, manager.render(plan));
954 -
955 - if (start) {
956 - yield* runSteps(manager.activate);
957 - yield* fs.remove(restartPendingPath, { force: true });
958 - }
959 - }).pipe(
957 + yield* writeDurably(unitPath, manager.render(plan));
958 + }),
959 + ),
960 + ).pipe(
961 + Effect.andThen(
962 + Effect.gen(function* () {
963 + if (start) {
964 + yield* runSteps(manager.activate);
965 + yield* withSelectionLock(
966 + statePath,
967 + Effect.gen(function* () {
968 + // A concurrent deferred install may now be waiting for a newer restart.
969 + const selected = parseServiceState(yield* fs.readFileString(statePath));
970 + if (selected?.activeVersion === input.cliVersion && selected.update === undefined) {
971 + yield* fs.remove(restartPendingPath, { force: true });
972 + }
973 + }),
974 + );
975 + }
976 + }),
977 + ),
978 Effect.mapError((cause) =>
979 cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause,
980 ),
apps/server/src/cloud/serviceProtocol.ts
+35 -3
@@ -1,15 +1,46 @@
1 import type { ServerSelfUpdateOutcome } from "@t3tools/contracts";
2
3 -// Bump for incompatible runtime layout changes; launchers survive self-updates. Protocol 3 requires standalone executables.
4 -export const SERVICE_LAUNCHER_PROTOCOL = 3 as const;
3 +// Protocol 4 requires coordinated selection writes and unit reconciliation. An
4 +// installed protocol-3 launcher needs one local update or service install with a restart:
5 +// remote updates replace its child, so cannot add these launcher guarantees.
6 +export const SERVICE_LAUNCHER_PROTOCOL = 4 as const;
7 export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
8 export const SERVICE_ENV_FILE = "service.env";
9 export const SERVICE_STATE_FILE = "service-state.json";
10 +/** Set by the unit or launch agent to its own file name, so the launcher can
11 + find the definition that started it. */
12 +export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT";
13 /** Written before explicit stop to distinguish shutdown from update handoff. */
14 export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
15 /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */
16 export const SERVICE_RESTART_PENDING_FILE = ".restart-pending";
17
18 +const unquoteSystemdValue = (raw: string) =>
19 + (raw.startsWith('"') && raw.endsWith('"')
20 + ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\")
21 + : raw
22 + ).replaceAll("%%", "%");
23 +
24 +const unescapeXmlText = (value: string) =>
25 + value.replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&amp;", "&");
26 +
27 +export function serviceUnitBaseDirOf(contents: string): string | undefined {
28 + const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1];
29 + if (systemd !== undefined) return unquoteSystemdValue(systemd.trim());
30 + const plist = /<key>T3CODE_HOME<\/key>\s*<string>([^<]*)<\/string>/.exec(contents)?.[1];
31 + return plist === undefined ? undefined : unescapeXmlText(plist);
32 +}
33 +
34 +/** The executable an installed unit or plist starts: the launcher's runtime. */
35 +export function serviceUnitProgramOf(contents: string): string | undefined {
36 + const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1];
37 + if (systemd !== undefined) return unquoteSystemdValue(systemd);
38 + const plist = /<key>ProgramArguments<\/key>\s*<array>\s*<string>([^<]*)<\/string>/.exec(
39 + contents,
40 + )?.[1];
41 + return plist === undefined ? undefined : unescapeXmlText(plist);
42 +}
43 +
44 export interface PendingServiceUpdate {
45 readonly id: string;
46 readonly fromVersion: string;
@@ -143,7 +174,8 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
174 if (!isRecord(value)) return undefined;
175 const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update);
176 if (
146 - value.protocol !== SERVICE_LAUNCHER_PROTOCOL ||
177 + // Protocol 3 has the same durable format; its installer can select a protocol-4 runtime.
178 + (value.protocol !== 3 && value.protocol !== SERVICE_LAUNCHER_PROTOCOL) ||
179 typeof value.activeVersion !== "string" ||
180 !isExactServiceVersion(value.activeVersion) ||
181 (value.update !== undefined && update === undefined) ||
apps/server/src/cloud/serviceSelectionLock.test.ts new
+30
@@ -0,0 +1,30 @@
1 +import * as NodeServices from "@effect/platform-node/NodeServices";
2 +import { assert, it } from "@effect/vitest";
3 +import * as Effect from "effect/Effect";
4 +import * as FileSystem from "effect/FileSystem";
5 +import * as Path from "effect/Path";
6 +
7 +import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts";
8 +
9 +it.layer(NodeServices.layer)("service selection lock", (it) => {
10 + it.effect("reclaims a lease from an earlier boot even when its PID is live again", () =>
11 + Effect.gen(function* () {
12 + const fs = yield* FileSystem.FileSystem;
13 + const path = yield* Path.Path;
14 + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-selection-lock-" });
15 + const selectionPath = path.join(root, "service-state.json");
16 + const lock = `${selectionPath}.selection-lock`;
17 + const stale = `${process.pid}.1.00000000-0000-4000-8000-000000000000`;
18 + yield* fs.makeDirectory(lock, { recursive: true });
19 + yield* fs.writeFileString(path.join(lock, stale), "");
20 +
21 + const release = yield* Effect.promise(() => acquireServiceSelectionLock(selectionPath));
22 + const owners = yield* fs.readDirectory(lock);
23 + assert.equal(owners.length, 1);
24 + assert.notEqual(owners[0], stale);
25 + assert.isTrue(owners[0]!.startsWith(`${process.pid}.`));
26 + release();
27 + assert.isFalse(yield* fs.exists(lock));
28 + }),
29 + );
30 +});
apps/server/src/cloud/serviceSelectionLock.ts new
+88
@@ -0,0 +1,88 @@
1 +// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone launcher, which has no Effect runtime.
2 +// @effect-diagnostics globalTimers:off -- Bounded cross-process lease acquisition in the standalone launcher.
3 +// @effect-diagnostics globalDate:off -- The standalone launcher has no Effect clock for its acquisition deadline.
4 +import * as NodeCrypto from "node:crypto";
5 +import * as NodeFS from "node:fs";
6 +import * as NodeOS from "node:os";
7 +import * as NodePath from "node:path";
8 +
9 +// Coarse boot time: a lease from an earlier boot is dead even when its PID is live again.
10 +const bootEpochSeconds = () => Math.round(Date.now() / 1000 - NodeOS.uptime());
11 +
12 +/** Serialize durable selection and unit writes, never manager stop/start commands. */
13 +export async function acquireServiceSelectionLock(selectionPath: string): Promise<() => void> {
14 + const directory = NodePath.dirname(selectionPath);
15 + NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 });
16 + const lock = selectionPath + ".selection-lock";
17 + const owner = `${process.pid}.${bootEpochSeconds()}.${NodeCrypto.randomUUID()}`;
18 + const candidate = `${lock}.${owner}`;
19 + const deadline = Date.now() + 30_000;
20 + const removeEmptyLock = () => {
21 + try {
22 + NodeFS.rmdirSync(lock);
23 + } catch (error) {
24 + if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes((error as NodeJS.ErrnoException).code ?? ""))
25 + throw error;
26 + }
27 + };
28 + NodeFS.mkdirSync(candidate, { mode: 0o700 });
29 + try {
30 + NodeFS.writeFileSync(NodePath.join(candidate, owner), "", { mode: 0o600 });
31 + while (true) {
32 + try {
33 + // Publish the owner atomically; rename cannot replace another populated directory.
34 + NodeFS.renameSync(candidate, lock);
35 + return () => {
36 + NodeFS.unlinkSync(NodePath.join(lock, owner));
37 + removeEmptyLock();
38 + };
39 + } catch (error) {
40 + if (!["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? ""))
41 + throw error;
42 + }
43 + let owners: string[];
44 + try {
45 + owners = NodeFS.readdirSync(lock);
46 + } catch (error) {
47 + if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
48 + throw error;
49 + }
50 + if (owners.length === 1) {
51 + const previous = owners[0]!;
52 + const [pidText, bootText] = previous.split(".");
53 + const pid = Number(pidText);
54 + if (!Number.isSafeInteger(pid) || pid <= 0)
55 + throw new Error(`Invalid service selection lock owner at ${lock}.`);
56 + const ownerBoot = Number(bootText);
57 + const fromEarlierBoot =
58 + Number.isSafeInteger(ownerBoot) && Math.abs(ownerBoot - bootEpochSeconds()) > 60;
59 + let dead = fromEarlierBoot;
60 + if (!dead) {
61 + try {
62 + process.kill(pid, 0);
63 + } catch (error) {
64 + const code = (error as NodeJS.ErrnoException).code;
65 + if (code !== "ESRCH" && code !== "EPERM") throw error;
66 + dead = code === "ESRCH";
67 + }
68 + }
69 + if (dead) {
70 + // Remove only this dead owner's unique file, never a replacement owner's.
71 + try {
72 + NodeFS.unlinkSync(NodePath.join(lock, previous));
73 + } catch (error) {
74 + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
75 + }
76 + }
77 + }
78 + removeEmptyLock();
79 + if (Date.now() >= deadline)
80 + throw new Error(
81 + `Service selection is locked at ${lock}. Retry when the other operation finishes.`,
82 + );
83 + await new Promise<void>((resolve) => setTimeout(resolve, 50));
84 + }
85 + } finally {
86 + NodeFS.rmSync(candidate, { recursive: true, force: true });
87 + }
88 +}
apps/server/src/serviceLauncher.test.ts
+264 -20
@@ -2,6 +2,8 @@
2 import type * as NodeChildProcess from "node:child_process";
3 // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4 import * as NodeFS from "node:fs";
5 +// @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback.
6 +import * as NodeFSP from "node:fs/promises";
7 import * as NodeSqlite from "node:sqlite";
8
9 import * as NodeServices from "@effect/platform-node/NodeServices";
@@ -13,10 +15,10 @@ import * as FileSystem from "effect/FileSystem";
15 import * as Path from "effect/Path";
16 import { vi } from "vite-plus/test";
17
18 +import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts";
19 import {
20 databaseBackupBlocker,
21 Launcher,
19 - main,
22 readServiceEnvironment,
23 type BackupSpaceProbe,
24 readServiceState,
@@ -112,6 +114,13 @@ it("rejects contradictory service state", () => {
114 );
115 });
116
117 +const unitPlan = (root: string, version: string) => ({
118 + program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"],
119 + baseDir: root,
120 + logPath: `${root}/logs/boot-service.log`,
121 + unitPath: `${root}/unit/t3code.service`,
122 +});
123 +
124 const writeFakeRuntime = (
125 fs: FileSystem.FileSystem,
126 path: Path.Path,
@@ -130,6 +139,32 @@ const writeFakeRuntime = (
139 return entryPath;
140 });
141
142 +const fakeServiceUnit = (
143 + fs: FileSystem.FileSystem,
144 + path: Path.Path,
145 + root: string,
146 + version: string,
147 + render = renderBootServiceUnit,
148 +) =>
149 + Effect.gen(function* () {
150 + const plan = unitPlan(root, version);
151 + yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true });
152 + const contents = render(plan);
153 + yield* fs.writeFileString(plan.unitPath, contents);
154 + const unit = {
155 + path: plan.unitPath,
156 + reloads: 0,
157 + loadedContents: contents,
158 + failReload: false,
159 + reload: async () => {
160 + unit.reloads += 1;
161 + if (unit.failReload) throw new Error("daemon-reload failed");
162 + unit.loadedContents = await NodeFSP.readFile(unit.path, "utf8");
163 + },
164 + };
165 + return unit;
166 + });
167 +
168 it.layer(NodeServices.layer)("service state persistence", (it) => {
169 it.effect(
170 "loads optional service.env literally without changing inherited or managed variables",
@@ -217,6 +252,105 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
252 }),
253 );
254
255 + it.effect.each([
256 + { unitVersion: "1.0.0", replacement: "an intentional downgrade" },
257 + { unitVersion: "1.2.0", replacement: "another T3 home" },
258 + ])(
259 + "recovers a rolled-back update with unit $unitVersion and preserves $replacement",
260 + ({ unitVersion, replacement }) =>
261 + Effect.gen(function* () {
262 + const fs = yield* FileSystem.FileSystem;
263 + const path = yield* Path.Path;
264 + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" });
265 + const statePath = path.join(root, "runtime", "service-state.json");
266 + yield* writeFakeRuntime(
267 + fs,
268 + path,
269 + path.join(root, "runtime", "versions", "1.1.0"),
270 + "setInterval(() => {}, 1_000);\n",
271 + );
272 + yield* Effect.promise(() =>
273 + writeServiceState(statePath, {
274 + protocol: SERVICE_LAUNCHER_PROTOCOL,
275 + activeVersion: "1.1.0",
276 + update: {
277 + id: "update-1",
278 + fromVersion: "1.1.0",
279 + targetVersion: "1.2.0",
280 + status: "rolled-back",
281 + },
282 + }),
283 + );
284 + const unit = yield* fakeServiceUnit(fs, path, root, unitVersion);
285 + const expected = renderBootServiceUnit(
286 + unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion),
287 + );
288 + const createLauncher = Effect.gen(function* () {
289 + return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), {
290 + serviceUnit: unit,
291 + });
292 + });
293 + const run = (launcher: Launcher) =>
294 + Effect.gen(function* () {
295 + const running = launcher.run();
296 + yield* Effect.promise(() => launcher.stop("SIGTERM"));
297 + yield* Effect.promise(() => running);
298 + });
299 +
300 + yield* run(yield* createLauncher);
301 + assert.equal(yield* fs.readFileString(unit.path), expected);
302 + assert.equal(unit.loadedContents, expected);
303 +
304 + yield* run(yield* createLauncher);
305 + assert.equal(yield* fs.readFileString(unit.path), expected);
306 + assert.equal(unit.loadedContents, expected);
307 +
308 + yield* Effect.promise(() =>
309 + writeServiceState(statePath, {
310 + protocol: SERVICE_LAUNCHER_PROTOCOL,
311 + activeVersion: "1.1.0",
312 + update: {
313 + id: "update-2",
314 + fromVersion: "1.0.0",
315 + targetVersion: "1.1.0",
316 + status: "committed",
317 + },
318 + }),
319 + );
320 + const staleLauncher = yield* createLauncher;
321 + // A local install can replace the selection or service owner without stopping this launcher.
322 + const selectedRoot =
323 + replacement === "another T3 home" ? path.join(root, "other-home") : root;
324 + yield* writeFakeRuntime(
325 + fs,
326 + path,
327 + path.join(selectedRoot, "runtime", "versions", "1.0.0"),
328 + "setInterval(() => {}, 1_000);\n",
329 + );
330 + const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json");
331 + yield* Effect.promise(() =>
332 + writeServiceState(selectedStatePath, {
333 + protocol: SERVICE_LAUNCHER_PROTOCOL,
334 + activeVersion: "1.0.0",
335 + }),
336 + );
337 + const selectedUnit = renderBootServiceUnit({
338 + ...unitPlan(selectedRoot, "1.0.0"),
339 + unitPath: unit.path,
340 + });
341 + yield* fs.writeFileString(unit.path, selectedUnit);
342 + yield* Effect.promise(() => unit.reload());
343 + const oldHomeState = yield* fs.readFileString(statePath);
344 + const selectedState = yield* fs.readFileString(selectedStatePath);
345 +
346 + yield* run(staleLauncher);
347 + assert.equal(yield* fs.readFileString(statePath), oldHomeState);
348 + assert.equal(yield* fs.readFileString(selectedStatePath), selectedState);
349 + assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement);
350 + assert.equal(unit.loadedContents, selectedUnit, replacement);
351 + }),
352 + );
353 +
354 it.effect("stays idle instead of exiting when another server owns the T3 home", () =>
355 Effect.gen(function* () {
356 const fs = yield* FileSystem.FileSystem;
@@ -257,25 +391,49 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
391 }),
392 );
393
260 - it.effect("commits only after the trial reports prepared", () =>
394 + const commitUpdate = (
395 + render = renderBootServiceUnit,
396 + missingUnit = false,
397 + legacyInstall = false,
398 + ) =>
399 Effect.gen(function* () {
400 const fs = yield* FileSystem.FileSystem;
401 const path = yield* Path.Path;
264 - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
402 + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service 100% & <test>-" });
403 const statePath = path.join(root, "runtime", "service-state.json");
404 const databasePath = path.join(root, "userdata", "state.sqlite");
405 + const activatedPath = path.join(root, "activated");
406 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
407 yield* fs.writeFileString(databasePath, "before trial");
269 - const encodedDatabasePath = JSON.stringify(databasePath);
408 + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render);
409 + if (missingUnit) yield* fs.remove(unit.path);
410 const childSource = `
411 +import assert from "node:assert/strict";
412 +import { existsSync, readFileSync, writeFileSync } from "node:fs";
413 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
414 +assert.equal(context.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
415 if (context.update?.status === "pending") {
273 - process.send({ type: "prepared", updateId: context.update.id });
416 + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
417 + assert.equal(state.activeVersion, "1.0.0");
418 + assert.equal(state.update.status, "pending");
419 + ${
420 + missingUnit
421 + ? `assert.equal(existsSync(${JSON.stringify(unit.path)}), false);`
422 + : `assert.equal(readFileSync(${JSON.stringify(unit.path)}, "utf8"), ${JSON.stringify(unit.loadedContents)});`
423 + }
424 process.on("message", (message) => {
275 - if (message.type === "committed") process.exit(0);
425 + if (message.type === "committed") {
426 + const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
427 + assert.equal(state.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
428 + assert.equal(state.activeVersion, "1.1.0");
429 + assert.equal(state.update.status, "committed");
430 + writeFileSync(${JSON.stringify(activatedPath)}, message.updateId);
431 + process.exit(0);
432 + }
433 });
434 + process.send({ type: "prepared", updateId: context.update.id });
435 } else if (context.update === undefined) {
278 - process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
436 + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
437 setInterval(() => {}, 1_000);
438 } else {
439 process.exit(0);
@@ -289,25 +447,97 @@ if (context.update?.status === "pending") {
447 childSource,
448 );
449 }
292 - yield* Effect.promise(() =>
293 - writeServiceState(statePath, {
294 - protocol: SERVICE_LAUNCHER_PROTOCOL,
295 - activeVersion: "1.0.0",
296 - }),
450 + // The old CLI installs the new runtime but writes state with its own protocol.
451 + yield* fs.writeFileString(
452 + statePath,
453 + `${JSON.stringify(
454 + {
455 + protocol: legacyInstall ? 3 : SERVICE_LAUNCHER_PROTOCOL,
456 + activeVersion: "1.0.0",
457 + },
458 + null,
459 + 2,
460 + )}\n`,
461 );
298 -
299 - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
300 - yield* Effect.promise(() =>
462 + const launcher = new Launcher(
463 + root,
464 + yield* Effect.promise(() => readServiceState(statePath)),
465 + { serviceUnit: unit },
466 + );
467 + const failure = yield* Effect.promise(() =>
468 launcher.run().then(
302 - () => Promise.reject(new Error("launcher unexpectedly completed")),
303 - () => Promise.resolve(),
469 + () => "launcher unexpectedly completed",
470 + (cause: unknown) => String(cause),
471 ),
472 );
306 -
473 + assert.include(failure, "Active child exited unexpectedly (0)");
474 const state = yield* Effect.promise(() => readServiceState(statePath));
475 assert.equal(state.activeVersion, "1.1.0");
476 assert.equal(state.update?.status, "committed");
310 - }),
477 + assert.equal(yield* fs.readFileString(activatedPath), state.update?.id);
478 + const recover = () =>
479 + Effect.gen(function* () {
480 + const fresh = new Launcher(
481 + root,
482 + yield* Effect.promise(() => readServiceState(statePath)),
483 + { serviceUnit: unit },
484 + );
485 + const running = fresh.run();
486 + yield* Effect.promise(() => fresh.stop("SIGTERM"));
487 + yield* Effect.promise(() => running);
488 + });
489 + return { fs, root, unit, recover };
490 + });
491 +
492 + it.effect.each([
493 + {
494 + name: "systemd after a protocol-3 local upgrade",
495 + render: renderBootServiceUnit,
496 + legacyInstall: true,
497 + },
498 + {
499 + name: "launchd",
500 + legacyInstall: false,
501 + render: (plan: Parameters<typeof renderBootServicePlist>[0]) =>
502 + renderBootServicePlist(plan, {
503 + homeDir: "/Users/test & <user>",
504 + environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0",
505 + }),
506 + },
507 + ])(
508 + "commits and repoints $name only after the trial reports prepared",
509 + ({ render, legacyInstall }) =>
510 + Effect.gen(function* () {
511 + const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall);
512 + const expected = render(unitPlan(root, "1.1.0"));
513 + assert.equal(yield* fs.readFileString(unit.path), expected);
514 + assert.equal(unit.loadedContents, expected);
515 + assert.equal(unit.reloads, 1);
516 + }),
517 + );
518 +
519 + it.effect(
520 + "keeps a durable commit through unit failure and retries a failed reload on restart",
521 + () =>
522 + Effect.gen(function* () {
523 + const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true);
524 + assert.equal(unit.reloads, 0);
525 + const original = renderBootServiceUnit(unitPlan(root, "1.0.0"));
526 + const expected = renderBootServiceUnit(unitPlan(root, "1.1.0"));
527 + assert.equal(unit.loadedContents, original);
528 + yield* fs.writeFileString(unit.path, original);
529 + unit.failReload = true;
530 + yield* recover();
531 + assert.equal(yield* fs.readFileString(unit.path), expected);
532 + assert.equal(unit.reloads, 1);
533 + assert.equal(unit.loadedContents, original);
534 +
535 + unit.failReload = false;
536 + yield* recover();
537 + assert.equal(yield* fs.readFileString(unit.path), expected);
538 + assert.equal(unit.loadedContents, expected);
539 + assert.equal(unit.reloads, 2);
540 + }),
541 );
542
543 it.effect("rolls back invalid trials with the startup environment", () =>
@@ -357,9 +587,18 @@ if (context.update?.status === "pending") {
587 activeVersion: "1.0.0",
588 }),
589 );
590 + const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0");
591
592 + const launcher = new Launcher(
593 + root,
594 + yield* Effect.promise(() => readServiceState(statePath)),
595 + {
596 + serviceUnit: unit,
597 + environment: yield* Effect.promise(() => readServiceEnvironment(root)),
598 + },
599 + );
600 yield* Effect.promise(() =>
362 - main().then(
601 + launcher.run().then(
602 () => Promise.reject(new Error("launcher unexpectedly completed")),
603 () => Promise.resolve(),
604 ),
@@ -368,6 +607,11 @@ if (context.update?.status === "pending") {
607 const state = yield* Effect.promise(() => readServiceState(statePath));
608 assert.equal(state.activeVersion, "1.0.0");
609 assert.equal(state.update?.status, "rolled-back");
610 + assert.equal(
611 + yield* fs.readFileString(unit.path),
612 + renderBootServiceUnit(unitPlan(root, "1.0.0")),
613 + );
614 + assert.equal(unit.reloads, 1);
615 assert.equal(
616 state.update?.status === "rolled-back" ? state.update.reason : undefined,
617 "invalid-prepared",
apps/server/src/serviceLauncher.ts
+199 -17
@@ -7,6 +7,7 @@ import * as NodeFS from "node:fs";
7 import * as NodeFSP from "node:fs/promises";
8 import * as NodePath from "node:path";
9 import * as NodeUtil from "node:util";
10 +import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11
12 import type {
13 PendingServiceUpdate,
@@ -21,12 +22,14 @@ import {
22 decodeServiceLauncherChildMessage,
23 isExactServiceVersion,
24 parseServiceState,
25 + serviceUnitBaseDirOf,
26 SERVICE_ENV_FILE,
27 SERVICE_LAUNCHER_CONTEXT_ENV,
28 SERVICE_LAUNCHER_PROTOCOL,
29 SERVICE_STATE_FILE,
30 SERVICE_RESTART_PENDING_FILE,
31 SERVICE_STOP_MARKER_FILE,
32 + SERVICE_UNIT_ENV,
33 } from "./cloud/serviceProtocol.ts";
34
35 const HANDOFF_DELAY_MS = 2_000;
@@ -64,6 +67,78 @@ const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
67 args: ["serve"],
68 });
69
70 +/**
71 + * The unit or launch agent that `t3 service install` wrote. Its program names
72 + * the runtime that ran the install, and a remote update never replaces it, so
73 + * after a commit the launcher points it at the committed runtime. Otherwise the
74 + * next start lands on a runtime that nothing in the service state records.
75 + */
76 +export interface ServiceUnit {
77 + readonly path: string;
78 + /** systemd keeps a loaded copy of the unit; launchd re-reads the plist at bootstrap. */
79 + readonly reload: () => Promise<void>;
80 +}
81 +
82 +function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined {
83 + const unitName = env[SERVICE_UNIT_ENV]?.trim();
84 + const home = env.HOME?.trim();
85 + if (!unitName || !home) return undefined;
86 + if (unitName.endsWith(".plist")) {
87 + return {
88 + path: NodePath.join(home, "Library", "LaunchAgents", unitName),
89 + reload: () => Promise.resolve(),
90 + };
91 + }
92 + // The manager's own environment, which decides where it reads user units.
93 + const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
94 + const configHome =
95 + xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
96 + ? xdgConfigHome
97 + : NodePath.join(home, ".config");
98 + return {
99 + path: NodePath.join(configHome, "systemd", "user", unitName),
100 + reload: () =>
101 + new Promise((resolve, reject) => {
102 + NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
103 + stdio: "ignore",
104 + timeout: 30_000,
105 + })
106 + .once("error", reject)
107 + .once("exit", (code, signal) =>
108 + code === 0
109 + ? resolve()
110 + : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
111 + );
112 + }),
113 + };
114 +}
115 +
116 +// systemd's ExecStart value, or the plist string before the __service-launcher argument.
117 +// Only the version segment changes, so quoted or escaped paths stay as rendered.
118 +// The launch agent's PATH names a versions directory without the executable; leave it.
119 +const SERVICE_UNIT_PROGRAMS = [
120 + /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
121 + /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
122 +] as const;
123 +
124 +/**
125 + * The unit contents with their program pointed at `version`'s runtime. A unit
126 + * already at that version or a newer one is returned unchanged: `t3 update`
127 + * may have repointed it to a later version while this commit was in flight.
128 + */
129 +function repointServiceUnit(contents: string, version: string): string {
130 + for (const program of SERVICE_UNIT_PROGRAMS) {
131 + const match = program.exec(contents);
132 + if (match === null) continue;
133 + const [, prefix = "", current = "", suffix = ""] = match;
134 + if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
135 + return contents;
136 + }
137 + return contents.replace(program, () => `${prefix}${version}${suffix}`);
138 + }
139 + return contents;
140 +}
141 +
142 const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
143 const RESTORE_MARKER = ".restore-pending";
144
@@ -445,8 +520,12 @@ export async function readServiceState(filePath: string): Promise<ServiceState>
520 }
521
522 export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
523 + await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
524 + await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
525 +}
526 +
527 +async function replaceFileDurably(filePath: string, contents: string): Promise<void> {
528 const directory = NodePath.dirname(filePath);
449 - await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 });
529 const tempPath = NodePath.join(
530 directory,
531 `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
@@ -454,7 +533,7 @@ export async function writeServiceState(filePath: string, state: ServiceState):
533 let handle: NodeFSP.FileHandle | undefined;
534 try {
535 handle = await NodeFSP.open(tempPath, "wx", 0o600);
457 - await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8");
536 + await handle.writeFile(contents, "utf8");
537 await handle.sync();
538 await handle.close();
539 handle = undefined;
@@ -542,6 +621,7 @@ export class Launcher {
621 #done = false;
622 readonly #completion = Promise.withResolvers<void>();
623 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
624 + readonly #serviceUnit: ServiceUnit | undefined;
625 readonly #environment: Readonly<NodeJS.ProcessEnv>;
626
627 constructor(
@@ -549,11 +629,13 @@ export class Launcher {
629 state: ServiceState,
630 options: {
631 readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
632 + readonly serviceUnit?: ServiceUnit;
633 readonly environment?: Readonly<NodeJS.ProcessEnv>;
634 } = {},
635 ) {
636 this.#baseDir = baseDir;
637 this.#backupSpaceProbe = options.backupSpaceProbe;
638 + this.#serviceUnit = options.serviceUnit;
639 this.#environment = { ...(options.environment ?? process.env) };
640 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
641 this.#state = state;
@@ -620,20 +702,66 @@ export class Launcher {
702 this.#timer = undefined;
703 }
704
705 + async #selectState(next: ServiceState): Promise<void> {
706 + const release = await acquireServiceSelectionLock(this.#statePath);
707 + try {
708 + const selected = await readServiceState(this.#statePath);
709 + // Local installs replace the selection and clear its update record. An old
710 + // launcher must not restore its own selection when asked to update again.
711 + if (
712 + selected.activeVersion !== this.#state.activeVersion ||
713 + selected.update?.id !== this.#state.update?.id ||
714 + selected.update?.status !== this.#state.update?.status
715 + ) {
716 + throw new Error("Service selection changed. Restart the service before updating.");
717 + }
718 + await writeServiceState(this.#statePath, next);
719 + this.#state = next;
720 + } finally {
721 + release();
722 + }
723 + }
724 +
725 async #recover(): Promise<void> {
726 // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
727 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
728 const restartPending = restartPendingPath(this.#baseDir);
627 - const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined);
628 - if (awaitedVersion?.trim() === this.#state.activeVersion) {
629 - await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined);
729 + try {
730 + const release = await acquireServiceSelectionLock(this.#statePath);
731 + try {
732 + const selected = await readServiceState(this.#statePath);
733 + const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
734 + () => undefined,
735 + );
736 + const awaited = awaitedVersion?.trim();
737 + if (
738 + selected.activeVersion === this.#state.activeVersion &&
739 + selected.update?.id === this.#state.update?.id &&
740 + selected.update?.status === this.#state.update?.status &&
741 + awaited !== undefined &&
742 + isExactServiceVersion(awaited) &&
743 + compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
744 + ) {
745 + await NodeFSP.rm(restartPending, { force: true });
746 + }
747 + } finally {
748 + release();
749 + }
750 + } catch (error) {
751 + process.stderr.write(
752 + `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
753 + );
754 }
755 const update = this.#state.update;
756 if (update?.status !== "pending") {
757 if (update !== undefined) {
758 await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
759 }
636 - await this.#startChild(this.#state.activeVersion, "active", update);
760 + try {
761 + await this.#startChild(this.#state.activeVersion, "active", update);
762 + } finally {
763 + if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
764 + }
765 return;
766 }
767 if (await databaseRestorePending(this.#baseDir, update)) {
@@ -780,8 +908,12 @@ export class Launcher {
908 status: "pending",
909 };
910 const next: ServiceState = { ...this.#state, update: pending };
783 - await writeServiceState(this.#statePath, next);
784 - this.#state = next;
911 + try {
912 + await this.#selectState(next);
913 + } catch (error) {
914 + await reject(error instanceof Error ? error.message : String(error));
915 + return;
916 + }
917 await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
918 this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
919 }
@@ -827,11 +959,53 @@ export class Launcher {
959 activeVersion: pending.targetVersion,
960 update: committed,
961 };
830 - await writeServiceState(this.#statePath, next);
831 - this.#state = next;
962 + await this.#selectState(next);
963 child.role = "active";
964 await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
834 - await sendMessage(child.process, { type: "committed", updateId: committed.id });
965 + try {
966 + await sendMessage(child.process, { type: "committed", updateId: committed.id });
967 + } finally {
968 + await this.#repointServiceUnit(committed.targetVersion);
969 + }
970 + }
971 +
972 + // Selection is durable before this best-effort unit reconciliation.
973 + async #repointServiceUnit(version: string): Promise<void> {
974 + const unit = this.#serviceUnit;
975 + if (unit === undefined) return;
976 + try {
977 + const release = await acquireServiceSelectionLock(unit.path);
978 + try {
979 + // The installer writes state before the unit under this lease. Do not
980 + // acquire its state lease here: the installer takes that lease first.
981 + const selected = await readServiceState(this.#statePath);
982 + if (
983 + selected.activeVersion !== this.#state.activeVersion ||
984 + selected.update?.id !== this.#state.update?.id ||
985 + selected.update?.status !== this.#state.update?.status
986 + ) {
987 + return;
988 + }
989 + const contents = await NodeFSP.readFile(unit.path, "utf8");
990 + const installedBaseDir = serviceUnitBaseDirOf(contents);
991 + if (
992 + installedBaseDir === undefined ||
993 + NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
994 + ) {
995 + return;
996 + }
997 + const repointed = repointServiceUnit(contents, version);
998 + if (repointed !== contents) await replaceFileDurably(unit.path, repointed);
999 + // A previous rename may have succeeded before reload failed.
1000 + await unit.reload();
1001 + } finally {
1002 + release();
1003 + }
1004 + } catch (error) {
1005 + process.stderr.write(
1006 + `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1007 + );
1008 + }
1009 }
1010
1011 async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
@@ -887,9 +1061,9 @@ export class Launcher {
1061 activeVersion: pending.fromVersion,
1062 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1063 };
890 - await writeServiceState(this.#statePath, next);
891 - this.#state = next;
1064 + await this.#selectState(next);
1065 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1066 + await this.#repointServiceUnit(next.activeVersion);
1067 }
1068 // Signal listeners alone do not keep Node alive.
1069 this.#timer = setInterval(() => {}, 2_147_483_647);
@@ -915,10 +1089,13 @@ export class Launcher {
1089 activeVersion: pending.fromVersion,
1090 update: outcome,
1091 };
918 - await writeServiceState(this.#statePath, next);
919 - this.#state = next;
1092 + await this.#selectState(next);
1093 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
921 - await this.#startChild(next.activeVersion, "active", outcome);
1094 + try {
1095 + await this.#startChild(next.activeVersion, "active", outcome);
1096 + } finally {
1097 + await this.#repointServiceUnit(next.activeVersion);
1098 + }
1099 }
1100 }
1101
@@ -930,5 +1107,10 @@ export async function main(): Promise<void> {
1107 const environment = await readServiceEnvironment(baseDir);
1108 const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1109 const state = await readServiceState(statePath);
933 - await new Launcher(baseDir, state, { environment }).run();
1110 + const serviceUnit = serviceUnitFromEnvironment();
1111 + await new Launcher(
1112 + baseDir,
1113 + state,
1114 + serviceUnit === undefined ? { environment } : { environment, serviceUnit },
1115 + ).run();
1116 }
docs/internals/server-updates.md
+20 -4
@@ -4,14 +4,30 @@ A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime
4 selected by systemd or launchd. It is the only runtime writer of durable service
5 state. Server children request updates over inherited IPC; they never rewrite
6 their service definition or select their own replacement. Local service commands
7 -may replace the launcher and state while the service is stopped. Foreground CLI
8 -processes do not self-update.
7 +may replace the launcher and state, optionally deferring the restart. Foreground
8 +CLI processes do not self-update.
9 +
10 +Both writers serialize state and unit writes through
11 +[file leases](../../apps/server/src/cloud/serviceSelectionLock.ts). The installer
12 +takes the state lease before the unit lease; the launcher never holds both, so a
13 +unit failure cannot block a durable commit. Manager stop/start commands run outside
14 +both leases, since stopping a launcher that waits on a lease would deadlock. A
15 +running launcher refuses further updates once a local command changes its selection.
16
17 Exact-version installs keep restarts independent of npm cache eviction or a moving
18 release tag. Installation and preflight happen in staging before publishing an
19 immutable runtime. Preflight checks the launcher protocol because a target that
13 -needs new rollback guarantees cannot safely run under an older launcher. Upgrading
14 -that launcher requires a local service update.
20 +needs new rollback guarantees cannot safely run under an older launcher. A remote
21 +update replaces only the child, so upgrading the launcher requires a local service
22 +update with a restart.
23 +
24 +After a commit or rollback the launcher points the unit at the selected runtime and
25 +reloads the manager, so the next service start runs that version's launcher; the
26 +protocol check is what makes that safe. Each start after a finished update retries this, because
27 +a previous reload may have failed, and leaves a unit naming a newer runtime or another
28 +T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload)
29 +and launchd (before bootstrap) can respawn the old launcher, which the new runtime
30 +refuses to run under.
31
32 ## Commit boundary
33
docs/user/updating.md
+5
@@ -118,6 +118,11 @@ asks before restarting the background service; if you decline, run
118 stop it and start it again afterwards with your usual options such as `--host`
119 or `--tailscale-serve`.
120
121 +A background service installed by an older release refuses **Update server**
122 +until you update it once on the host and let that update restart it. If you
123 +decline the restart, the service can stay down after a crash or reboot until you
124 +run `t3 service restart`.
125 +
126 If you run the server with `npx` rather than an installed `t3`, there is
127 nothing to update on the host: stop the server and relaunch it as
128 `npx t3@<client-version>` with the same subcommand and options.