The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 23:30 UTC
fix(server): a remote update points the service unit at the runtime it committed (#19652)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Ibrahim Elkamali committed
Nov 11, 2026 at 12:48 UTC
d0d0608b6be8c50cdc38b5045de76c5d0b9407cb
9 files changed
+937
-128
apps/server/src/cloud/bootService.test.ts
+194
@@ -1,3 +1,5 @@
1
+// @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers.
2
+import * as NodeChildProcess from "node:child_process";
3
import * as NodeServices from "@effect/platform-node/NodeServices";
4
import { expect, it } from "@effect/vitest";
5
import * as HostProcess from "@t3tools/shared/HostProcess";
@@ -5,11 +7,13 @@ import * as ConfigProvider from "effect/ConfigProvider";
7
import * as Duration from "effect/Duration";
8
import * as Effect from "effect/Effect";
9
import * as FileSystem from "effect/FileSystem";
10
+import * as Fiber from "effect/Fiber";
11
import * as Layer from "effect/Layer";
12
import * as Path from "effect/Path";
13
import { CliError } from "effect/cli";
14
import { HttpClient } from "effect/http";
15
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
16
+import { vi } from "vite-plus/test";
17
18
import * as ProcessRunner from "../processRunner.ts";
19
import * as BootService from "./bootService.ts";
@@ -22,6 +26,24 @@ import {
26
serviceStateHasPendingUpdate,
27
} from "./serviceProtocol.ts";
28
29
+const leaseObserver = vi.hoisted(() => ({ contended: (_destination: string) => {} }));
30
+vi.mock("node:fs", async (importOriginal) => {
31
+ const actual = await importOriginal<typeof import("node:fs")>();
32
+ return {
33
+ ...actual,
34
+ renameSync: (...args: Parameters<typeof actual.renameSync>) => {
35
+ try {
36
+ return actual.renameSync(...args);
37
+ } catch (error) {
38
+ if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) {
39
+ leaseObserver.contended(String(args[1]));
40
+ }
41
+ throw error;
42
+ }
43
+ },
44
+ };
45
+});
46
+
47
it("reads the served T3 home back out of a rendered unit or plist", () => {
48
const plan = (baseDir: string) => ({
49
program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"],
@@ -442,6 +464,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
464
current: false,
465
installedVersion: "1.2.4-nightly.1",
466
});
467
+ yield* service.install({ allowDowngrade: true });
468
+ expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
469
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
470
+ activeVersion: "1.2.3",
471
+ });
472
}
473
}),
474
);
@@ -575,6 +602,168 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
602
expect(yield* newer.restart).toBe("restarted");
603
expect((yield* newer.status).problems).not.toContain("restart-pending");
604
expect((yield* newer.status).current).toBe(true);
605
+
606
+ const latest = yield* makeService(undefined, "1.2.5");
607
+ yield* latest.install({ start: true });
608
+ expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
609
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
610
+ activeVersion: "1.2.5",
611
+ });
612
+ expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.5/t3");
613
+ expect((yield* latest.status).current).toBe(true);
614
+ expect(
615
+ yield* fs.exists(path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE)),
616
+ ).toBe(false);
617
+ }),
618
+ );
619
+
620
+ it.effect(
621
+ "serializes a local install with another process reconciling an older unit snapshot",
622
+ () =>
623
+ Effect.gen(function* () {
624
+ const { service, fs, statePath, makeService, runtime } = yield* makeHarness();
625
+ const path = yield* Path.Path;
626
+ const plan = yield* service.install();
627
+ const unit = yield* fs.readFileString(plan.unitPath);
628
+ yield* fs.writeFileString(
629
+ plan.unitPath,
630
+ unit.replace("versions/1.2.3/t3", "versions/1.2.2/t3"),
631
+ );
632
+ yield* fs.writeFileString(
633
+ statePath,
634
+ JSON.stringify({
635
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
636
+ activeVersion: "1.2.3",
637
+ update: {
638
+ id: "committed",
639
+ fromVersion: "1.2.2",
640
+ targetVersion: "1.2.3",
641
+ status: "committed",
642
+ },
643
+ }),
644
+ );
645
+ const rejection = path.join(plan.baseDir, "rejected-update");
646
+ const databasePath = path.join(plan.baseDir, "state.sqlite");
647
+ yield* fs.writeFileString(databasePath, "database");
648
+ yield* fs.writeFileString(
649
+ runtime.entryPath,
650
+ `#!${process.execPath}
651
+import { writeFileSync } from "node:fs";
652
+process.on("message", message => {
653
+ if (message === "request-update") process.send({ type: "request-update", targetVersion: "1.2.5", dbPath: ${JSON.stringify(databasePath)} });
654
+ if (message.type === "update-rejected") {
655
+ writeFileSync(${JSON.stringify(rejection)}, message.reason);
656
+ process.exit(0);
657
+ }
658
+});
659
+`,
660
+ );
661
+ yield* fs.chmod(runtime.entryPath, 0o755);
662
+ const newer = yield* makeService(undefined, "1.2.4");
663
+ yield* makeService(undefined, "1.2.5");
664
+ const snapshot = Promise.withResolvers<void>();
665
+ const milestone = Promise.withResolvers<void>();
666
+ const script = path.join(plan.baseDir, "launcher-contention.mjs");
667
+ yield* fs.writeFileString(
668
+ script,
669
+ `
670
+import fs from "node:fs/promises";
671
+import cp from "node:child_process";
672
+import { syncBuiltinESMExports } from "node:module";
673
+const server = Promise.withResolvers();
674
+const originalSpawn = cp.spawn;
675
+cp.spawn = (...args) => { const child = originalSpawn(...args); server.resolve(child); return child; };
676
+process.on("message", async message => {
677
+ if (message === "installed") (await server.promise).send("request-update");
678
+});
679
+const originalRead = fs.readFile;
680
+fs.readFile = async (...args) => {
681
+ const contents = await originalRead(...args);
682
+ if (args[0] === ${JSON.stringify(plan.unitPath)}) {
683
+ fs.readFile = originalRead;
684
+ syncBuiltinESMExports();
685
+ const resume = new Promise(resolve => process.once("message", resolve));
686
+ process.send("snapshot");
687
+ await resume;
688
+ }
689
+ return contents;
690
+};
691
+syncBuiltinESMExports();
692
+const { Launcher, readServiceState } = await import(${JSON.stringify(import.meta.resolve("../serviceLauncher.ts"))});
693
+const launcher = new Launcher(${JSON.stringify(plan.baseDir)}, await readServiceState(${JSON.stringify(statePath)}), {
694
+ serviceUnit: { path: ${JSON.stringify(plan.unitPath)}, reload: async () => {} },
695
+});
696
+try { await launcher.run(); throw Error("unexpected completion"); }
697
+catch (error) {
698
+ if (!error.message.includes("Active child exited unexpectedly (0)")) throw error;
699
+}
700
+const { acquireServiceSelectionLock } = await import(${JSON.stringify(import.meta.resolve("./serviceSelectionLock.ts"))});
701
+// Exit owning the real lease, as a crashed process would.
702
+await acquireServiceSelectionLock(${JSON.stringify(plan.unitPath)});
703
+process.disconnect();
704
+`,
705
+ );
706
+ const child = yield* Effect.acquireRelease(
707
+ Effect.sync(() =>
708
+ NodeChildProcess.fork(script, [], {
709
+ execArgv: [],
710
+ stdio: ["ignore", "inherit", "inherit", "ipc"],
711
+ }),
712
+ ),
713
+ (child) =>
714
+ Effect.sync(() => {
715
+ if (child.exitCode === null && child.signalCode === null) child.kill();
716
+ }),
717
+ );
718
+ const exited = new Promise<void>((resolve, reject) => {
719
+ child.once("error", reject);
720
+ child.once("exit", (code) =>
721
+ code === 0 ? resolve() : reject(new Error(`launcher exited ${code}`)),
722
+ );
723
+ });
724
+ child.on("message", (message) => {
725
+ if (message === "snapshot") snapshot.resolve();
726
+ });
727
+ yield* Effect.promise(() =>
728
+ Promise.race([
729
+ snapshot.promise,
730
+ exited.then(() => {
731
+ throw new Error("launcher exited before snapshot");
732
+ }),
733
+ ]),
734
+ );
735
+ leaseObserver.contended = (destination) => {
736
+ if (destination === plan.unitPath + ".selection-lock") milestone.resolve();
737
+ };
738
+ yield* Effect.addFinalizer(() =>
739
+ Effect.sync(() => {
740
+ leaseObserver.contended = () => {};
741
+ }),
742
+ );
743
+ const installing = yield* newer
744
+ .install({ start: false })
745
+ .pipe(Effect.ensuring(Effect.sync(() => milestone.resolve())), Effect.forkChild);
746
+ // Resume the launcher once the installer contends for the unit lease, or once an
747
+ // uncoordinated installer has finished, which makes the stale snapshot win.
748
+ yield* Effect.promise(() => milestone.promise);
749
+ child.send("resume");
750
+ yield* Fiber.join(installing);
751
+ child.send("installed");
752
+ yield* Effect.promise(() => exited);
753
+ expect(yield* fs.readFileString(rejection)).toContain("Service selection changed");
754
+ expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
755
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
756
+ activeVersion: "1.2.4",
757
+ });
758
+ expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3");
759
+ expect(
760
+ yield* fs.readFileString(
761
+ path.join(path.dirname(statePath), SERVICE_RESTART_PENDING_FILE),
762
+ ),
763
+ ).toBe("1.2.4\n");
764
+ // The next installer reclaims the dead owner and releases its own lease.
765
+ yield* newer.install({ start: false });
766
+ expect(yield* fs.exists(plan.unitPath + ".selection-lock")).toBe(false);
767
}),
768
);
769
@@ -617,6 +806,11 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
806
const error = yield* service.install({ start: false }).pipe(Effect.flip);
807
expect(error._tag).toBe("BootServiceUpdatePendingError");
808
expect(yield* fs.readFileString(statePath)).toBe(pendingState);
809
+ yield* fs.remove((yield* service.status).unitPath);
810
+ expect((yield* service.install({ start: false }).pipe(Effect.flip))._tag).toBe(
811
+ "BootServiceUpdatePendingError",
812
+ );
813
+ expect(yield* fs.readFileString(statePath)).toBe(pendingState);
814
}),
815
);
816
apps/server/src/cloud/bootService.ts
+102
-84
@@ -16,6 +16,7 @@ import * as Schema from "effect/Schema";
16
import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease";
17
18
import * as ProcessRunner from "../processRunner.ts";
19
+import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts";
20
import {
21
ensurePinnedRuntimeInstalled,
22
launcherOwnsVersionsDir,
@@ -28,20 +29,26 @@ import {
29
SERVICE_LAUNCHER_PROTOCOL,
30
SERVICE_RESTART_PENDING_FILE,
31
SERVICE_STATE_FILE,
32
+ SERVICE_UNIT_ENV,
33
compareExactServiceVersions,
34
+ serviceUnitBaseDirOf as bootServiceBaseDirOf,
35
parseServiceState,
36
serviceStateActiveVersion,
37
serviceStateHasPendingUpdate,
38
type ServiceState,
39
} from "./serviceProtocol.ts";
40
41
+export {
42
+ serviceUnitBaseDirOf as bootServiceBaseDirOf,
43
+ serviceUnitProgramOf as bootServiceProgramOf,
44
+} from "./serviceProtocol.ts";
45
+
46
const BOOT_SERVICE_NAME = "t3code";
47
export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`;
48
// `.service` suffix keeps the label distinct from the desktop app's bundle id
49
// (com.t3tools.t3code), so launchd and TCC records never collide.
50
export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service";
51
export const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`;
44
-const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT";
52
export const BOOT_SERVICE_LOG_FILE = "boot-service.log";
53
54
/** systemd expands `%` specifiers, including in unquoted append-log paths. */
@@ -56,32 +63,6 @@ function quoteSystemdValue(value: string): string {
63
: escaped;
64
}
65
59
-const unquoteSystemdValue = (raw: string) =>
60
- (raw.startsWith('"') && raw.endsWith('"')
61
- ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\")
62
- : raw
63
- ).replaceAll("%%", "%");
64
-
65
-const unescapeXmlText = (value: string) =>
66
- value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&");
67
-
68
-export function bootServiceBaseDirOf(contents: string): string | undefined {
69
- const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1];
70
- if (systemd !== undefined) return unquoteSystemdValue(systemd.trim());
71
- const plist = /<key>T3CODE_HOME<\/key>\s*<string>([^<]*)<\/string>/.exec(contents)?.[1];
72
- return plist === undefined ? undefined : unescapeXmlText(plist);
73
-}
74
-
75
-/** The executable an installed unit or plist starts: the launcher's runtime. */
76
-export function bootServiceProgramOf(contents: string): string | undefined {
77
- const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1];
78
- if (systemd !== undefined) return unquoteSystemdValue(systemd);
79
- const plist = /<key>ProgramArguments<\/key>\s*<array>\s*<string>([^<]*)<\/string>/.exec(
80
- contents,
81
- )?.[1];
82
- return plist === undefined ? undefined : unescapeXmlText(plist);
83
-}
84
-
66
export interface BootServicePlan {
67
readonly program: ReadonlyArray<string>;
68
readonly baseDir: string;
@@ -104,7 +85,7 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
85
"Type=simple",
86
"WorkingDirectory=%h",
87
`Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`,
107
- `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`,
88
+ `Environment=${SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`,
89
`ExecStart=${plan.program.map(quoteSystemdValue).join(" ")}`,
90
// Let the launcher mark an explicit stop before it signals the server.
91
// systemd still SIGKILLs the whole cgroup if graceful shutdown times out.
@@ -162,7 +143,7 @@ export function renderBootServicePlist(
143
` <string>${escapeXmlText(options.environmentPath)}</string>`,
144
` <key>T3CODE_HOME</key>`,
145
` <string>${escapeXmlText(plan.baseDir)}</string>`,
165
- ` <key>${BOOT_SERVICE_UNIT_ENV}</key>`,
146
+ ` <key>${SERVICE_UNIT_ENV}</key>`,
147
` <string>${BOOT_SERVICE_PLIST_FILE}</string>`,
148
` </dict>`,
149
` <key>WorkingDirectory</key>`,
@@ -496,9 +477,12 @@ export class BootServicePrerequisiteError extends Schema.TaggedError<BootService
477
478
export class BootServiceUpdatePendingError extends Schema.TaggedError<BootServiceUpdatePendingError>()(
479
"BootServiceUpdatePendingError",
499
- {},
480
+ { recoveryCommand: Schema.optional(Schema.String) },
481
) {
482
override get message(): string {
483
+ if (this.recoveryCommand !== undefined) {
484
+ return `A remote server update is still pending, but its service definition is missing. With the service stopped, run this on the server machine: ${this.recoveryCommand}\nWait for the update to finish or roll back, stop the foreground launcher with Ctrl-C, then retry the install.`;
485
+ }
486
return "A remote server update is still pending. Wait for it to finish, then retry.";
487
}
488
}
@@ -606,7 +590,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
590
const xdgConfigHome = yield* Config.String("XDG_CONFIG_HOME").pipe(Config.withDefault(""));
591
const installerPath = yield* Config.String("PATH").pipe(Config.withDefault(""));
592
const runningInsideService =
609
- (yield* Config.String(BOOT_SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) ===
593
+ (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) ===
594
BOOT_SERVICE_UNIT_FILE;
595
const fs = yield* FileSystem.FileSystem;
596
const path = yield* Path.Path;
@@ -799,6 +783,16 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
783
if (remaining[0]) return yield* new BootServicePrerequisiteError({ problem: remaining[0] });
784
});
785
786
+ const withSelectionLock = <A, E, R>(selectionPath: string, operation: Effect.Effect<A, E, R>) =>
787
+ Effect.acquireUseRelease(
788
+ Effect.tryPromise({
789
+ try: () => acquireServiceSelectionLock(selectionPath),
790
+ catch: (cause) => new BootServiceInstallError({ cause }),
791
+ }),
792
+ () => operation,
793
+ (release) => Effect.sync(release),
794
+ );
795
+
796
const install = Effect.fn("cloud.boot_service.install")(function* (options?: {
797
readonly allowDowngrade?: boolean;
798
readonly start?: boolean;
@@ -876,8 +870,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
870
const installed = yield* fs
871
.exists(unitPath)
872
.pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
879
- // With start=false, changing startup-only files is safe except during a remote update,
880
- // when the launcher also writes its state file.
873
+ // Stop outside the selection lock: the launcher may be waiting to acquire it.
874
const start = options?.start !== false;
875
if (installed && start && runningInsideService && manager.kind === "systemd") {
876
// Stopping the unit from inside it kills this process before the start.
@@ -890,7 +883,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
883
// Transient properties reach systemd literally, unlike unit-file specifiers.
884
`--property=StandardOutput=append:${logPath}`,
885
`--property=StandardError=append:${logPath}`,
893
- `--setenv=${BOOT_SERVICE_UNIT_ENV}=`,
886
+ `--setenv=${SERVICE_UNIT_ENV}=`,
887
runtimePaths.entryPath,
888
"service",
889
"install",
@@ -905,58 +898,83 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
898
yield* runSteps(manager.stop);
899
}
900
908
- yield* Effect.gen(function* () {
909
- if (installed) {
910
- const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);
911
- if (Option.isSome(previousStateText)) {
912
- if (serviceStateHasPendingUpdate(previousStateText.value)) {
913
- return yield* new BootServiceUpdatePendingError();
901
+ // State first, then unit. The launcher acquires only one at a time, and a
902
+ // failure to lock the unit must never prevent its durable update commit.
903
+ yield* withSelectionLock(
904
+ statePath,
905
+ withSelectionLock(
906
+ unitPath,
907
+ Effect.gen(function* () {
908
+ const currentlyInstalled = yield* fs.exists(unitPath);
909
+ const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option);
910
+ if (Option.isSome(previousStateText)) {
911
+ const installedVersion = serviceStateActiveVersion(previousStateText.value);
912
+ if (serviceStateHasPendingUpdate(previousStateText.value)) {
913
+ const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`;
914
+ return yield* new BootServiceUpdatePendingError(
915
+ !currentlyInstalled && installedVersion !== undefined
916
+ ? {
917
+ recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`,
918
+ }
919
+ : {},
920
+ );
921
+ }
922
+ // Read the final version after stopping the launcher; a remote update may finish after the status check.
923
+ if (
924
+ installedVersion !== undefined &&
925
+ options?.allowDowngrade !== true &&
926
+ compareExactServiceVersions(input.cliVersion, installedVersion) < 0
927
+ ) {
928
+ return yield* new BootServiceDowngradeRefusedError({
929
+ installedVersion,
930
+ targetVersion: input.cliVersion,
931
+ });
932
+ }
933
}
915
- // Read the final version after stopping the launcher; a remote update may finish after the status check.
916
- const installedVersion = serviceStateActiveVersion(previousStateText.value);
917
- if (
918
- installedVersion !== undefined &&
919
- options?.allowDowngrade !== true &&
920
- compareExactServiceVersions(input.cliVersion, installedVersion) < 0
921
- ) {
922
- return yield* new BootServiceDowngradeRefusedError({
923
- installedVersion,
924
- targetVersion: input.cliVersion,
925
- });
934
+ yield* fs
935
+ .makeDirectory(path.dirname(unitPath), { recursive: true })
936
+ .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
937
+ // Files can match the new version even when activation fails; keep that install retryable.
938
+ yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
939
+ yield* writeDurably(
940
+ statePath,
941
+ // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document.
942
+ `${JSON.stringify(
943
+ {
944
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
945
+ activeVersion: input.cliVersion,
946
+ } satisfies ServiceState,
947
+ null,
948
+ 2,
949
+ )}\n`,
950
+ );
951
+ if (!start && (currentlyInstalled || Option.isSome(previousStateText))) {
952
+ const written = yield* fs.readFileString(statePath);
953
+ if (serviceStateActiveVersion(written) !== input.cliVersion) {
954
+ return yield* new BootServiceUpdatePendingError();
955
+ }
956
}
927
- }
928
- }
929
- yield* fs
930
- .makeDirectory(path.dirname(unitPath), { recursive: true })
931
- .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })));
932
- // Files can match the new version even when activation fails; keep that install retryable.
933
- yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 });
934
- yield* writeDurably(
935
- statePath,
936
- // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document.
937
- `${JSON.stringify(
938
- {
939
- protocol: SERVICE_LAUNCHER_PROTOCOL,
940
- activeVersion: input.cliVersion,
941
- } satisfies ServiceState,
942
- null,
943
- 2,
944
- )}\n`,
945
- );
946
- if (!start && installed) {
947
- // A concurrent remote update can overwrite launcher state; detect it before repointing the unit.
948
- const written = yield* fs.readFileString(statePath);
949
- if (serviceStateActiveVersion(written) !== input.cliVersion) {
950
- return yield* new BootServiceUpdatePendingError();
951
- }
952
- }
953
- yield* writeDurably(unitPath, manager.render(plan));
954
-
955
- if (start) {
956
- yield* runSteps(manager.activate);
957
- yield* fs.remove(restartPendingPath, { force: true });
958
- }
959
- }).pipe(
957
+ yield* writeDurably(unitPath, manager.render(plan));
958
+ }),
959
+ ),
960
+ ).pipe(
961
+ Effect.andThen(
962
+ Effect.gen(function* () {
963
+ if (start) {
964
+ yield* runSteps(manager.activate);
965
+ yield* withSelectionLock(
966
+ statePath,
967
+ Effect.gen(function* () {
968
+ // A concurrent deferred install may now be waiting for a newer restart.
969
+ const selected = parseServiceState(yield* fs.readFileString(statePath));
970
+ if (selected?.activeVersion === input.cliVersion && selected.update === undefined) {
971
+ yield* fs.remove(restartPendingPath, { force: true });
972
+ }
973
+ }),
974
+ );
975
+ }
976
+ }),
977
+ ),
978
Effect.mapError((cause) =>
979
cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause,
980
),
apps/server/src/cloud/serviceProtocol.ts
+35
-3
@@ -1,15 +1,46 @@
1
import type { ServerSelfUpdateOutcome } from "@t3tools/contracts";
2
3
-// Bump for incompatible runtime layout changes; launchers survive self-updates. Protocol 3 requires standalone executables.
4
-export const SERVICE_LAUNCHER_PROTOCOL = 3 as const;
3
+// Protocol 4 requires coordinated selection writes and unit reconciliation. An
4
+// installed protocol-3 launcher needs one local update or service install with a restart:
5
+// remote updates replace its child, so cannot add these launcher guarantees.
6
+export const SERVICE_LAUNCHER_PROTOCOL = 4 as const;
7
export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
8
export const SERVICE_ENV_FILE = "service.env";
9
export const SERVICE_STATE_FILE = "service-state.json";
10
+/** Set by the unit or launch agent to its own file name, so the launcher can
11
+ find the definition that started it. */
12
+export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT";
13
/** Written before explicit stop to distinguish shutdown from update handoff. */
14
export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
15
/** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */
16
export const SERVICE_RESTART_PENDING_FILE = ".restart-pending";
17
18
+const unquoteSystemdValue = (raw: string) =>
19
+ (raw.startsWith('"') && raw.endsWith('"')
20
+ ? raw.slice(1, -1).replaceAll('\\"', '"').replaceAll("\\\\", "\\")
21
+ : raw
22
+ ).replaceAll("%%", "%");
23
+
24
+const unescapeXmlText = (value: string) =>
25
+ value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&");
26
+
27
+export function serviceUnitBaseDirOf(contents: string): string | undefined {
28
+ const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1];
29
+ if (systemd !== undefined) return unquoteSystemdValue(systemd.trim());
30
+ const plist = /<key>T3CODE_HOME<\/key>\s*<string>([^<]*)<\/string>/.exec(contents)?.[1];
31
+ return plist === undefined ? undefined : unescapeXmlText(plist);
32
+}
33
+
34
+/** The executable an installed unit or plist starts: the launcher's runtime. */
35
+export function serviceUnitProgramOf(contents: string): string | undefined {
36
+ const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1];
37
+ if (systemd !== undefined) return unquoteSystemdValue(systemd);
38
+ const plist = /<key>ProgramArguments<\/key>\s*<array>\s*<string>([^<]*)<\/string>/.exec(
39
+ contents,
40
+ )?.[1];
41
+ return plist === undefined ? undefined : unescapeXmlText(plist);
42
+}
43
+
44
export interface PendingServiceUpdate {
45
readonly id: string;
46
readonly fromVersion: string;
@@ -143,7 +174,8 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
174
if (!isRecord(value)) return undefined;
175
const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update);
176
if (
146
- value.protocol !== SERVICE_LAUNCHER_PROTOCOL ||
177
+ // Protocol 3 has the same durable format; its installer can select a protocol-4 runtime.
178
+ (value.protocol !== 3 && value.protocol !== SERVICE_LAUNCHER_PROTOCOL) ||
179
typeof value.activeVersion !== "string" ||
180
!isExactServiceVersion(value.activeVersion) ||
181
(value.update !== undefined && update === undefined) ||
apps/server/src/cloud/serviceSelectionLock.test.ts
new
+30
@@ -0,0 +1,30 @@
1
+import * as NodeServices from "@effect/platform-node/NodeServices";
2
+import { assert, it } from "@effect/vitest";
3
+import * as Effect from "effect/Effect";
4
+import * as FileSystem from "effect/FileSystem";
5
+import * as Path from "effect/Path";
6
+
7
+import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts";
8
+
9
+it.layer(NodeServices.layer)("service selection lock", (it) => {
10
+ it.effect("reclaims a lease from an earlier boot even when its PID is live again", () =>
11
+ Effect.gen(function* () {
12
+ const fs = yield* FileSystem.FileSystem;
13
+ const path = yield* Path.Path;
14
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-selection-lock-" });
15
+ const selectionPath = path.join(root, "service-state.json");
16
+ const lock = `${selectionPath}.selection-lock`;
17
+ const stale = `${process.pid}.1.00000000-0000-4000-8000-000000000000`;
18
+ yield* fs.makeDirectory(lock, { recursive: true });
19
+ yield* fs.writeFileString(path.join(lock, stale), "");
20
+
21
+ const release = yield* Effect.promise(() => acquireServiceSelectionLock(selectionPath));
22
+ const owners = yield* fs.readDirectory(lock);
23
+ assert.equal(owners.length, 1);
24
+ assert.notEqual(owners[0], stale);
25
+ assert.isTrue(owners[0]!.startsWith(`${process.pid}.`));
26
+ release();
27
+ assert.isFalse(yield* fs.exists(lock));
28
+ }),
29
+ );
30
+});
apps/server/src/cloud/serviceSelectionLock.ts
new
+88
@@ -0,0 +1,88 @@
1
+// @effect-diagnostics nodeBuiltinImport:off -- Shared with the standalone launcher, which has no Effect runtime.
2
+// @effect-diagnostics globalTimers:off -- Bounded cross-process lease acquisition in the standalone launcher.
3
+// @effect-diagnostics globalDate:off -- The standalone launcher has no Effect clock for its acquisition deadline.
4
+import * as NodeCrypto from "node:crypto";
5
+import * as NodeFS from "node:fs";
6
+import * as NodeOS from "node:os";
7
+import * as NodePath from "node:path";
8
+
9
+// Coarse boot time: a lease from an earlier boot is dead even when its PID is live again.
10
+const bootEpochSeconds = () => Math.round(Date.now() / 1000 - NodeOS.uptime());
11
+
12
+/** Serialize durable selection and unit writes, never manager stop/start commands. */
13
+export async function acquireServiceSelectionLock(selectionPath: string): Promise<() => void> {
14
+ const directory = NodePath.dirname(selectionPath);
15
+ NodeFS.mkdirSync(directory, { recursive: true, mode: 0o700 });
16
+ const lock = selectionPath + ".selection-lock";
17
+ const owner = `${process.pid}.${bootEpochSeconds()}.${NodeCrypto.randomUUID()}`;
18
+ const candidate = `${lock}.${owner}`;
19
+ const deadline = Date.now() + 30_000;
20
+ const removeEmptyLock = () => {
21
+ try {
22
+ NodeFS.rmdirSync(lock);
23
+ } catch (error) {
24
+ if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes((error as NodeJS.ErrnoException).code ?? ""))
25
+ throw error;
26
+ }
27
+ };
28
+ NodeFS.mkdirSync(candidate, { mode: 0o700 });
29
+ try {
30
+ NodeFS.writeFileSync(NodePath.join(candidate, owner), "", { mode: 0o600 });
31
+ while (true) {
32
+ try {
33
+ // Publish the owner atomically; rename cannot replace another populated directory.
34
+ NodeFS.renameSync(candidate, lock);
35
+ return () => {
36
+ NodeFS.unlinkSync(NodePath.join(lock, owner));
37
+ removeEmptyLock();
38
+ };
39
+ } catch (error) {
40
+ if (!["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? ""))
41
+ throw error;
42
+ }
43
+ let owners: string[];
44
+ try {
45
+ owners = NodeFS.readdirSync(lock);
46
+ } catch (error) {
47
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
48
+ throw error;
49
+ }
50
+ if (owners.length === 1) {
51
+ const previous = owners[0]!;
52
+ const [pidText, bootText] = previous.split(".");
53
+ const pid = Number(pidText);
54
+ if (!Number.isSafeInteger(pid) || pid <= 0)
55
+ throw new Error(`Invalid service selection lock owner at ${lock}.`);
56
+ const ownerBoot = Number(bootText);
57
+ const fromEarlierBoot =
58
+ Number.isSafeInteger(ownerBoot) && Math.abs(ownerBoot - bootEpochSeconds()) > 60;
59
+ let dead = fromEarlierBoot;
60
+ if (!dead) {
61
+ try {
62
+ process.kill(pid, 0);
63
+ } catch (error) {
64
+ const code = (error as NodeJS.ErrnoException).code;
65
+ if (code !== "ESRCH" && code !== "EPERM") throw error;
66
+ dead = code === "ESRCH";
67
+ }
68
+ }
69
+ if (dead) {
70
+ // Remove only this dead owner's unique file, never a replacement owner's.
71
+ try {
72
+ NodeFS.unlinkSync(NodePath.join(lock, previous));
73
+ } catch (error) {
74
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
75
+ }
76
+ }
77
+ }
78
+ removeEmptyLock();
79
+ if (Date.now() >= deadline)
80
+ throw new Error(
81
+ `Service selection is locked at ${lock}. Retry when the other operation finishes.`,
82
+ );
83
+ await new Promise<void>((resolve) => setTimeout(resolve, 50));
84
+ }
85
+ } finally {
86
+ NodeFS.rmSync(candidate, { recursive: true, force: true });
87
+ }
88
+}
apps/server/src/serviceLauncher.test.ts
+264
-20
@@ -2,6 +2,8 @@
2
import type * as NodeChildProcess from "node:child_process";
3
// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4
import * as NodeFS from "node:fs";
5
+// @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback.
6
+import * as NodeFSP from "node:fs/promises";
7
import * as NodeSqlite from "node:sqlite";
8
9
import * as NodeServices from "@effect/platform-node/NodeServices";
@@ -13,10 +15,10 @@ import * as FileSystem from "effect/FileSystem";
15
import * as Path from "effect/Path";
16
import { vi } from "vite-plus/test";
17
18
+import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts";
19
import {
20
databaseBackupBlocker,
21
Launcher,
19
- main,
22
readServiceEnvironment,
23
type BackupSpaceProbe,
24
readServiceState,
@@ -112,6 +114,13 @@ it("rejects contradictory service state", () => {
114
);
115
});
116
117
+const unitPlan = (root: string, version: string) => ({
118
+ program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"],
119
+ baseDir: root,
120
+ logPath: `${root}/logs/boot-service.log`,
121
+ unitPath: `${root}/unit/t3code.service`,
122
+});
123
+
124
const writeFakeRuntime = (
125
fs: FileSystem.FileSystem,
126
path: Path.Path,
@@ -130,6 +139,32 @@ const writeFakeRuntime = (
139
return entryPath;
140
});
141
142
+const fakeServiceUnit = (
143
+ fs: FileSystem.FileSystem,
144
+ path: Path.Path,
145
+ root: string,
146
+ version: string,
147
+ render = renderBootServiceUnit,
148
+) =>
149
+ Effect.gen(function* () {
150
+ const plan = unitPlan(root, version);
151
+ yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true });
152
+ const contents = render(plan);
153
+ yield* fs.writeFileString(plan.unitPath, contents);
154
+ const unit = {
155
+ path: plan.unitPath,
156
+ reloads: 0,
157
+ loadedContents: contents,
158
+ failReload: false,
159
+ reload: async () => {
160
+ unit.reloads += 1;
161
+ if (unit.failReload) throw new Error("daemon-reload failed");
162
+ unit.loadedContents = await NodeFSP.readFile(unit.path, "utf8");
163
+ },
164
+ };
165
+ return unit;
166
+ });
167
+
168
it.layer(NodeServices.layer)("service state persistence", (it) => {
169
it.effect(
170
"loads optional service.env literally without changing inherited or managed variables",
@@ -217,6 +252,105 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
252
}),
253
);
254
255
+ it.effect.each([
256
+ { unitVersion: "1.0.0", replacement: "an intentional downgrade" },
257
+ { unitVersion: "1.2.0", replacement: "another T3 home" },
258
+ ])(
259
+ "recovers a rolled-back update with unit $unitVersion and preserves $replacement",
260
+ ({ unitVersion, replacement }) =>
261
+ Effect.gen(function* () {
262
+ const fs = yield* FileSystem.FileSystem;
263
+ const path = yield* Path.Path;
264
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" });
265
+ const statePath = path.join(root, "runtime", "service-state.json");
266
+ yield* writeFakeRuntime(
267
+ fs,
268
+ path,
269
+ path.join(root, "runtime", "versions", "1.1.0"),
270
+ "setInterval(() => {}, 1_000);\n",
271
+ );
272
+ yield* Effect.promise(() =>
273
+ writeServiceState(statePath, {
274
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
275
+ activeVersion: "1.1.0",
276
+ update: {
277
+ id: "update-1",
278
+ fromVersion: "1.1.0",
279
+ targetVersion: "1.2.0",
280
+ status: "rolled-back",
281
+ },
282
+ }),
283
+ );
284
+ const unit = yield* fakeServiceUnit(fs, path, root, unitVersion);
285
+ const expected = renderBootServiceUnit(
286
+ unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion),
287
+ );
288
+ const createLauncher = Effect.gen(function* () {
289
+ return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), {
290
+ serviceUnit: unit,
291
+ });
292
+ });
293
+ const run = (launcher: Launcher) =>
294
+ Effect.gen(function* () {
295
+ const running = launcher.run();
296
+ yield* Effect.promise(() => launcher.stop("SIGTERM"));
297
+ yield* Effect.promise(() => running);
298
+ });
299
+
300
+ yield* run(yield* createLauncher);
301
+ assert.equal(yield* fs.readFileString(unit.path), expected);
302
+ assert.equal(unit.loadedContents, expected);
303
+
304
+ yield* run(yield* createLauncher);
305
+ assert.equal(yield* fs.readFileString(unit.path), expected);
306
+ assert.equal(unit.loadedContents, expected);
307
+
308
+ yield* Effect.promise(() =>
309
+ writeServiceState(statePath, {
310
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
311
+ activeVersion: "1.1.0",
312
+ update: {
313
+ id: "update-2",
314
+ fromVersion: "1.0.0",
315
+ targetVersion: "1.1.0",
316
+ status: "committed",
317
+ },
318
+ }),
319
+ );
320
+ const staleLauncher = yield* createLauncher;
321
+ // A local install can replace the selection or service owner without stopping this launcher.
322
+ const selectedRoot =
323
+ replacement === "another T3 home" ? path.join(root, "other-home") : root;
324
+ yield* writeFakeRuntime(
325
+ fs,
326
+ path,
327
+ path.join(selectedRoot, "runtime", "versions", "1.0.0"),
328
+ "setInterval(() => {}, 1_000);\n",
329
+ );
330
+ const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json");
331
+ yield* Effect.promise(() =>
332
+ writeServiceState(selectedStatePath, {
333
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
334
+ activeVersion: "1.0.0",
335
+ }),
336
+ );
337
+ const selectedUnit = renderBootServiceUnit({
338
+ ...unitPlan(selectedRoot, "1.0.0"),
339
+ unitPath: unit.path,
340
+ });
341
+ yield* fs.writeFileString(unit.path, selectedUnit);
342
+ yield* Effect.promise(() => unit.reload());
343
+ const oldHomeState = yield* fs.readFileString(statePath);
344
+ const selectedState = yield* fs.readFileString(selectedStatePath);
345
+
346
+ yield* run(staleLauncher);
347
+ assert.equal(yield* fs.readFileString(statePath), oldHomeState);
348
+ assert.equal(yield* fs.readFileString(selectedStatePath), selectedState);
349
+ assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement);
350
+ assert.equal(unit.loadedContents, selectedUnit, replacement);
351
+ }),
352
+ );
353
+
354
it.effect("stays idle instead of exiting when another server owns the T3 home", () =>
355
Effect.gen(function* () {
356
const fs = yield* FileSystem.FileSystem;
@@ -257,25 +391,49 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
391
}),
392
);
393
260
- it.effect("commits only after the trial reports prepared", () =>
394
+ const commitUpdate = (
395
+ render = renderBootServiceUnit,
396
+ missingUnit = false,
397
+ legacyInstall = false,
398
+ ) =>
399
Effect.gen(function* () {
400
const fs = yield* FileSystem.FileSystem;
401
const path = yield* Path.Path;
264
- const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
402
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service 100% & <test>-" });
403
const statePath = path.join(root, "runtime", "service-state.json");
404
const databasePath = path.join(root, "userdata", "state.sqlite");
405
+ const activatedPath = path.join(root, "activated");
406
yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
407
yield* fs.writeFileString(databasePath, "before trial");
269
- const encodedDatabasePath = JSON.stringify(databasePath);
408
+ const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render);
409
+ if (missingUnit) yield* fs.remove(unit.path);
410
const childSource = `
411
+import assert from "node:assert/strict";
412
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
413
const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
414
+assert.equal(context.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
415
if (context.update?.status === "pending") {
273
- process.send({ type: "prepared", updateId: context.update.id });
416
+ const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
417
+ assert.equal(state.activeVersion, "1.0.0");
418
+ assert.equal(state.update.status, "pending");
419
+ ${
420
+ missingUnit
421
+ ? `assert.equal(existsSync(${JSON.stringify(unit.path)}), false);`
422
+ : `assert.equal(readFileSync(${JSON.stringify(unit.path)}, "utf8"), ${JSON.stringify(unit.loadedContents)});`
423
+ }
424
process.on("message", (message) => {
275
- if (message.type === "committed") process.exit(0);
425
+ if (message.type === "committed") {
426
+ const state = JSON.parse(readFileSync(${JSON.stringify(statePath)}, "utf8"));
427
+ assert.equal(state.protocol, ${SERVICE_LAUNCHER_PROTOCOL});
428
+ assert.equal(state.activeVersion, "1.1.0");
429
+ assert.equal(state.update.status, "committed");
430
+ writeFileSync(${JSON.stringify(activatedPath)}, message.updateId);
431
+ process.exit(0);
432
+ }
433
});
434
+ process.send({ type: "prepared", updateId: context.update.id });
435
} else if (context.update === undefined) {
278
- process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
436
+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
437
setInterval(() => {}, 1_000);
438
} else {
439
process.exit(0);
@@ -289,25 +447,97 @@ if (context.update?.status === "pending") {
447
childSource,
448
);
449
}
292
- yield* Effect.promise(() =>
293
- writeServiceState(statePath, {
294
- protocol: SERVICE_LAUNCHER_PROTOCOL,
295
- activeVersion: "1.0.0",
296
- }),
450
+ // The old CLI installs the new runtime but writes state with its own protocol.
451
+ yield* fs.writeFileString(
452
+ statePath,
453
+ `${JSON.stringify(
454
+ {
455
+ protocol: legacyInstall ? 3 : SERVICE_LAUNCHER_PROTOCOL,
456
+ activeVersion: "1.0.0",
457
+ },
458
+ null,
459
+ 2,
460
+ )}\n`,
461
);
298
-
299
- const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
300
- yield* Effect.promise(() =>
462
+ const launcher = new Launcher(
463
+ root,
464
+ yield* Effect.promise(() => readServiceState(statePath)),
465
+ { serviceUnit: unit },
466
+ );
467
+ const failure = yield* Effect.promise(() =>
468
launcher.run().then(
302
- () => Promise.reject(new Error("launcher unexpectedly completed")),
303
- () => Promise.resolve(),
469
+ () => "launcher unexpectedly completed",
470
+ (cause: unknown) => String(cause),
471
),
472
);
306
-
473
+ assert.include(failure, "Active child exited unexpectedly (0)");
474
const state = yield* Effect.promise(() => readServiceState(statePath));
475
assert.equal(state.activeVersion, "1.1.0");
476
assert.equal(state.update?.status, "committed");
310
- }),
477
+ assert.equal(yield* fs.readFileString(activatedPath), state.update?.id);
478
+ const recover = () =>
479
+ Effect.gen(function* () {
480
+ const fresh = new Launcher(
481
+ root,
482
+ yield* Effect.promise(() => readServiceState(statePath)),
483
+ { serviceUnit: unit },
484
+ );
485
+ const running = fresh.run();
486
+ yield* Effect.promise(() => fresh.stop("SIGTERM"));
487
+ yield* Effect.promise(() => running);
488
+ });
489
+ return { fs, root, unit, recover };
490
+ });
491
+
492
+ it.effect.each([
493
+ {
494
+ name: "systemd after a protocol-3 local upgrade",
495
+ render: renderBootServiceUnit,
496
+ legacyInstall: true,
497
+ },
498
+ {
499
+ name: "launchd",
500
+ legacyInstall: false,
501
+ render: (plan: Parameters<typeof renderBootServicePlist>[0]) =>
502
+ renderBootServicePlist(plan, {
503
+ homeDir: "/Users/test & <user>",
504
+ environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0",
505
+ }),
506
+ },
507
+ ])(
508
+ "commits and repoints $name only after the trial reports prepared",
509
+ ({ render, legacyInstall }) =>
510
+ Effect.gen(function* () {
511
+ const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall);
512
+ const expected = render(unitPlan(root, "1.1.0"));
513
+ assert.equal(yield* fs.readFileString(unit.path), expected);
514
+ assert.equal(unit.loadedContents, expected);
515
+ assert.equal(unit.reloads, 1);
516
+ }),
517
+ );
518
+
519
+ it.effect(
520
+ "keeps a durable commit through unit failure and retries a failed reload on restart",
521
+ () =>
522
+ Effect.gen(function* () {
523
+ const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true);
524
+ assert.equal(unit.reloads, 0);
525
+ const original = renderBootServiceUnit(unitPlan(root, "1.0.0"));
526
+ const expected = renderBootServiceUnit(unitPlan(root, "1.1.0"));
527
+ assert.equal(unit.loadedContents, original);
528
+ yield* fs.writeFileString(unit.path, original);
529
+ unit.failReload = true;
530
+ yield* recover();
531
+ assert.equal(yield* fs.readFileString(unit.path), expected);
532
+ assert.equal(unit.reloads, 1);
533
+ assert.equal(unit.loadedContents, original);
534
+
535
+ unit.failReload = false;
536
+ yield* recover();
537
+ assert.equal(yield* fs.readFileString(unit.path), expected);
538
+ assert.equal(unit.loadedContents, expected);
539
+ assert.equal(unit.reloads, 2);
540
+ }),
541
);
542
543
it.effect("rolls back invalid trials with the startup environment", () =>
@@ -357,9 +587,18 @@ if (context.update?.status === "pending") {
587
activeVersion: "1.0.0",
588
}),
589
);
590
+ const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0");
591
592
+ const launcher = new Launcher(
593
+ root,
594
+ yield* Effect.promise(() => readServiceState(statePath)),
595
+ {
596
+ serviceUnit: unit,
597
+ environment: yield* Effect.promise(() => readServiceEnvironment(root)),
598
+ },
599
+ );
600
yield* Effect.promise(() =>
362
- main().then(
601
+ launcher.run().then(
602
() => Promise.reject(new Error("launcher unexpectedly completed")),
603
() => Promise.resolve(),
604
),
@@ -368,6 +607,11 @@ if (context.update?.status === "pending") {
607
const state = yield* Effect.promise(() => readServiceState(statePath));
608
assert.equal(state.activeVersion, "1.0.0");
609
assert.equal(state.update?.status, "rolled-back");
610
+ assert.equal(
611
+ yield* fs.readFileString(unit.path),
612
+ renderBootServiceUnit(unitPlan(root, "1.0.0")),
613
+ );
614
+ assert.equal(unit.reloads, 1);
615
assert.equal(
616
state.update?.status === "rolled-back" ? state.update.reason : undefined,
617
"invalid-prepared",
apps/server/src/serviceLauncher.ts
+199
-17
@@ -7,6 +7,7 @@ import * as NodeFS from "node:fs";
7
import * as NodeFSP from "node:fs/promises";
8
import * as NodePath from "node:path";
9
import * as NodeUtil from "node:util";
10
+import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts";
11
12
import type {
13
PendingServiceUpdate,
@@ -21,12 +22,14 @@ import {
22
decodeServiceLauncherChildMessage,
23
isExactServiceVersion,
24
parseServiceState,
25
+ serviceUnitBaseDirOf,
26
SERVICE_ENV_FILE,
27
SERVICE_LAUNCHER_CONTEXT_ENV,
28
SERVICE_LAUNCHER_PROTOCOL,
29
SERVICE_STATE_FILE,
30
SERVICE_RESTART_PENDING_FILE,
31
SERVICE_STOP_MARKER_FILE,
32
+ SERVICE_UNIT_ENV,
33
} from "./cloud/serviceProtocol.ts";
34
35
const HANDOFF_DELAY_MS = 2_000;
@@ -64,6 +67,78 @@ const runtimeSpawnArguments = (paths: ReturnType<typeof runtimePaths>) => ({
67
args: ["serve"],
68
});
69
70
+/**
71
+ * The unit or launch agent that `t3 service install` wrote. Its program names
72
+ * the runtime that ran the install, and a remote update never replaces it, so
73
+ * after a commit the launcher points it at the committed runtime. Otherwise the
74
+ * next start lands on a runtime that nothing in the service state records.
75
+ */
76
+export interface ServiceUnit {
77
+ readonly path: string;
78
+ /** systemd keeps a loaded copy of the unit; launchd re-reads the plist at bootstrap. */
79
+ readonly reload: () => Promise<void>;
80
+}
81
+
82
+function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined {
83
+ const unitName = env[SERVICE_UNIT_ENV]?.trim();
84
+ const home = env.HOME?.trim();
85
+ if (!unitName || !home) return undefined;
86
+ if (unitName.endsWith(".plist")) {
87
+ return {
88
+ path: NodePath.join(home, "Library", "LaunchAgents", unitName),
89
+ reload: () => Promise.resolve(),
90
+ };
91
+ }
92
+ // The manager's own environment, which decides where it reads user units.
93
+ const xdgConfigHome = env.XDG_CONFIG_HOME?.trim();
94
+ const configHome =
95
+ xdgConfigHome && NodePath.isAbsolute(xdgConfigHome)
96
+ ? xdgConfigHome
97
+ : NodePath.join(home, ".config");
98
+ return {
99
+ path: NodePath.join(configHome, "systemd", "user", unitName),
100
+ reload: () =>
101
+ new Promise((resolve, reject) => {
102
+ NodeChildProcess.spawn("systemctl", ["--user", "daemon-reload"], {
103
+ stdio: "ignore",
104
+ timeout: 30_000,
105
+ })
106
+ .once("error", reject)
107
+ .once("exit", (code, signal) =>
108
+ code === 0
109
+ ? resolve()
110
+ : reject(new Error(`systemctl --user daemon-reload exited with ${code ?? signal}`)),
111
+ );
112
+ }),
113
+ };
114
+}
115
+
116
+// systemd's ExecStart value, or the plist string before the __service-launcher argument.
117
+// Only the version segment changes, so quoted or escaped paths stay as rendered.
118
+// The launch agent's PATH names a versions directory without the executable; leave it.
119
+const SERVICE_UNIT_PROGRAMS = [
120
+ /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m,
121
+ /(<string>[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*<string>__service-launcher<\/string>)/,
122
+] as const;
123
+
124
+/**
125
+ * The unit contents with their program pointed at `version`'s runtime. A unit
126
+ * already at that version or a newer one is returned unchanged: `t3 update`
127
+ * may have repointed it to a later version while this commit was in flight.
128
+ */
129
+function repointServiceUnit(contents: string, version: string): string {
130
+ for (const program of SERVICE_UNIT_PROGRAMS) {
131
+ const match = program.exec(contents);
132
+ if (match === null) continue;
133
+ const [, prefix = "", current = "", suffix = ""] = match;
134
+ if (isExactServiceVersion(current) && compareExactServiceVersions(current, version) >= 0) {
135
+ return contents;
136
+ }
137
+ return contents.replace(program, () => `${prefix}${version}${suffix}`);
138
+ }
139
+ return contents;
140
+}
141
+
142
const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const;
143
const RESTORE_MARKER = ".restore-pending";
144
@@ -445,8 +520,12 @@ export async function readServiceState(filePath: string): Promise<ServiceState>
520
}
521
522
export async function writeServiceState(filePath: string, state: ServiceState): Promise<void> {
523
+ await NodeFSP.mkdir(NodePath.dirname(filePath), { recursive: true, mode: 0o700 });
524
+ await replaceFileDurably(filePath, `${JSON.stringify(state, null, 2)}\n`);
525
+}
526
+
527
+async function replaceFileDurably(filePath: string, contents: string): Promise<void> {
528
const directory = NodePath.dirname(filePath);
449
- await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 });
529
const tempPath = NodePath.join(
530
directory,
531
`.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`,
@@ -454,7 +533,7 @@ export async function writeServiceState(filePath: string, state: ServiceState):
533
let handle: NodeFSP.FileHandle | undefined;
534
try {
535
handle = await NodeFSP.open(tempPath, "wx", 0o600);
457
- await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8");
536
+ await handle.writeFile(contents, "utf8");
537
await handle.sync();
538
await handle.close();
539
handle = undefined;
@@ -542,6 +621,7 @@ export class Launcher {
621
#done = false;
622
readonly #completion = Promise.withResolvers<void>();
623
readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
624
+ readonly #serviceUnit: ServiceUnit | undefined;
625
readonly #environment: Readonly<NodeJS.ProcessEnv>;
626
627
constructor(
@@ -549,11 +629,13 @@ export class Launcher {
629
state: ServiceState,
630
options: {
631
readonly backupSpaceProbe?: Partial<BackupSpaceProbe>;
632
+ readonly serviceUnit?: ServiceUnit;
633
readonly environment?: Readonly<NodeJS.ProcessEnv>;
634
} = {},
635
) {
636
this.#baseDir = baseDir;
637
this.#backupSpaceProbe = options.backupSpaceProbe;
638
+ this.#serviceUnit = options.serviceUnit;
639
this.#environment = { ...(options.environment ?? process.env) };
640
this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
641
this.#state = state;
@@ -620,20 +702,66 @@ export class Launcher {
702
this.#timer = undefined;
703
}
704
705
+ async #selectState(next: ServiceState): Promise<void> {
706
+ const release = await acquireServiceSelectionLock(this.#statePath);
707
+ try {
708
+ const selected = await readServiceState(this.#statePath);
709
+ // Local installs replace the selection and clear its update record. An old
710
+ // launcher must not restore its own selection when asked to update again.
711
+ if (
712
+ selected.activeVersion !== this.#state.activeVersion ||
713
+ selected.update?.id !== this.#state.update?.id ||
714
+ selected.update?.status !== this.#state.update?.status
715
+ ) {
716
+ throw new Error("Service selection changed. Restart the service before updating.");
717
+ }
718
+ await writeServiceState(this.#statePath, next);
719
+ this.#state = next;
720
+ } finally {
721
+ release();
722
+ }
723
+ }
724
+
725
async #recover(): Promise<void> {
726
// Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races.
727
await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
728
const restartPending = restartPendingPath(this.#baseDir);
627
- const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined);
628
- if (awaitedVersion?.trim() === this.#state.activeVersion) {
629
- await NodeFSP.rm(restartPending, { force: true }).catch(() => undefined);
729
+ try {
730
+ const release = await acquireServiceSelectionLock(this.#statePath);
731
+ try {
732
+ const selected = await readServiceState(this.#statePath);
733
+ const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(
734
+ () => undefined,
735
+ );
736
+ const awaited = awaitedVersion?.trim();
737
+ if (
738
+ selected.activeVersion === this.#state.activeVersion &&
739
+ selected.update?.id === this.#state.update?.id &&
740
+ selected.update?.status === this.#state.update?.status &&
741
+ awaited !== undefined &&
742
+ isExactServiceVersion(awaited) &&
743
+ compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0
744
+ ) {
745
+ await NodeFSP.rm(restartPending, { force: true });
746
+ }
747
+ } finally {
748
+ release();
749
+ }
750
+ } catch (error) {
751
+ process.stderr.write(
752
+ `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`,
753
+ );
754
}
755
const update = this.#state.update;
756
if (update?.status !== "pending") {
757
if (update !== undefined) {
758
await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined);
759
}
636
- await this.#startChild(this.#state.activeVersion, "active", update);
760
+ try {
761
+ await this.#startChild(this.#state.activeVersion, "active", update);
762
+ } finally {
763
+ if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion);
764
+ }
765
return;
766
}
767
if (await databaseRestorePending(this.#baseDir, update)) {
@@ -780,8 +908,12 @@ export class Launcher {
908
status: "pending",
909
};
910
const next: ServiceState = { ...this.#state, update: pending };
783
- await writeServiceState(this.#statePath, next);
784
- this.#state = next;
911
+ try {
912
+ await this.#selectState(next);
913
+ } catch (error) {
914
+ await reject(error instanceof Error ? error.message : String(error));
915
+ return;
916
+ }
917
await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
918
this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
919
}
@@ -827,11 +959,53 @@ export class Launcher {
959
activeVersion: pending.targetVersion,
960
update: committed,
961
};
830
- await writeServiceState(this.#statePath, next);
831
- this.#state = next;
962
+ await this.#selectState(next);
963
child.role = "active";
964
await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
834
- await sendMessage(child.process, { type: "committed", updateId: committed.id });
965
+ try {
966
+ await sendMessage(child.process, { type: "committed", updateId: committed.id });
967
+ } finally {
968
+ await this.#repointServiceUnit(committed.targetVersion);
969
+ }
970
+ }
971
+
972
+ // Selection is durable before this best-effort unit reconciliation.
973
+ async #repointServiceUnit(version: string): Promise<void> {
974
+ const unit = this.#serviceUnit;
975
+ if (unit === undefined) return;
976
+ try {
977
+ const release = await acquireServiceSelectionLock(unit.path);
978
+ try {
979
+ // The installer writes state before the unit under this lease. Do not
980
+ // acquire its state lease here: the installer takes that lease first.
981
+ const selected = await readServiceState(this.#statePath);
982
+ if (
983
+ selected.activeVersion !== this.#state.activeVersion ||
984
+ selected.update?.id !== this.#state.update?.id ||
985
+ selected.update?.status !== this.#state.update?.status
986
+ ) {
987
+ return;
988
+ }
989
+ const contents = await NodeFSP.readFile(unit.path, "utf8");
990
+ const installedBaseDir = serviceUnitBaseDirOf(contents);
991
+ if (
992
+ installedBaseDir === undefined ||
993
+ NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir)
994
+ ) {
995
+ return;
996
+ }
997
+ const repointed = repointServiceUnit(contents, version);
998
+ if (repointed !== contents) await replaceFileDurably(unit.path, repointed);
999
+ // A previous rename may have succeeded before reload failed.
1000
+ await unit.reload();
1001
+ } finally {
1002
+ release();
1003
+ }
1004
+ } catch (error) {
1005
+ process.stderr.write(
1006
+ `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`,
1007
+ );
1008
+ }
1009
}
1010
1011
async #handlePreparedTimeout(child: ManagedChild): Promise<void> {
@@ -887,9 +1061,9 @@ export class Launcher {
1061
activeVersion: pending.fromVersion,
1062
update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
1063
};
890
- await writeServiceState(this.#statePath, next);
891
- this.#state = next;
1064
+ await this.#selectState(next);
1065
await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
1066
+ await this.#repointServiceUnit(next.activeVersion);
1067
}
1068
// Signal listeners alone do not keep Node alive.
1069
this.#timer = setInterval(() => {}, 2_147_483_647);
@@ -915,10 +1089,13 @@ export class Launcher {
1089
activeVersion: pending.fromVersion,
1090
update: outcome,
1091
};
918
- await writeServiceState(this.#statePath, next);
919
- this.#state = next;
1092
+ await this.#selectState(next);
1093
await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
921
- await this.#startChild(next.activeVersion, "active", outcome);
1094
+ try {
1095
+ await this.#startChild(next.activeVersion, "active", outcome);
1096
+ } finally {
1097
+ await this.#repointServiceUnit(next.activeVersion);
1098
+ }
1099
}
1100
}
1101
@@ -930,5 +1107,10 @@ export async function main(): Promise<void> {
1107
const environment = await readServiceEnvironment(baseDir);
1108
const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
1109
const state = await readServiceState(statePath);
933
- await new Launcher(baseDir, state, { environment }).run();
1110
+ const serviceUnit = serviceUnitFromEnvironment();
1111
+ await new Launcher(
1112
+ baseDir,
1113
+ state,
1114
+ serviceUnit === undefined ? { environment } : { environment, serviceUnit },
1115
+ ).run();
1116
}
docs/internals/server-updates.md
+20
-4
@@ -4,14 +4,30 @@ A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime
4
selected by systemd or launchd. It is the only runtime writer of durable service
5
state. Server children request updates over inherited IPC; they never rewrite
6
their service definition or select their own replacement. Local service commands
7
-may replace the launcher and state while the service is stopped. Foreground CLI
8
-processes do not self-update.
7
+may replace the launcher and state, optionally deferring the restart. Foreground
8
+CLI processes do not self-update.
9
+
10
+Both writers serialize state and unit writes through
11
+[file leases](../../apps/server/src/cloud/serviceSelectionLock.ts). The installer
12
+takes the state lease before the unit lease; the launcher never holds both, so a
13
+unit failure cannot block a durable commit. Manager stop/start commands run outside
14
+both leases, since stopping a launcher that waits on a lease would deadlock. A
15
+running launcher refuses further updates once a local command changes its selection.
16
17
Exact-version installs keep restarts independent of npm cache eviction or a moving
18
release tag. Installation and preflight happen in staging before publishing an
19
immutable runtime. Preflight checks the launcher protocol because a target that
13
-needs new rollback guarantees cannot safely run under an older launcher. Upgrading
14
-that launcher requires a local service update.
20
+needs new rollback guarantees cannot safely run under an older launcher. A remote
21
+update replaces only the child, so upgrading the launcher requires a local service
22
+update with a restart.
23
+
24
+After a commit or rollback the launcher points the unit at the selected runtime and
25
+reloads the manager, so the next service start runs that version's launcher; the
26
+protocol check is what makes that safe. Each start after a finished update retries this, because
27
+a previous reload may have failed, and leaves a unit naming a newer runtime or another
28
+T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload)
29
+and launchd (before bootstrap) can respawn the old launcher, which the new runtime
30
+refuses to run under.
31
32
## Commit boundary
33
docs/user/updating.md
+5
@@ -118,6 +118,11 @@ asks before restarting the background service; if you decline, run
118
stop it and start it again afterwards with your usual options such as `--host`
119
or `--tailscale-serve`.
120
121
+A background service installed by an older release refuses **Update server**
122
+until you update it once on the host and let that update restart it. If you
123
+decline the restart, the service can stay down after a crash or reboot until you
124
+run `t3 service restart`.
125
+
126
If you run the server with `npx` rather than an installed `t3`, there is
127
nothing to update on the host: stop the server and relaunch it as
128
`npx t3@<client-version>` with the same subcommand and options.