The best way to control Claude Code, Codex and more from any computer or phone
feat(server): `t3 service protect` keeps the server alive when agents fill memory (#27239)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 18:04 UTC
44d508fd8b44794e51d82480052f5a041640e411
12 files changed
+919
-20
apps/server/src/cli/service.test.ts
+31
@@ -90,6 +90,37 @@ it.layer(Layer.mergeAll(NodeServices.layer, NetService.layer))("service commands
90
}),
91
);
92
93
+ it.effect("protect refuses root before creating any T3 directories", () =>
94
+ Effect.gen(function* () {
95
+ const fs = yield* FileSystem.FileSystem;
96
+ const parent = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-cli-test-" });
97
+ const baseDir = `${parent}/t3`;
98
+ vi.spyOn(BootService, "layer").mockReturnValue(
99
+ Layer.succeed(BootService.BootService, makeTestService(status).service),
100
+ );
101
+
102
+ const output = yield* Command.runWith(serviceCommand, { version: packageJson.version })([
103
+ "protect",
104
+ "--base-dir",
105
+ baseDir,
106
+ ]).pipe(
107
+ Effect.andThen(TestConsole.logLines),
108
+ Effect.map((lines) => lines.join("\n")),
109
+ Effect.provideService(HostProcess.Environment, {}),
110
+ Effect.provideService(HostProcess.Platform, "linux"),
111
+ Effect.provideService(HostProcess.UserId, 0),
112
+ Effect.provide(
113
+ Layer.mergeAll(
114
+ ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })),
115
+ Layer.fresh(TestConsole.layer),
116
+ ),
117
+ ),
118
+ );
119
+ expect(output).toContain("without sudo");
120
+ expect(yield* fs.exists(baseDir)).toBe(false);
121
+ }),
122
+ );
123
+
124
// regression: #13239 a stale nvm-installed CLI reported "already using" its own version
125
it.effect.each(["install", "update"] as const)(
126
"%s names the CLI that found nothing to change",
apps/server/src/cli/service.ts
+132
-2
@@ -2,12 +2,14 @@ import * as HostProcess from "@t3tools/shared/HostProcess";
2
import * as Console from "effect/Console";
3
import * as Effect from "effect/Effect";
4
import * as Layer from "effect/Layer";
5
+import * as Path from "effect/Path";
6
import * as Terminal from "effect/Terminal";
7
import { Command, Flag, GlobalFlag, Prompt } from "effect/cli";
8
import { FetchHttpClient } from "effect/http";
9
10
import packageJson from "../../package.json" with { type: "json" };
11
import * as BootService from "../cloud/bootService.ts";
12
+import * as ServiceProtection from "../cloud/serviceProtection.ts";
13
import { compareExactServiceVersions } from "../cloud/serviceProtocol.ts";
14
import type * as ServerConfig from "../config.ts";
15
import * as ProcessRunner from "../processRunner.ts";
@@ -68,7 +70,23 @@ function formatServiceChange(
70
return `${result.previouslyInstalled ? "Updated" : "Installed"} T3 Code service with t3@${packageJson.version}.\nLogs: ${result.plan.logPath}`;
71
}
72
71
-function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion: string): string {
73
+function formatProtection(
74
+ protection: ServiceProtection.ServiceProtectionState | undefined,
75
+): ReadonlyArray<string> {
76
+ if (protection === undefined) return [];
77
+ return [
78
+ protection.unprotected === undefined
79
+ ? ` Memory protection: on${protection.setByProtect ? " (t3 service protect)" : ""}`
80
+ : ` Memory protection: off, ${protection.unprotected} grants none. Run \`t3 service protect\`.`,
81
+ ...(protection.oomScoreAdj === undefined ? [] : [` OOM score: ${protection.oomScoreAdj}`]),
82
+ ];
83
+}
84
+
85
+function formatServiceStatus(
86
+ status: BootService.BootServiceStatus,
87
+ cliVersion: string,
88
+ protection?: ServiceProtection.ServiceProtectionState,
89
+): string {
90
if (!status.supported) {
91
return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd";
92
}
@@ -89,6 +107,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
107
` Status: installed · t3@${installedVersion} (newer than this t3@${cliVersion} CLI)`,
108
` Unit: ${status.unitPath}`,
109
` Logs: ${status.logPath}`,
110
+ ...formatProtection(protection),
111
...problems,
112
` Next: Run \`t3 update ${installedVersion}\` to match it, or pass \`--allow-downgrade\` to \`t3 service install\` explicitly.`,
113
].join("\n");
@@ -98,6 +117,7 @@ function formatServiceStatus(status: BootService.BootServiceStatus, cliVersion:
117
` Status: ${status.current ? `installed · t3@${installedVersion}` : "needs an update or repair"}`,
118
` Unit: ${status.unitPath}`,
119
` Logs: ${status.logPath}`,
120
+ ...formatProtection(protection),
121
...problems,
122
...(status.current ? [] : [" Next: Run `t3 service install` to repair it."]),
123
].join("\n");
@@ -210,12 +230,121 @@ const serviceStatusCommand = Command.make("status", projectLocationFlags).pipe(
230
flags,
231
Effect.gen(function* () {
232
const service = yield* BootService.BootService;
213
- yield* Console.log(formatServiceStatus(yield* service.status, packageJson.version));
233
+ const path = yield* Path.Path;
234
+ const status = yield* service.status;
235
+ const protection =
236
+ status.installed && (yield* HostProcess.Platform) === "linux"
237
+ ? yield* ServiceProtection.readProtection(path.dirname(status.unitPath)).pipe(
238
+ Effect.provide(ProcessRunner.layer),
239
+ )
240
+ : undefined;
241
+ yield* Console.log(formatServiceStatus(status, packageJson.version, protection));
242
}),
243
),
244
),
245
);
246
247
+const formatDropIns = (dropIns: ReadonlyArray<ServiceProtection.DropIn>) =>
248
+ dropIns.map(
249
+ ({ path, contents }) => ` ${path}: ${contents.trimEnd().split("\n").slice(1).join(" ")}`,
250
+ );
251
+
252
+const confirm = (yes: boolean, message: string) =>
253
+ yes ? Effect.succeed(true) : Prompt.run(Prompt.Confirm({ message, initial: false }));
254
+
255
+const serviceProtectCommand = Command.make("protect", {
256
+ ...projectLocationFlags,
257
+ undo: Flag.Boolean("undo").pipe(
258
+ Flag.withDescription("Remove the protection this command set up."),
259
+ Flag.withDefault(false),
260
+ ),
261
+ yes: Flag.Boolean("yes").pipe(
262
+ Flag.withDescription("Answer yes to every question, including SSH and Tailscale."),
263
+ Flag.withDefault(false),
264
+ ),
265
+}).pipe(
266
+ Command.withDescription(
267
+ "Protect the T3 Code server from agents under memory pressure (Linux with systemd). Asks for sudo.",
268
+ ),
269
+ Command.withHandler((flags) =>
270
+ Effect.gen(function* () {
271
+ const unsupported = "Service protection needs Linux with systemd.";
272
+ if ((yield* HostProcess.Platform) !== "linux") return yield* Console.log(unsupported);
273
+ // Before resolving config, which creates T3 directories that root would then own.
274
+ if ((yield* HostProcess.UserId) === 0) {
275
+ return yield* Console.log(
276
+ "Run `t3 service protect` as the user that runs T3 Code, without sudo. It asks for sudo when it needs root.",
277
+ );
278
+ }
279
+ return yield* runServiceCommand(
280
+ flags,
281
+ Effect.gen(function* () {
282
+ const uid = yield* HostProcess.UserId;
283
+ const remote = yield* ServiceProtection.detectRemoteAccessUnits;
284
+ const { unitPath } = yield* (yield* BootService.BootService).status;
285
+ if (uid === undefined || remote === undefined || unitPath === "") {
286
+ return yield* Console.log(unsupported);
287
+ }
288
+ const userUnitDir = (yield* Path.Path).dirname(unitPath);
289
+
290
+ if (flags.undo) {
291
+ const installed = yield* ServiceProtection.installedProtection(
292
+ ServiceProtection.protectionPlan({
293
+ uid,
294
+ userUnitDir,
295
+ remoteAccessUnits: ServiceProtection.REMOTE_ACCESS_UNITS,
296
+ }),
297
+ );
298
+ if (installed === undefined) {
299
+ return yield* Console.log("T3 Code service protection is not set up.");
300
+ }
301
+ yield* Console.log(
302
+ [
303
+ "Removes:",
304
+ ...[...installed.system, ...installed.user, installed.managerDefaults].map(
305
+ ({ path, shared }) =>
306
+ ` ${path}${shared === true ? " (kept while another user is protected)" : ""}`,
307
+ ),
308
+ ].join("\n"),
309
+ );
310
+ if (!(yield* confirm(flags.yes, "Remove T3 Code service protection?"))) return;
311
+ yield* ServiceProtection.removeProtection(installed);
312
+ return yield* Console.log(
313
+ "Removed T3 Code service protection. The server's OOM score returns to normal after `t3 service restart`.",
314
+ );
315
+ }
316
+
317
+ const names = remote.map((unit) => unit.replace(/\.service$/, "")).join(" and ");
318
+ const protectRemote =
319
+ remote.length > 0 &&
320
+ (yield* confirm(
321
+ flags.yes,
322
+ `Also protect ${names} so you can still reach this machine under memory pressure?`,
323
+ ));
324
+ const plan = ServiceProtection.protectionPlan({
325
+ uid,
326
+ userUnitDir,
327
+ remoteAccessUnits: protectRemote ? remote : [],
328
+ });
329
+ yield* Console.log(
330
+ [
331
+ "As root, through sudo:",
332
+ ...formatDropIns(plan.system),
333
+ "As you:",
334
+ ...formatDropIns([plan.managerDefaults, ...plan.user]),
335
+ ].join("\n"),
336
+ );
337
+ if (!(yield* confirm(flags.yes, "Apply these settings?"))) return;
338
+ yield* ServiceProtection.applyProtection(plan);
339
+ yield* Console.log(
340
+ "Protected T3 Code. Memory protection applies now; the server takes its OOM score after `t3 service restart`. Undo with `t3 service protect --undo`.",
341
+ );
342
+ }).pipe(Effect.provide(ProcessRunner.layer)),
343
+ );
344
+ }),
345
+ ),
346
+);
347
+
348
export type ServiceOnboardingOutcome = "ready" | "pending" | "skipped";
349
350
export const offerServiceDuringOnboarding = Effect.gen(function* () {
@@ -305,6 +434,7 @@ export const serviceCommand = Command.make("service").pipe(
434
serviceRestartCommand,
435
serviceUninstallCommand,
436
serviceStatusCommand,
437
+ serviceProtectCommand,
438
serviceUpdateCommand,
439
]),
440
);
apps/server/src/cloud/bootService.test.ts
+7
-1
@@ -56,7 +56,13 @@ it("protects the server unit from agents under CPU and memory pressure", () => {
56
unitPath: "/home/theo/.config/systemd/user/t3code.service",
57
});
58
const service = unit.slice(unit.indexOf("[Service]"), unit.indexOf("[Install]"));
59
- for (const line of ["CPUWeight=1000", "IOWeight=1000", "MemoryLow=1536M", "OOMPolicy=continue"]) {
59
+ for (const line of [
60
+ "CPUWeight=1000",
61
+ "IOWeight=1000",
62
+ "MemoryLow=1536M",
63
+ "OOMScoreAdjust=100",
64
+ "OOMPolicy=continue",
65
+ ]) {
66
expect(service.split("\n")).toContain(line);
67
}
68
});
apps/server/src/cloud/bootService.ts
+3
@@ -116,6 +116,9 @@ export function renderBootServiceUnit(plan: BootServicePlan): string {
116
// Reclaim takes the agents' unprotected pages first. The kernel caps this at the
117
// protection of user.slice and user@.service, which only root can grant.
118
"MemoryLow=1536M",
119
+ // A user manager's own score, the lowest it may set: below its other services (200) and agents (800).
120
+ // `t3 service protect` lowers it further through a drop-in.
121
+ "OOMScoreAdjust=100",
122
// systemd-oomd ignores this preference when the watched cgroup is root-owned, as user-1000.slice usually is.
123
"ManagedOOMPreference=avoid",
124
`StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`,
apps/server/src/cloud/serviceProtection.test.ts
new
+265
@@ -0,0 +1,265 @@
1
+// @effect-diagnostics nodeBuiltinImport:off
2
+import * as NodeChildProcess from "node:child_process";
3
+import * as NodeFS from "node:fs";
4
+import * as NodeOS from "node:os";
5
+import * as NodePath from "node:path";
6
+
7
+import * as NodeServices from "@effect/platform-node/NodeServices";
8
+import { describe, expect, it } from "@effect/vitest";
9
+import * as HostProcess from "@t3tools/shared/HostProcess";
10
+import * as Effect from "effect/Effect";
11
+import * as FileSystem from "effect/FileSystem";
12
+import * as Sink from "effect/Sink";
13
+import * as Stream from "effect/Stream";
14
+import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
15
+
16
+import * as ProcessRunner from "../processRunner.ts";
17
+import * as ServiceProtection from "./serviceProtection.ts";
18
+
19
+const temporaryDirectory = () =>
20
+ NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-service-protection-"));
21
+
22
+describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("renderPrivilegedScript", () => {
23
+ it("writes and removes only its drop-ins and gives running main processes their score", () => {
24
+ const root = temporaryDirectory();
25
+ const target = NodeChildProcess.spawn("sleep", ["60"], { stdio: "ignore" });
26
+ try {
27
+ const systemUnitDir = NodePath.join(root, "system");
28
+ const plan = (uid: number, remoteAccessUnits: ReadonlyArray<string>) =>
29
+ ServiceProtection.protectionPlan({
30
+ uid,
31
+ userUnitDir: "/home/theo/.config/systemd/user",
32
+ remoteAccessUnits,
33
+ systemUnitDir,
34
+ });
35
+ const theo = plan(1000, ["tailscaled.service", "ssh.service"]);
36
+ expect(theo.system.find((dropIn) => dropIn.unit === "user@1000.service")?.contents).toBe(
37
+ "[Service]\nMemoryLow=2G\nOOMScoreAdjust=-900\n",
38
+ );
39
+ expect(theo.managerDefaults.path).toBe(
40
+ "/home/theo/.config/systemd/user.conf.d/90-t3code-protect.conf",
41
+ );
42
+ expect(theo.oomUnits).toEqual(["user@1000.service", "tailscaled.service", "ssh.service"]);
43
+
44
+ const unrelated = NodePath.join(systemUnitDir, "ssh.service.d", "50-admin.conf");
45
+ NodeFS.mkdirSync(NodePath.dirname(unrelated), { recursive: true });
46
+ NodeFS.writeFileSync(unrelated, "[Service]\nNice=1\n");
47
+ const bin = NodePath.join(root, "bin");
48
+ NodeFS.mkdirSync(bin);
49
+ // Raising a score needs no privilege, so the test's own child stands in for a main process.
50
+ NodeFS.writeFileSync(
51
+ NodePath.join(bin, "systemctl"),
52
+ [
53
+ "#!/bin/sh",
54
+ 'echo "$*" >>"$CALLS"',
55
+ 'case "$*" in',
56
+ ' *MainPID*) echo "$TARGET_PID" ;;',
57
+ " *OOMScoreAdjust*) echo 900 ;;",
58
+ "esac",
59
+ "",
60
+ ].join("\n"),
61
+ { mode: 0o755 },
62
+ );
63
+ const calls = NodePath.join(root, "calls");
64
+ const runScript = (protection: ServiceProtection.ProtectionPlan, mode: "apply" | "undo") => {
65
+ const result = NodeChildProcess.spawnSync(
66
+ "/bin/sh",
67
+ ["-c", ServiceProtection.renderPrivilegedScript(protection, mode)],
68
+ {
69
+ encoding: "utf8",
70
+ env: {
71
+ PATH: `${bin}:/usr/bin:/bin`,
72
+ CALLS: calls,
73
+ TARGET_PID: String(target.pid),
74
+ },
75
+ },
76
+ );
77
+ expect(result.stderr).toBe("");
78
+ expect(result.status).toBe(0);
79
+ };
80
+ const exists = (unit: string) =>
81
+ NodeFS.existsSync(NodePath.join(systemUnitDir, `${unit}.d`, "90-t3code-protect.conf"));
82
+
83
+ runScript(theo, "apply");
84
+ runScript(plan(1001, []), "apply");
85
+ for (const { path, contents } of theo.system) {
86
+ expect(NodeFS.readFileSync(path, "utf8")).toBe(contents);
87
+ }
88
+ expect(NodeFS.readFileSync(`/proc/${target.pid}/oom_score_adj`, "utf8").trim()).toBe("900");
89
+ expect(NodeFS.readFileSync(calls, "utf8").split("\n")[0]).toBe("daemon-reload");
90
+
91
+ // Undo removes every drop-in a user could own, and keeps shared ones while another user is protected.
92
+ const undo = (uid: number) => plan(uid, ServiceProtection.REMOTE_ACCESS_UNITS);
93
+ runScript(undo(1000), "undo");
94
+ expect(["user-1000.slice", "user@1000.service"].some(exists)).toBe(false);
95
+ expect(
96
+ ["user.slice", "system.slice", "tailscaled.service", "ssh.service"].every(exists),
97
+ ).toBe(true);
98
+
99
+ runScript(undo(1001), "undo");
100
+ for (const { path } of undo(1001).system) expect(NodeFS.existsSync(path)).toBe(false);
101
+ expect(NodeFS.existsSync(NodePath.join(systemUnitDir, "user.slice.d"))).toBe(false);
102
+ expect(NodeFS.readFileSync(unrelated, "utf8")).toBe("[Service]\nNice=1\n");
103
+ } finally {
104
+ target.kill();
105
+ NodeFS.rmSync(root, { recursive: true, force: true });
106
+ }
107
+ });
108
+});
109
+
110
+it.each([
111
+ [
112
+ "Debian, where sshd.service aliases ssh.service",
113
+ "Id=tailscaled.service\nLoadState=loaded\n\nId=ssh.service\nLoadState=loaded\n\nId=ssh.service\nLoadState=loaded\n",
114
+ ["tailscaled.service", "ssh.service"],
115
+ ],
116
+ [
117
+ "Fedora without Tailscale",
118
+ "Id=tailscaled.service\nLoadState=not-found\n\nId=ssh.service\nLoadState=not-found\n\nId=sshd.service\nLoadState=loaded\n",
119
+ ["sshd.service"],
120
+ ],
121
+])("finds remote-access services on %s", (_, shown, expected) => {
122
+ expect(ServiceProtection.loadedRemoteAccessUnits(shown)).toEqual(expected);
123
+});
124
+
125
+const fakeSudo = (exitCode: number, onRun: () => void) =>
126
+ ChildProcessSpawner.make((command) =>
127
+ Effect.gen(function* () {
128
+ if (command._tag !== "StandardCommand" || command.command !== "sudo") {
129
+ return yield* Effect.die("Unexpected command");
130
+ }
131
+ onRun();
132
+ return ChildProcessSpawner.makeHandle({
133
+ pid: ChildProcessSpawner.ProcessId(123),
134
+ stdout: Stream.empty,
135
+ stderr: Stream.empty,
136
+ all: Stream.empty,
137
+ exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)),
138
+ isRunning: Effect.succeed(false),
139
+ kill: () => Effect.void,
140
+ stdin: Sink.drain,
141
+ getInputFd: () => Sink.drain,
142
+ getOutputFd: () => Stream.empty,
143
+ unref: Effect.succeed(Effect.void),
144
+ });
145
+ }),
146
+ );
147
+
148
+const runner = (stdout: string) =>
149
+ ProcessRunner.ProcessRunner.of({
150
+ run: () =>
151
+ Effect.succeed({
152
+ stdout,
153
+ stderr: "",
154
+ code: ChildProcessSpawner.ExitCode(0),
155
+ timedOut: false,
156
+ stdoutTruncated: false,
157
+ stderrTruncated: false,
158
+ stdoutInvalidUtf8: false,
159
+ stderrInvalidUtf8: false,
160
+ }),
161
+ });
162
+
163
+it.effect("lowers no score before the manager default is in place, and undoes in reverse", () => {
164
+ const home = temporaryDirectory();
165
+ return Effect.gen(function* () {
166
+ const plan = ServiceProtection.protectionPlan({
167
+ uid: 1000,
168
+ userUnitDir: NodePath.join(home, ".config", "systemd", "user"),
169
+ remoteAccessUnits: [],
170
+ systemUnitDir: NodePath.join(home, "system"),
171
+ });
172
+ const managerDropIn = plan.system.find(({ unit }) => unit === "user@1000.service")!.path;
173
+ const present = () => ({
174
+ managerDefaults: NodeFS.existsSync(plan.managerDefaults.path),
175
+ user: plan.user.filter(({ path }) => NodeFS.existsSync(path)).length,
176
+ });
177
+ const withSudo = <A, E, R>(
178
+ effect: Effect.Effect<A, E, R>,
179
+ exitCode: number,
180
+ writesManagerDropIn = false,
181
+ ) => {
182
+ let atSudo: ReturnType<typeof present> | undefined;
183
+ return effect.pipe(
184
+ Effect.provideService(
185
+ ChildProcessSpawner.ChildProcessSpawner,
186
+ fakeSudo(exitCode, () => {
187
+ atSudo = present();
188
+ if (writesManagerDropIn) {
189
+ NodeFS.mkdirSync(NodePath.dirname(managerDropIn), { recursive: true });
190
+ NodeFS.writeFileSync(managerDropIn, "[Service]\nOOMScoreAdjust=-900\n");
191
+ }
192
+ }),
193
+ ),
194
+ Effect.exit,
195
+ Effect.map((exit) => ({ exit, atSudo })),
196
+ );
197
+ };
198
+ const refused = yield* withSudo(ServiceProtection.applyProtection(plan), 1);
199
+ expect(refused.exit._tag).toBe("Failure");
200
+ expect(refused.atSudo).toEqual({ managerDefaults: true, user: 0 });
201
+ expect(present()).toEqual({ managerDefaults: false, user: 0 });
202
+
203
+ // Failing after it lowered the manager, sudo must leave the manager default in place.
204
+ const partial = yield* withSudo(ServiceProtection.applyProtection(plan), 1, true);
205
+ expect(partial.exit._tag).toBe("Failure");
206
+ expect(present()).toEqual({ managerDefaults: true, user: 0 });
207
+ NodeFS.rmSync(plan.managerDefaults.path);
208
+ NodeFS.rmSync(managerDropIn);
209
+
210
+ const applied = yield* withSudo(ServiceProtection.applyProtection(plan), 0);
211
+ expect(applied.atSudo).toEqual({ managerDefaults: true, user: 0 });
212
+ expect(present()).toEqual({ managerDefaults: true, user: plan.user.length });
213
+
214
+ const removed = yield* withSudo(ServiceProtection.removeProtection(plan), 0);
215
+ expect(removed.atSudo).toEqual({ managerDefaults: true, user: 0 });
216
+ expect(present()).toEqual({ managerDefaults: false, user: 0 });
217
+ }).pipe(
218
+ Effect.ensuring(Effect.sync(() => NodeFS.rmSync(home, { recursive: true, force: true }))),
219
+ Effect.provideService(ProcessRunner.ProcessRunner, runner("")),
220
+ Effect.provide(NodeServices.layer),
221
+ );
222
+});
223
+
224
+it.effect("counts memory.min as protection and names the first cgroup that grants none", () => {
225
+ const root = temporaryDirectory();
226
+ const controlGroup = [
227
+ "user.slice",
228
+ "user-1000.slice",
229
+ "user@1000.service",
230
+ "app.slice",
231
+ "t3code.service",
232
+ ];
233
+ const write = (depth: number, file: string, value: string) => {
234
+ const directory = NodePath.join(root, ...controlGroup.slice(0, depth));
235
+ NodeFS.mkdirSync(directory, { recursive: true });
236
+ NodeFS.writeFileSync(NodePath.join(directory, file), `${value}\n`);
237
+ };
238
+ write(1, "memory.low", "0");
239
+ write(1, "memory.min", "805306368");
240
+ write(2, "memory.low", "2147483648");
241
+ write(3, "memory.low", "2147483648");
242
+ write(4, "memory.low", "0");
243
+ write(4, "memory.min", "0");
244
+ write(5, "memory.low", "1610612736");
245
+ return Effect.gen(function* () {
246
+ const fs = yield* FileSystem.FileSystem;
247
+ const read = ServiceProtection.readProtection("/home/theo/.config/systemd/user").pipe(
248
+ Effect.provideService(FileSystem.FileSystem, {
249
+ ...fs,
250
+ readFileString: (path, encoding) =>
251
+ fs.readFileString(path.replace("/sys/fs/cgroup", root), encoding),
252
+ }),
253
+ );
254
+ expect((yield* read)?.unprotected).toBe("app.slice");
255
+ write(4, "memory.low", "1610612736");
256
+ expect((yield* read)?.unprotected).toBeUndefined();
257
+ }).pipe(
258
+ Effect.ensuring(Effect.sync(() => NodeFS.rmSync(root, { recursive: true, force: true }))),
259
+ Effect.provideService(
260
+ ProcessRunner.ProcessRunner,
261
+ runner(`ControlGroup=/${controlGroup.join("/")}\nMainPID=0\n`),
262
+ ),
263
+ Effect.provide(NodeServices.layer),
264
+ );
265
+});
apps/server/src/cloud/serviceProtection.ts
new
+342
@@ -0,0 +1,342 @@
1
+import * as Effect from "effect/Effect";
2
+import * as FileSystem from "effect/FileSystem";
3
+import * as Option from "effect/Option";
4
+import * as Path from "effect/Path";
5
+import * as Schema from "effect/Schema";
6
+import * as ChildProcess from "effect/process/ChildProcess";
7
+import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
8
+
9
+import * as ProcessRunner from "../processRunner.ts";
10
+import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts";
11
+
12
+const PROTECT_DROP_IN = "90-t3code-protect.conf";
13
+const SYSTEM_UNIT_DIR = "/etc/systemd/system";
14
+export const REMOTE_ACCESS_UNITS = ["tailscaled.service", "ssh.service", "sshd.service"];
15
+
16
+// Covers app.slice (the server's 1536M) and init.scope (the user manager's 256M).
17
+const USER_CHAIN_LOW = "MemoryLow=2G";
18
+// Near the floor, but unlike -1000 the kernel can still kill it as a last resort.
19
+const PROTECTED_OOM_SCORE = "OOMScoreAdjust=-900";
20
+
21
+export interface DropIn {
22
+ readonly unit: string;
23
+ readonly path: string;
24
+ readonly contents: string;
25
+ /** Host-wide, so undo keeps it while another user's protection remains. */
26
+ readonly shared?: boolean;
27
+}
28
+
29
+export interface ProtectionPlan {
30
+ readonly systemUnitDir: string;
31
+ /** Written through sudo. */
32
+ readonly system: ReadonlyArray<DropIn>;
33
+ /**
34
+ * Written before the manager's own score drops. Unset, a user manager hands
35
+ * its services its own score plus 100: -800 once it runs at -900.
36
+ */
37
+ readonly managerDefaults: DropIn;
38
+ readonly user: ReadonlyArray<DropIn>;
39
+ /** System units whose running main process takes its configured OOM score at once. */
40
+ readonly oomUnits: ReadonlyArray<string>;
41
+}
42
+
43
+const dropIn = (
44
+ directory: string,
45
+ unit: string,
46
+ section: string,
47
+ properties: ReadonlyArray<string>,
48
+ shared = false,
49
+): DropIn => ({
50
+ unit,
51
+ path: `${directory}/${unit}.d/${PROTECT_DROP_IN}`,
52
+ contents: [`[${section}]`, ...properties, ""].join("\n"),
53
+ ...(shared ? { shared } : {}),
54
+});
55
+
56
+/**
57
+ * Memory protection only reaches the server when every ancestor cgroup grants
58
+ * it, and only root can grant it on user.slice and the user manager.
59
+ */
60
+export function protectionPlan(input: {
61
+ readonly uid: number;
62
+ readonly userUnitDir: string;
63
+ readonly remoteAccessUnits: ReadonlyArray<string>;
64
+ readonly systemUnitDir?: string;
65
+}): ProtectionPlan {
66
+ const manager = `user@${input.uid}.service`;
67
+ const remote = input.remoteAccessUnits;
68
+ const systemUnitDir = input.systemUnitDir ?? SYSTEM_UNIT_DIR;
69
+ const system = (unit: string, section: string, properties: ReadonlyArray<string>) =>
70
+ dropIn(systemUnitDir, unit, section, properties);
71
+ const shared = (unit: string, section: string, properties: ReadonlyArray<string>) =>
72
+ dropIn(systemUnitDir, unit, section, properties, true);
73
+ const user = (unit: string, section: string, properties: ReadonlyArray<string>) =>
74
+ dropIn(input.userUnitDir, unit, section, properties);
75
+ return {
76
+ systemUnitDir,
77
+ system: [
78
+ shared("user.slice", "Slice", [USER_CHAIN_LOW]),
79
+ system(`user-${input.uid}.slice`, "Slice", [USER_CHAIN_LOW]),
80
+ system(manager, "Service", [USER_CHAIN_LOW, PROTECTED_OOM_SCORE]),
81
+ ...(remote.length > 0 ? [shared("system.slice", "Slice", ["MemoryLow=512M"])] : []),
82
+ ...remote.map((unit) => shared(unit, "Service", ["MemoryLow=256M", PROTECTED_OOM_SCORE])),
83
+ ],
84
+ managerDefaults: {
85
+ unit: "user.conf",
86
+ path: `${input.userUnitDir.replace(/\/user$/, "")}/user.conf.d/${PROTECT_DROP_IN}`,
87
+ contents: "[Manager]\nDefaultOOMScoreAdjust=200\n",
88
+ },
89
+ user: [
90
+ user("app.slice", "Slice", ["MemoryLow=1536M"]),
91
+ user("init.scope", "Scope", ["MemoryLow=256M"]),
92
+ user(BOOT_SERVICE_UNIT_FILE, "Service", [PROTECTED_OOM_SCORE]),
93
+ ],
94
+ oomUnits: [manager, ...remote],
95
+ };
96
+}
97
+
98
+const quoteShell = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`;
99
+const directoryOf = (file: string) => file.slice(0, file.lastIndexOf("/"));
100
+
101
+export function renderPrivilegedScript(plan: ProtectionPlan, mode: "apply" | "undo"): string {
102
+ const lines = ["set -e"];
103
+ const remove = (dropIns: ReadonlyArray<DropIn>, indent = "") => {
104
+ for (const { path } of dropIns) {
105
+ lines.push(
106
+ `${indent}rm -f ${quoteShell(path)}`,
107
+ `${indent}rmdir ${quoteShell(directoryOf(path))} 2>/dev/null || true`,
108
+ );
109
+ }
110
+ };
111
+ if (mode === "apply") {
112
+ for (const { path, contents } of plan.system) {
113
+ lines.push(
114
+ `mkdir -p ${quoteShell(directoryOf(path))}`,
115
+ `cat >${quoteShell(path)} <<'T3CODE_PROTECT'`,
116
+ contents.trimEnd(),
117
+ "T3CODE_PROTECT",
118
+ );
119
+ }
120
+ } else {
121
+ remove(plan.system.filter((dropIn) => dropIn.shared !== true));
122
+ const otherUsers = `${quoteShell(plan.systemUnitDir)}/user@*.service.d/${PROTECT_DROP_IN}`;
123
+ lines.push(`set -- ${otherUsers}`, 'if [ ! -e "$1" ]; then');
124
+ remove(
125
+ plan.system.filter((dropIn) => dropIn.shared === true),
126
+ " ",
127
+ );
128
+ lines.push("fi");
129
+ }
130
+ lines.push(
131
+ "systemctl daemon-reload",
132
+ // OOMScoreAdjust applies at exec; this gives running main processes the configured score.
133
+ `for unit in ${plan.oomUnits.map(quoteShell).join(" ")}; do`,
134
+ ' pid=$(systemctl show --property=MainPID --value "$unit")',
135
+ ' if [ "${pid:-0}" -gt 0 ]; then',
136
+ ' systemctl show --property=OOMScoreAdjust --value "$unit" >"/proc/$pid/oom_score_adj" || true',
137
+ " fi",
138
+ "done",
139
+ "",
140
+ );
141
+ return lines.join("\n");
142
+}
143
+
144
+export class ServiceProtectionError extends Schema.TaggedError<ServiceProtectionError>()(
145
+ "ServiceProtectionError",
146
+ {
147
+ step: Schema.String,
148
+ exitCode: Schema.optional(Schema.Number),
149
+ cause: Schema.optional(Schema.Defect()),
150
+ },
151
+) {
152
+ override get message(): string {
153
+ return this.exitCode === undefined
154
+ ? `Could not ${this.step}.`
155
+ : `Could not ${this.step} (exit code ${this.exitCode}).`;
156
+ }
157
+}
158
+
159
+function parseUnits(text: string): Array<Map<string, string>> {
160
+ return text
161
+ .split(/\n\s*\n/)
162
+ .map(
163
+ (block) =>
164
+ new Map(
165
+ block
166
+ .split("\n")
167
+ .filter((line) => line.includes("="))
168
+ .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]),
169
+ ),
170
+ )
171
+ .filter((values) => values.size > 0);
172
+}
173
+
174
+/** Loaded remote-access units by canonical name, so an alias such as sshd.service counts once. */
175
+export function loadedRemoteAccessUnits(systemctlShow: string): ReadonlyArray<string> {
176
+ const units = parseUnits(systemctlShow)
177
+ .filter((values) => values.get("LoadState") === "loaded")
178
+ .map((values) => values.get("Id") ?? "")
179
+ .filter((id) => REMOTE_ACCESS_UNITS.includes(id));
180
+ return [...new Set(units)];
181
+}
182
+
183
+const run = Effect.fn("ServiceProtection.run")(function* (
184
+ step: string,
185
+ command: string,
186
+ args: ReadonlyArray<string>,
187
+) {
188
+ const runner = yield* ProcessRunner.ProcessRunner;
189
+ const result = yield* runner
190
+ .run({ command, args, timeout: "30 seconds" })
191
+ .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause })));
192
+ if (result.code !== 0) {
193
+ return yield* new ServiceProtectionError({ step, exitCode: Number(result.code) });
194
+ }
195
+ return result.stdout;
196
+});
197
+
198
+const userSettingsError = (cause: unknown) =>
199
+ new ServiceProtectionError({ step: "change systemd user settings", cause });
200
+
201
+/** Undefined when there is no systemd system manager to ask. */
202
+export const detectRemoteAccessUnits = run("list remote-access services", "systemctl", [
203
+ "show",
204
+ "--property=Id,LoadState",
205
+ ...REMOTE_ACCESS_UNITS,
206
+]).pipe(
207
+ Effect.map(loadedRemoteAccessUnits),
208
+ Effect.option,
209
+ Effect.map(Option.getOrUndefined),
210
+ Effect.withSpan("ServiceProtection.detectRemoteAccessUnits"),
211
+);
212
+
213
+const reloadUserManager = run("reload systemd user units", "systemctl", [
214
+ "--user",
215
+ "daemon-reload",
216
+]);
217
+
218
+const writeDropIns = Effect.fn("ServiceProtection.writeDropIns")(function* (
219
+ dropIns: ReadonlyArray<DropIn>,
220
+) {
221
+ const fs = yield* FileSystem.FileSystem;
222
+ for (const { path, contents } of dropIns) {
223
+ yield* fs.makeDirectory(directoryOf(path), { recursive: true });
224
+ yield* fs.writeFileString(path, contents);
225
+ }
226
+}, Effect.mapError(userSettingsError));
227
+
228
+const removeDropIns = Effect.fn("ServiceProtection.removeDropIns")(function* (
229
+ dropIns: ReadonlyArray<DropIn>,
230
+) {
231
+ const fs = yield* FileSystem.FileSystem;
232
+ for (const { path } of dropIns) {
233
+ yield* fs.remove(path, { force: true });
234
+ yield* fs.remove(directoryOf(path)).pipe(Effect.ignore);
235
+ }
236
+}, Effect.mapError(userSettingsError));
237
+
238
+const runPrivileged = Effect.fn("ServiceProtection.runPrivileged")(function* (script: string) {
239
+ const step = "change the system's service settings";
240
+ const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
241
+ const exitCode = yield* spawner
242
+ .exitCode(
243
+ ChildProcess.make("sudo", ["/bin/sh", "-c", script], {
244
+ stdin: "inherit",
245
+ stdout: "inherit",
246
+ stderr: "inherit",
247
+ }),
248
+ )
249
+ .pipe(Effect.mapError((cause) => new ServiceProtectionError({ step, cause })));
250
+ if (exitCode !== 0) {
251
+ return yield* new ServiceProtectionError({ step, exitCode: Number(exitCode) });
252
+ }
253
+});
254
+
255
+export const applyProtection = Effect.fn("ServiceProtection.apply")(function* (
256
+ plan: ProtectionPlan,
257
+) {
258
+ const fs = yield* FileSystem.FileSystem;
259
+ const hadManagerDefaults = yield* fs
260
+ .exists(plan.managerDefaults.path)
261
+ .pipe(Effect.orElseSucceed(() => false));
262
+ yield* writeDropIns([plan.managerDefaults]);
263
+ // A partial run may already have lowered the manager; its default must then stay.
264
+ const manager = plan.system.find(({ unit }) => unit.startsWith("user@"));
265
+ const rollBack = Effect.gen(function* () {
266
+ if (hadManagerDefaults) return;
267
+ if (manager !== undefined && (yield* fs.exists(manager.path))) return;
268
+ yield* removeDropIns([plan.managerDefaults]);
269
+ }).pipe(Effect.ignore);
270
+ yield* runPrivileged(renderPrivilegedScript(plan, "apply")).pipe(Effect.tapError(() => rollBack));
271
+ yield* writeDropIns(plan.user);
272
+ yield* reloadUserManager;
273
+});
274
+
275
+export const removeProtection = Effect.fn("ServiceProtection.remove")(function* (
276
+ plan: ProtectionPlan,
277
+) {
278
+ yield* removeDropIns(plan.user);
279
+ yield* reloadUserManager;
280
+ if (plan.system.length > 0) yield* runPrivileged(renderPrivilegedScript(plan, "undo"));
281
+ yield* removeDropIns([plan.managerDefaults]);
282
+ yield* reloadUserManager;
283
+});
284
+
285
+/** The plan narrowed to drop-ins that exist, for undo. */
286
+export const installedProtection = Effect.fn("ServiceProtection.installed")(function* (
287
+ plan: ProtectionPlan,
288
+) {
289
+ const fs = yield* FileSystem.FileSystem;
290
+ const exists = (dropIn: DropIn) => fs.exists(dropIn.path).pipe(Effect.orElseSucceed(() => false));
291
+ const system = yield* Effect.filter(plan.system, exists);
292
+ const user = yield* Effect.filter(plan.user, exists);
293
+ const managerDefaults = yield* exists(plan.managerDefaults);
294
+ return system.length + user.length === 0 && !managerDefaults
295
+ ? undefined
296
+ : { ...plan, system, user };
297
+});
298
+
299
+export interface ServiceProtectionState {
300
+ /** The first cgroup from the top that grants the server no memory protection. */
301
+ readonly unprotected: string | undefined;
302
+ readonly oomScoreAdj: number | undefined;
303
+ readonly setByProtect: boolean;
304
+}
305
+
306
+/** Reads what the kernel applies to the running service, whoever configured it. */
307
+export const readProtection = Effect.fn("ServiceProtection.read")(function* (userUnitDir: string) {
308
+ const fs = yield* FileSystem.FileSystem;
309
+ const path = yield* Path.Path;
310
+ const shown = yield* run("read the service's cgroup", "systemctl", [
311
+ "--user",
312
+ "show",
313
+ BOOT_SERVICE_UNIT_FILE,
314
+ "--property=ControlGroup,MainPID",
315
+ ]).pipe(Effect.option);
316
+ if (Option.isNone(shown)) return undefined;
317
+ const values = parseUnits(shown.value)[0] ?? new Map<string, string>();
318
+ const controlGroup = (values.get("ControlGroup") ?? "").split("/").filter(Boolean);
319
+ if (controlGroup.length === 0) return undefined;
320
+ const read = (file: string) =>
321
+ fs.readFileString(file).pipe(
322
+ Effect.map((text) => text.trim()),
323
+ Effect.orElseSucceed(() => ""),
324
+ );
325
+ let unprotected: string | undefined;
326
+ const grants = (value: string) => value !== "" && value !== "0";
327
+ for (let depth = 1; depth <= controlGroup.length && unprotected === undefined; depth++) {
328
+ const directory = path.join("/sys/fs/cgroup", ...controlGroup.slice(0, depth));
329
+ const low = yield* read(path.join(directory, "memory.low"));
330
+ const min = yield* read(path.join(directory, "memory.min"));
331
+ if (!grants(low) && !grants(min)) unprotected = controlGroup[depth - 1];
332
+ }
333
+ const pid = Number(values.get("MainPID") ?? 0);
334
+ const score = pid > 0 ? yield* read(`/proc/${pid}/oom_score_adj`) : "";
335
+ return {
336
+ unprotected,
337
+ oomScoreAdj: score === "" ? undefined : Number(score),
338
+ setByProtect: yield* fs
339
+ .exists(path.join(userUnitDir, `${BOOT_SERVICE_UNIT_FILE}.d`, PROTECT_DROP_IN))
340
+ .pipe(Effect.orElseSucceed(() => false)),
341
+ } satisfies ServiceProtectionState;
342
+});
apps/server/src/preview/ServerBrowser.ts
+14
@@ -43,6 +43,7 @@ import {
43
SERVER_BROWSER_AUTOMATION_CLIENT_ID,
44
type PreviewAppearancePreference,
45
} from "@t3tools/contracts";
46
+import { AgentScope, agentScopeLauncherScript } from "@t3tools/shared/AgentScope";
47
import * as HostProcess from "@t3tools/shared/HostProcess";
48
import { normalizePreviewUrl } from "@t3tools/shared/preview";
49
import { resolvePreviewViewport } from "@t3tools/shared/previewViewport";
@@ -460,6 +461,8 @@ const make = Effect.gen(function* () {
461
const runPromise = Effect.runPromiseWith(yield* Effect.context<never>());
462
const launchServices = yield* Effect.context<ChildProcessSpawner.ChildProcessSpawner>();
463
const setupCommand = yield* resolveRootCliCommand(PreviewBrowserHost.SETUP_SUBCOMMAND);
464
+ const agentScope = yield* AgentScope;
465
+ const launchersDir = NodePath.join(config.stateDir, "server-browser", "launchers");
466
467
const tabs = new Map<string, ServerTab>();
468
const pendingTabs = new Map<string, Promise<ServerTab>>();
@@ -473,6 +476,17 @@ const make = Effect.gen(function* () {
476
const contexts = new ServerBrowserContexts({
477
profilesDir: NodePath.join(config.stateDir, "server-browser", "profiles"),
478
executable: () => runPromise(previewBrowser.executable),
479
+ // An agent scope keeps Chrome out of the server's memory protection and OOM score.
480
+ launcher: async (executable) => {
481
+ const launch = await runPromise(
482
+ agentScope.wrap({ command: executable, args: [], name: "preview-browser" }),
483
+ );
484
+ if (launch.command === executable) return undefined;
485
+ await NodeFSP.mkdir(launchersDir, { recursive: true });
486
+ const launcher = NodePath.join(launchersDir, NodeCrypto.randomUUID());
487
+ await NodeFSP.writeFile(launcher, agentScopeLauncherScript(launch), { mode: 0o700 });
488
+ return { path: launcher, release: () => NodeFSP.rm(launcher, { force: true }) };
489
+ },
490
// Playwright's launch error drops Chrome's own output; its sandbox note survives.
491
diagnose: (executable, cause) =>
492
Effect.runPromiseWith(launchServices)(
apps/server/src/preview/ServerBrowserContexts.test.ts
+51
-5
@@ -55,6 +55,17 @@ const options = () => ({
55
env: {},
56
});
57
58
+const withLauncher = () => {
59
+ const release = vi.fn(async () => {});
60
+ return {
61
+ release,
62
+ configuration: {
63
+ ...options(),
64
+ launcher: vi.fn(async (executable: string) => ({ path: `${executable}-launcher`, release })),
65
+ },
66
+ };
67
+};
68
+
69
beforeEach(() => {
70
launches.launch.mockReset();
71
launches.persistent.mockReset();
@@ -66,19 +77,29 @@ describe("ServerBrowserContexts", () => {
77
it("lazily shares one browser across concurrent isolated sessions and reuses each context", async () => {
78
const browser = makeBrowser();
79
const launch = Promise.withResolvers<Browser>();
69
- launches.launch.mockReturnValue(launch.promise);
70
- const configuration = options();
80
+ const started = Promise.withResolvers<void>();
81
+ launches.launch.mockImplementation(() => {
82
+ started.resolve();
83
+ return launch.promise;
84
+ });
85
+ const { configuration, release } = withLauncher();
86
const pool = new ServerBrowserContexts(configuration);
87
expect(configuration.executable).not.toHaveBeenCalled();
88
const first = pool.contextFor("default", "agent-a");
89
const again = pool.contextFor("default", "agent-a");
90
const second = pool.contextFor("default", "agent-b");
91
const incognito = pool.contextFor(INCOGNITO_BROWSER_PROFILE_ID);
92
+ await started.promise;
93
+ // Playwright may retry the same path until the launch settles.
94
+ expect(release).not.toHaveBeenCalled();
95
launch.resolve(browser as unknown as Browser);
96
const [a, reused, b, privateContext] = await Promise.all([first, again, second, incognito]);
97
+ expect(release).toHaveBeenCalledTimes(1);
98
expect(a).toBe(reused);
99
expect(new Set([a, b, privateContext]).size).toBe(3);
81
- expect(launches.launch).toHaveBeenCalledTimes(1);
100
+ expect(launches.launch).toHaveBeenCalledExactlyOnceWith(
101
+ expect.objectContaining({ executablePath: "/test/chromium-launcher" }),
102
+ );
103
expect(launches.persistent).not.toHaveBeenCalled();
104
expect(browser.newContext).toHaveBeenCalledWith({
105
viewport: { width: 1280, height: 800 },
@@ -94,19 +115,44 @@ describe("ServerBrowserContexts", () => {
115
const browser = makeBrowser();
116
launches.persistent.mockResolvedValue(human as unknown as BrowserContext);
117
launches.launch.mockResolvedValue(browser as unknown as Browser);
97
- const pool = new ServerBrowserContexts(options());
118
+ const { configuration, release } = withLauncher();
119
+ const pool = new ServerBrowserContexts(configuration);
120
const persistent = await pool.contextFor("work/team");
121
expect(await pool.contextFor("work/team")).toBe(persistent);
122
expect(await pool.contextFor("work/team", "agent")).not.toBe(persistent);
123
expect(launches.persistent).toHaveBeenCalledExactlyOnceWith(
124
"/test/profiles/work%2Fteam",
103
- expect.objectContaining({ chromiumSandbox: true, executablePath: "/test/chromium" }),
125
+ expect.objectContaining({ chromiumSandbox: true, executablePath: "/test/chromium-launcher" }),
126
);
127
+ expect(release).toHaveBeenCalledTimes(2);
128
expect(launches.mkdir).toHaveBeenCalledWith("/test/profiles/work%2Fteam", { recursive: true });
129
await pool.close();
130
expect(human.close).toHaveBeenCalledTimes(1);
131
});
132
133
+ it("diagnoses a launcher failure against the real executable", async () => {
134
+ const diagnosed = new Error("diagnosed");
135
+ const diagnose = vi.fn(async () => diagnosed);
136
+ const pool = new ServerBrowserContexts({
137
+ ...options(),
138
+ diagnose,
139
+ launcher: vi.fn(async () => {
140
+ throw new Error("EACCES");
141
+ }),
142
+ });
143
+ await expect(pool.contextFor("default", "agent")).rejects.toBe(diagnosed);
144
+ expect(diagnose).toHaveBeenCalledExactlyOnceWith("/test/chromium", expect.any(Error));
145
+ expect(launches.launch).not.toHaveBeenCalled();
146
+ });
147
+
148
+ it("creates no launcher for a profile whose storage cannot be created", async () => {
149
+ launches.mkdir.mockRejectedValueOnce(new Error("EACCES"));
150
+ const { configuration } = withLauncher();
151
+ const pool = new ServerBrowserContexts(configuration);
152
+ await expect(pool.contextFor("work/team")).rejects.toThrow("EACCES");
153
+ expect(configuration.launcher).not.toHaveBeenCalled();
154
+ });
155
+
156
it("clears a profile by closing its open context before deleting its storage", async () => {
157
const human = makeContext();
158
const browser = makeBrowser();
apps/server/src/preview/ServerBrowserContexts.ts
+29
-11
@@ -17,6 +17,13 @@ const loadPlaywright = () =>
17
interface Options {
18
readonly profilesDir: string;
19
readonly executable: () => Promise<string>;
20
+ /**
21
+ * A path that starts `executable`, such as one that moves Chrome out of the server's cgroup.
22
+ * Released after the whole launch call, since Playwright retries the same path after a glibc race.
23
+ */
24
+ readonly launcher?: (
25
+ executable: string,
26
+ ) => Promise<{ readonly path: string; readonly release: () => Promise<void> } | undefined>;
27
readonly diagnose?: (executable: string, cause: unknown) => Promise<unknown>;
28
readonly env?: NodeJS.ProcessEnv;
29
readonly onContextClose?: (context: BrowserContext) => void;
@@ -79,10 +86,10 @@ export class ServerBrowserContexts {
86
}
87
88
private async launchOptions() {
82
- const executablePath = await this.options.executable();
89
+ const executable = await this.options.executable();
90
const env = this.options.env ?? process.env;
91
return {
85
- executablePath,
92
+ executable,
93
env,
94
args: [
95
"--disable-gpu",
@@ -96,19 +103,26 @@ export class ServerBrowserContexts {
103
};
104
}
105
99
- private async launch<A>(options: { readonly executablePath: string }, start: () => Promise<A>) {
106
+ private async launch<A>(executable: string, start: (executablePath: string) => Promise<A>) {
107
+ let release: (() => Promise<void>) | undefined;
108
try {
101
- return await start();
109
+ const launcher = await this.options.launcher?.(executable);
110
+ release = launcher?.release;
111
+ return await start(launcher?.path ?? executable);
112
} catch (cause) {
103
- throw (await this.options.diagnose?.(options.executablePath, cause)) ?? cause;
113
+ throw (await this.options.diagnose?.(executable, cause)) ?? cause;
114
+ } finally {
115
+ await release?.().catch(constVoid);
116
}
117
}
118
119
private sharedBrowser() {
120
if (!this.browser) {
109
- const launched = this.launchOptions().then(async (options) => {
121
+ const launched = this.launchOptions().then(async ({ executable, ...options }) => {
122
const { chromium } = loadPlaywright();
111
- const browser = await this.launch(options, () => chromium.launch(options));
123
+ const browser = await this.launch(executable, (executablePath) =>
124
+ chromium.launch({ ...options, executablePath }),
125
+ );
126
browser.on("disconnected", () => {
127
if (this.browser === launched) this.browser = undefined;
128
});
@@ -156,11 +170,15 @@ export class ServerBrowserContexts {
170
return browser.newContext(contextOptions);
171
}
172
const directory = this.profileDirectory(profileId);
159
- const options = await this.launchOptions();
160
- const { chromium } = loadPlaywright();
173
await NodeFSP.mkdir(directory, { recursive: true });
162
- return this.launch(options, () =>
163
- chromium.launchPersistentContext(directory, { ...options, ...contextOptions }),
174
+ const { executable, ...options } = await this.launchOptions();
175
+ const { chromium } = loadPlaywright();
176
+ return this.launch(executable, (executablePath) =>
177
+ chromium.launchPersistentContext(directory, {
178
+ ...options,
179
+ ...contextOptions,
180
+ executablePath,
181
+ }),
182
);
183
}
184
docs/user/background-service.md
+18
@@ -90,6 +90,24 @@ Windows background services are not supported.
90
T3 Connect can offer service installation during setup, but the two are managed
91
separately. Signing out of T3 Connect does not stop or uninstall the service.
92
93
+## Keep the server responsive when agents use all memory
94
+
95
+On Linux, agents already run with lower priority than the server. When they
96
+fill the machine's memory, though, the kernel may still slow down or stop the
97
+server, because only an administrator can reserve memory for it. Run, as your
98
+normal user:
99
+
100
+```sh
101
+t3 service protect
102
+```
103
+
104
+It shows the system settings it will change, asks before applying them, and
105
+then asks for your sudo password. If the machine runs Tailscale or SSH, it
106
+also offers to protect them, so you can still reach the machine. Memory
107
+protection takes effect at once. The server's protection from the kernel's
108
+out-of-memory killer takes effect after `t3 service restart`. `t3 service status` reports whether
109
+the protection is in place, and `t3 service protect --undo` removes it.
110
+
111
## Troubleshooting
112
113
Start with `t3 service status` on the host. It prints the log path and, on Linux,
packages/shared/src/AgentScope.test.ts
+19
-1
@@ -1,9 +1,12 @@
1
// @effect-diagnostics nodeBuiltinImport:off
2
import * as NodeChildProcess from "node:child_process";
3
+import * as NodeFS from "node:fs";
4
+import * as NodeOS from "node:os";
5
+import * as NodePath from "node:path";
6
7
import { describe, expect, it } from "@effect/vitest";
8
6
-import { AGENT_OOM_SCORE_ADJ, agentScopeCommand } from "./AgentScope.ts";
9
+import { AGENT_OOM_SCORE_ADJ, agentScopeCommand, agentScopeLauncherScript } from "./AgentScope.ts";
10
import * as HostProcess from "./HostProcess.ts";
11
12
describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("agentScopeCommand", () => {
@@ -25,5 +28,20 @@ describe.skipIf(HostProcess.Platform.defaultValue() !== "linux")("agentScopeComm
28
expect(Number(pid)).toBe(child.pid);
29
expect(Number(score)).toBe(AGENT_OOM_SCORE_ADJ);
30
expect(args).toEqual(["a b", "$HOME"]);
31
+
32
+ const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-launcher-"));
33
+ try {
34
+ const launcher = NodePath.join(directory, "launch's");
35
+ NodeFS.writeFileSync(launcher, agentScopeLauncherScript(launch), { mode: 0o700 });
36
+ const launched = NodeChildProcess.spawnSync(launcher, ["--flag", "it's"], {
37
+ encoding: "utf8",
38
+ });
39
+ const [launchedPid, launchedScore, ...launchedArgs] = launched.stdout.trimEnd().split("\n");
40
+ expect(Number(launchedPid)).toBe(launched.pid);
41
+ expect(Number(launchedScore)).toBe(AGENT_OOM_SCORE_ADJ);
42
+ expect(launchedArgs).toEqual(["a b", "$HOME", "--flag", "it's"]);
43
+ } finally {
44
+ NodeFS.rmSync(directory, { recursive: true, force: true });
45
+ }
46
});
47
});
packages/shared/src/AgentScope.ts
+8
@@ -74,6 +74,14 @@ export interface AgentSystemdScope {
74
readonly options: ReadonlyArray<string>;
75
}
76
77
+const quoteShell = (value: string) => `'${value.replaceAll("'", `'\\''`)}'`;
78
+
79
+/** An executable file for launchers that take a path, not a command line, such as Playwright. */
80
+export function agentScopeLauncherScript(launch: AgentScopeCommand): string {
81
+ const command = [launch.command, ...launch.args].map(quoteShell).join(" ");
82
+ return `#!/bin/sh\nexec ${command} "$@"\n`;
83
+}
84
+
85
/** Exec at every step preserves the agent PID, stdio, and signals; systemd-run places it and descendants in the scope. */
86
export function agentScopeCommand(input: {
87
readonly command: string;