@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 490 lines 19.1 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off -- fake systemd binaries live in a temp dir.
2 import * as NodeFS from "node:fs";
3 import * as NodeOS from "node:os";
4 import * as NodePath from "node:path";
5
6 import * as NodeServices from "@effect/platform-node/NodeServices";
7 import { assert, describe, expect, it } from "@effect/vitest";
8 import { AgentScope } from "@t3tools/shared/AgentScope";
9 import * as HostProcess from "@t3tools/shared/HostProcess";
10 import * as Effect from "effect/Effect";
11 import * as FileSystem from "effect/FileSystem";
12 import * as Layer from "effect/Layer";
13 import * as Logger from "effect/Logger";
14 import { ChildProcess, ChildProcessSpawner } from "effect/process";
15 import { vi } from "vite-plus/test";
16
17 import * as ServerConfig from "../config.ts";
18 import { makeCodexAppServerSpawnCommand } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
19 import * as ProcessRunner from "../processRunner.ts";
20 import * as ProviderProcesses from "./providerProcesses.ts";
21 import { classifyScope, make, ownedAgentSlice, type ScopeState } from "./agentScope.ts";
22
23 const GiB = 1024 ** 3;
24 const memory = vi.hoisted(() => ({ total: 16 * 1024 ** 3, swapKb: 2 * 1024 ** 2 }));
25 vi.mock("node:os", async (importOriginal) => ({
26 ...(await importOriginal<typeof import("node:os")>()),
27 totalmem: () => memory.total,
28 }));
29
30 const running: ScopeState = {
31 loadState: "loaded",
32 activeState: "active",
33 result: "success",
34 oomKills: 0,
35 populated: true,
36 };
37
38 describe("classifyScope", () => {
39 it("reports an OOM kill only once systemd records it", () => {
40 expect(classifyScope({ ...running, activeState: "failed", result: "oom-kill" })).toBe(
41 "oom-killed",
42 );
43 expect(classifyScope({ ...running, oomKills: 1 })).toBe("stopping");
44 expect(classifyScope({ ...running, populated: false })).toBe("stopping");
45 expect(classifyScope({ ...running, activeState: "deactivating" })).toBe("stopping");
46 });
47 });
48
49 interface FakeUnit {
50 activeState: string;
51 result: string;
52 controlGroup?: string;
53 }
54
55 const withFakeSystemd = <A, E>(
56 body: (input: {
57 readonly scope: Effect.Success<typeof make>;
58 readonly units: Map<string, FakeUnit>;
59 readonly calls: ReadonlyArray<ReadonlyArray<string>>;
60 readonly failingVerbs: Set<string>;
61 readonly stateDir: string;
62 readonly cgroupRoot: string;
63 }) => Effect.Effect<A, E>,
64 // A user manager older than systemd 253 rejects OOMPolicy on scopes.
65 {
66 oomPolicy = true,
67 stopFails = false,
68 leftovers = false,
69 liveOwner = false,
70 }: {
71 readonly oomPolicy?: boolean;
72 readonly stopFails?: boolean;
73 readonly leftovers?: boolean;
74 readonly liveOwner?: boolean;
75 } = {},
76 ) =>
77 Effect.gen(function* () {
78 const bin = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-agent-scope-"));
79 for (const name of ["systemd-run", "systemctl", "agent"]) {
80 NodeFS.writeFileSync(NodePath.join(bin, name), "", { mode: 0o755 });
81 }
82 const cgroupRoot = NodePath.join(bin, "cgroup");
83 const units = new Map<string, FakeUnit>();
84 const calls: Array<ReadonlyArray<string>> = [];
85 const failingVerbs = new Set<string>();
86 const output = (stdout: string, code = 0): ProcessRunner.ProcessRunOutput => ({
87 stdout,
88 stderr: "",
89 code: ChildProcessSpawner.ExitCode(code),
90 timedOut: false,
91 stdoutTruncated: false,
92 stderrTruncated: false,
93 stdoutInvalidUtf8: false,
94 stderrInvalidUtf8: false,
95 });
96 const runner = ProcessRunner.ProcessRunner.of({
97 run: ({ args }) =>
98 Effect.sync(() => {
99 calls.push(args);
100 if (!oomPolicy && args.some((arg) => arg.startsWith("--property=OOMPolicy="))) {
101 return output("", 1);
102 }
103 const [, verb, unit = ""] = args;
104 if (failingVerbs.has(verb!)) return output("", 1);
105 if (stopFails && verb === "stop") {
106 return {
107 ...output(""),
108 stderr: `Failed to stop ${args.at(-1)}: Unit not loaded.`,
109 code: ChildProcessSpawner.ExitCode(5),
110 };
111 }
112 if (verb === "reset-failed") units.delete(unit);
113 if (verb !== "show") return output("");
114 const state = units.get(unit);
115 return output(
116 state === undefined
117 ? "LoadState=not-found\nActiveState=inactive\nResult=success\n"
118 : `LoadState=loaded\nActiveState=${state.activeState}\nResult=${state.result}\n` +
119 `ControlGroup=${state.controlGroup ?? ""}\n`,
120 );
121 }),
122 });
123 const fileSystem = yield* FileSystem.FileSystem;
124 const { stateDir } = yield* ServerConfig.ServerConfig;
125 if (leftovers) {
126 units.set(ownedAgentSlice(stateDir), { activeState: "active", result: "success" });
127 }
128 const scope = yield* make.pipe(
129 Effect.provideService(ProviderProcesses.ProviderProcesses, { hasOtherLiveOwner: liveOwner }),
130 Effect.provideService(ProcessRunner.ProcessRunner, runner),
131 Effect.provideService(HostProcess.Platform, "linux"),
132 Effect.provideService(HostProcess.Environment, { PATH: bin, XDG_RUNTIME_DIR: "/run/user/1" }),
133 Effect.provideService(FileSystem.FileSystem, {
134 ...fileSystem,
135 readFileString: (path, encoding) =>
136 path === "/proc/meminfo"
137 ? Effect.sync(() => `MemTotal: 16384 kB\nSwapTotal: ${memory.swapKb} kB\n`)
138 : fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
139 writeFileString: (path, data, options) =>
140 fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
141 }),
142 );
143 return yield* body({ scope, units, calls, failingVerbs, stateDir, cgroupRoot }).pipe(
144 Effect.ensuring(Effect.sync(() => NodeFS.rmSync(bin, { recursive: true, force: true }))),
145 );
146 }).pipe(
147 Effect.scoped,
148 Effect.provide(
149 ServerConfig.layerTest(process.cwd(), { prefix: "t3-agent-scope-" }).pipe(
150 Layer.provideMerge(NodeServices.layer),
151 ),
152 ),
153 );
154
155 const unitOf = (launch: Pick<ChildProcess.StandardCommand, "args">) =>
156 launch.args
157 .map(String)
158 .find((arg) => arg.startsWith("--unit="))
159 ?.slice("--unit=".length) ?? "";
160
161 const sliceOf = (launch: { readonly args: ReadonlyArray<string> }) =>
162 launch.args.find((arg) => arg.startsWith("--slice="))?.slice("--slice=".length) ?? "";
163
164 /** An active scope as systemd and the kernel report it, with the agent itself gone. */
165 const exitedAgent = (
166 units: Map<string, FakeUnit>,
167 cgroupRoot: string,
168 unit: string,
169 { populated, oomKills }: { readonly populated: boolean; readonly oomKills: number | undefined },
170 ) => {
171 const controlGroup = `/${unit}`;
172 NodeFS.mkdirSync(NodePath.join(cgroupRoot, controlGroup), { recursive: true });
173 NodeFS.writeFileSync(
174 NodePath.join(cgroupRoot, controlGroup, "cgroup.events"),
175 `populated ${populated ? 1 : 0}\nfrozen 0\n`,
176 );
177 // An undefined count leaves memory.events unreadable.
178 if (oomKills !== undefined) {
179 NodeFS.writeFileSync(
180 NodePath.join(cgroupRoot, controlGroup, "memory.events"),
181 `low 0\nhigh 0\nmax 0\noom ${oomKills}\noom_kill ${oomKills}\noom_group_kill 0\n`,
182 );
183 }
184 units.set(unit, { activeState: "active", result: "success", controlGroup });
185 };
186
187 const stopsOf = (calls: ReadonlyArray<ReadonlyArray<string>>) =>
188 calls.filter(([, verb]) => verb === "stop");
189 const launchCodex = (scope: Effect.Success<typeof make>, threadId: string) =>
190 makeCodexAppServerSpawnCommand({ command: "agent", args: ["app-server"], threadId }).pipe(
191 Effect.provideService(AgentScope, scope),
192 Effect.provideService(HostProcess.Platform, "linux"),
193 Effect.map((command) => {
194 assert(ChildProcess.isStandardCommand(command));
195 return command;
196 }),
197 );
198
199 describe("AgentScope service", () => {
200 it.effect("labels a thread from its newest scope until its next session clears it", () =>
201 withFakeSystemd(({ scope, units }) =>
202 Effect.gen(function* () {
203 const first = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
204 units.set(unitOf(first), { activeState: "failed", result: "oom-kill" });
205 const second = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
206 units.set(unitOf(second), { activeState: "active", result: "success" });
207 expect(yield* scope.oomKilled("a")).toBe(false);
208
209 units.set(unitOf(second), { activeState: "failed", result: "oom-kill" });
210 expect(yield* scope.oomKilled("a")).toBe(true);
211 units.delete(unitOf(second));
212 expect(yield* scope.oomKilled("a")).toBe(true);
213
214 yield* scope.clear("a");
215 expect(yield* scope.oomKilled("a")).toBe(false);
216 }),
217 ),
218 );
219
220 it.effect(
221 "keeps shared Codex scopes running without attributing their OOM to the launch thread",
222 () =>
223 withFakeSystemd(({ scope, units, calls }) =>
224 Effect.gen(function* () {
225 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
226 expect(agent.args).toContain("--property=OOMPolicy=stop");
227 units.set(unitOf(agent), { activeState: "failed", result: "oom-kill" });
228 expect(yield* scope.oomKilled("a")).toBe(true);
229
230 const codex = yield* launchCodex(scope, "a");
231 expect(codex.args).toContain("--property=OOMPolicy=continue");
232 expect(codex.args).not.toContain("--property=OOMPolicy=stop");
233 expect(codex.args).toContain("--property=TimeoutStopSec=10s");
234 expect(codex.args).toContain("--collect");
235 units.set(unitOf(codex), { activeState: "active", result: "success" });
236 expect(yield* scope.oomKilled("a")).toBe(false);
237 units.set(unitOf(codex), { activeState: "failed", result: "oom-kill" });
238 expect(yield* scope.oomKilled("a")).toBe(false);
239 expect(
240 calls.filter(([, verb, unit]) => verb === "show" && unit === unitOf(codex)),
241 ).toEqual([]);
242 }),
243 ),
244 );
245
246 it.effect.each([{ oomPolicy: true }, { oomPolicy: false }])(
247 "Force stop kills a thread's own scope but never a shared Codex scope (OOM policy $oomPolicy)",
248 ({ oomPolicy }) =>
249 withFakeSystemd(
250 ({ scope, units, calls, cgroupRoot }) =>
251 Effect.gen(function* () {
252 const agent = yield* scope.wrap({
253 command: "agent",
254 args: [],
255 name: "t",
256 threadId: "a",
257 });
258 exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
259 const codex = yield* launchCodex(scope, "b");
260 exitedAgent(units, cgroupRoot, unitOf(codex), { populated: true, oomKills: 0 });
261
262 expect(yield* scope.kill("a")).toBe(true);
263 expect(
264 NodeFS.readFileSync(NodePath.join(cgroupRoot, unitOf(agent), "cgroup.kill"), "utf8"),
265 ).toBe("1");
266 expect(yield* scope.kill("b")).toBe(false);
267 expect(NodeFS.existsSync(NodePath.join(cgroupRoot, unitOf(codex), "cgroup.kill"))).toBe(
268 false,
269 );
270 expect(calls.filter(([, verb]) => verb === "kill")).toEqual([]);
271 }),
272 { oomPolicy },
273 ),
274 );
275
276 it.effect("keeps a running scope while hundreds of other threads launch", () =>
277 withFakeSystemd(({ scope, units }) =>
278 Effect.gen(function* () {
279 const live = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "live" });
280 units.set(unitOf(live), { activeState: "active", result: "success" });
281 for (let index = 0; index < 600; index++) {
282 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: `other-${index}` });
283 }
284 units.set(unitOf(live), { activeState: "failed", result: "oom-kill" });
285 expect(yield* scope.oomKilled("live")).toBe(true);
286 }),
287 ),
288 );
289
290 it.effect("delegates every agent scope so rootless containers can manage their cgroups", () =>
291 withFakeSystemd(({ scope }) =>
292 Effect.gen(function* () {
293 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
294 const codex = yield* launchCodex(scope, "a");
295 const terminal = yield* scope.wrap({
296 command: "agent",
297 args: [],
298 name: "terminal",
299 survivesRestart: true,
300 });
301 for (const launch of [agent, codex, terminal]) {
302 expect(launch.args).toContain("--property=Delegate=yes");
303 }
304 }),
305 ),
306 );
307
308 it.effect("caps agent memory and yields CPU, disk, and tasks to the server", () =>
309 withFakeSystemd(({ scope, calls }) =>
310 Effect.gen(function* () {
311 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
312 const setProperties = calls.filter(([, verb]) => verb === "set-property");
313 expect(setProperties).toEqual([
314 [
315 "--user",
316 "set-property",
317 "--runtime",
318 "app-t3code-agents.slice",
319 expect.stringMatching(/^MemoryHigh=\d+$/),
320 expect.stringMatching(/^MemoryMax=\d+$/),
321 ],
322 [
323 "--user",
324 "set-property",
325 "--runtime",
326 "app-t3code.slice",
327 "CPUWeight=10",
328 "IOWeight=10",
329 "TasksMax=25%",
330 ],
331 ]);
332 }),
333 ),
334 );
335
336 it.effect("follows a live memory or swap change at the next launch", () => {
337 memory.total = 16 * GiB;
338 memory.swapKb = 2 * 1024 ** 2;
339 return withFakeSystemd(({ scope, calls, failingVerbs }) =>
340 Effect.gen(function* () {
341 const launch = scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
342 const memoryLimits = () =>
343 calls
344 .filter(
345 ([, verb, , slice]) => verb === "set-property" && slice === "app-t3code-agents.slice",
346 )
347 .map((args) => args.slice(4));
348 yield* launch;
349 yield* launch;
350 expect(memoryLimits()).toEqual([[`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`]]);
351
352 // A VM balloon or `pct set --memory` grows RAM while the server keeps running.
353 memory.total = 22 * GiB;
354 failingVerbs.add("set-property");
355 yield* launch;
356 yield* launch;
357 failingVerbs.delete("set-property");
358 yield* launch;
359 yield* launch;
360 const grown = [`MemoryHigh=${16 * GiB}`, `MemoryMax=${18 * GiB}`];
361 expect(memoryLimits()).toEqual([
362 [`MemoryHigh=${10 * GiB}`, `MemoryMax=${12 * GiB}`],
363 grown,
364 grown,
365 grown,
366 ]);
367
368 // Without swap, MemoryHigh only throttles; MemoryMax must OOM-kill the runaway.
369 memory.swapKb = 0;
370 yield* launch;
371 expect(memoryLimits().at(-1)).toEqual(["MemoryHigh=infinity", `MemoryMax=${18 * GiB}`]);
372 memory.swapKb = 2 * 1024 ** 2;
373 }),
374 );
375 });
376
377 it.effect("stops agents a killed server left before startup completes", () =>
378 withFakeSystemd(
379 ({ scope, calls, stateDir }) =>
380 Effect.gen(function* () {
381 expect(stopsOf(calls)).toEqual([["--user", "stop", ownedAgentSlice(stateDir)]]);
382 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
383 expect(sliceOf(agent)).toBe(ownedAgentSlice(stateDir));
384 expect(agent.args).toContain("--property=TimeoutStopSec=10s");
385 const terminal = yield* scope.wrap({
386 command: "agent",
387 args: [],
388 name: "terminal",
389 survivesRestart: true,
390 });
391 expect(sliceOf(terminal)).toBe("app-t3code-agents.slice");
392 expect(stopsOf(calls)).toHaveLength(1);
393 }),
394 { leftovers: true },
395 ),
396 );
397
398 it.effect("leaves the slice to another live server sharing this home", () =>
399 withFakeSystemd(({ calls }) => Effect.sync(() => expect(stopsOf(calls)).toEqual([])), {
400 leftovers: true,
401 liveOwner: true,
402 }),
403 );
404
405 it.effect("stops leftover processes only in an active agent scope", () =>
406 withFakeSystemd(({ scope, units, calls, cgroupRoot }) =>
407 Effect.gen(function* () {
408 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
409 const unit = unitOf(agent);
410 expect(agent.unit).toBe(unit);
411
412 yield* scope.stop("session-2.scope");
413 expect(calls.some((args) => args.includes("session-2.scope"))).toBe(false);
414 yield* scope.stop(unit);
415 exitedAgent(units, cgroupRoot, unit, { populated: false, oomKills: 0 });
416 yield* scope.stop(unit);
417 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 0 });
418 units.get(unit)!.activeState = "deactivating";
419 yield* scope.stop(unit);
420 expect(stopsOf(calls)).toEqual([]);
421
422 units.get(unit)!.activeState = "active";
423 yield* scope.stop(unit);
424 expect(stopsOf(calls)).toEqual([["--user", "stop", "--no-block", unit]]);
425 expect(yield* scope.oomKilled("a")).toBe(false);
426 }),
427 ),
428 );
429
430 it.effect.each([true, false])(
431 "cleans up an OOM exit without losing its diagnosis (OOMPolicy=%s)",
432 (oomPolicy) =>
433 withFakeSystemd(
434 ({ scope, units, calls, cgroupRoot }) =>
435 Effect.gen(function* () {
436 const agent = yield* scope.wrap({
437 command: "agent",
438 args: [],
439 name: "t",
440 threadId: "a",
441 });
442 const unit = unitOf(agent);
443 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: undefined });
444 yield* scope.stop(unit);
445 expect(stopsOf(calls)).toEqual(
446 oomPolicy ? [] : [["--user", "stop", "--no-block", unit]],
447 );
448
449 // The kernel records the kill before systemd records the OOM result.
450 exitedAgent(units, cgroupRoot, unit, { populated: true, oomKills: 1 });
451 yield* scope.stop(unit);
452 expect(stopsOf(calls)).toHaveLength(oomPolicy ? 0 : 2);
453
454 if (oomPolicy) {
455 units.set(unit, { activeState: "failed", result: "oom-kill" });
456 expect(yield* scope.oomKilled("a")).toBe(true);
457 units.delete(unit);
458 expect(yield* scope.oomKilled("a")).toBe(true);
459 }
460 }),
461 { oomPolicy },
462 ),
463 );
464
465 it.effect("reports failed daemon cleanup without failing session teardown", () => {
466 const warnings: Array<unknown> = [];
467 const logger = Logger.make(({ message }) => {
468 warnings.push(message);
469 });
470 return withFakeSystemd(
471 ({ scope, units, cgroupRoot }) =>
472 Effect.gen(function* () {
473 const agent = yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
474 exitedAgent(units, cgroupRoot, unitOf(agent), { populated: true, oomKills: 0 });
475 yield* scope.stop(unitOf(agent));
476 expect(warnings.flat()).toContain("Could not stop an exited agent's scope");
477 }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))),
478 { stopFails: true },
479 );
480 });
481
482 it.effect("leaves a quiet slice alone", () =>
483 withFakeSystemd(({ scope, calls }) =>
484 Effect.gen(function* () {
485 yield* scope.wrap({ command: "agent", args: [], name: "t", threadId: "a" });
486 expect(calls.filter(([, verb]) => verb === "stop")).toEqual([]);
487 }),
488 ),
489 );
490 });