@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 474 lines 18.5 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off -- unit names and RAM size come from the host.
2 import * as NodeCrypto from "node:crypto";
3 import * as NodeOS from "node:os";
4
5 import {
6 AgentScope,
7 agentScopeCommand,
8 type AgentScopeShape,
9 type AgentSystemdScope,
10 } from "@t3tools/shared/AgentScope";
11 import * as HostProcess from "@t3tools/shared/HostProcess";
12 import { resolveCommandPath } from "@t3tools/shared/shell";
13 import * as Effect from "effect/Effect";
14 import * as FileSystem from "effect/FileSystem";
15 import * as Layer from "effect/Layer";
16 import * as Path from "effect/Path";
17 import * as Semaphore from "effect/Semaphore";
18
19 import * as ServerConfig from "../config.ts";
20 import * as ProcessRunner from "../processRunner.ts";
21 import * as ProviderProcesses from "./providerProcesses.ts";
22
23 const AGENT_SLICE = "app-t3code-agents.slice";
24 // Owns every agent slice and competes with the server's unit or scope under app.slice.
25 const AGENTS_PARENT_SLICE = "app-t3code.slice";
26 // Low weights yield CPU and disk to the server under contention, where the user manager
27 // has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents
28 // below logind's 33% per-user task limit, leaving forks for the server.
29 const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"];
30
31 export function ownedAgentSlice(stateDir: string): string {
32 const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
33 return `app-t3code-agents-${hash}.slice`;
34 }
35
36 const GiB = 1024 ** 3;
37
38 /**
39 * Agents leave about 6 GB free before throttling and 4 GB before an OOM kill (at
40 * least half and three quarters of RAM go to agents) so the server and sshd keep
41 * working. Without swap, MemoryHigh stalls every agent and never reaches the kill.
42 */
43 function agentSliceMemoryLimits(totalBytes: number, hasSwap: boolean) {
44 return {
45 high: hasSwap ? Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)) : "infinity",
46 max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)),
47 };
48 }
49
50 export interface ScopeState {
51 readonly loadState: string;
52 readonly activeState: string;
53 readonly result: string;
54 readonly oomKills: number | undefined;
55 readonly populated: boolean | undefined;
56 }
57
58 /** Pipes can close before systemd records oom-kill; stopping, empty or OOM-counted scopes remain pending. */
59 export function classifyScope(state: ScopeState): "oom-killed" | "gone" | "running" | "stopping" {
60 if (state.result === "oom-kill") return "oom-killed";
61 if (
62 state.loadState !== "loaded" ||
63 state.activeState === "inactive" ||
64 state.activeState === "failed"
65 ) {
66 return "gone";
67 }
68 if (
69 state.activeState === "deactivating" ||
70 (state.oomKills ?? 0) > 0 ||
71 state.populated === false
72 ) {
73 return "stopping";
74 }
75 return "running";
76 }
77
78 function parseKeyValues(text: string): Map<string, string> {
79 const values = new Map<string, string>();
80 for (const line of text.split("\n")) {
81 const index = line.indexOf("=");
82 if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim());
83 }
84 return values;
85 }
86
87 function parseCgroupCounter(text: string, key: string): number | undefined {
88 const match = new RegExp(`^${key} (\\d+)$`, "m").exec(text);
89 return match === null ? undefined : Number(match[1]);
90 }
91
92 // systemd's default 90 s stop timeout would outlast our leftover-agent wait.
93 // Delegation lets rootless containers (Podman, kind) manage their own cgroups.
94 const SCOPE_OPTIONS = ["--property=TimeoutStopSec=10s", "--property=Delegate=yes"];
95 const STOPPING_POLL_INTERVAL = "100 millis";
96 const STOPPING_POLL_ATTEMPTS = 20;
97 const CLEAR_FAILED_INTERVAL = "1 second";
98 const CLEAR_FAILED_ATTEMPTS = 120;
99 const MAX_TRACKED_THREADS = 500;
100
101 export const make = Effect.gen(function* () {
102 const platform = yield* HostProcess.Platform;
103 if (platform !== "linux") return AgentScope.defaultValue();
104
105 const runner = yield* ProcessRunner.ProcessRunner;
106 const fileSystem = yield* FileSystem.FileSystem;
107 const path = yield* Path.Path;
108 const hostEnvironment = yield* HostProcess.Environment;
109 const { stateDir } = yield* ServerConfig.ServerConfig;
110 const { hasOtherLiveOwner } = yield* ProviderProcesses.ProviderProcesses;
111 const ownedSlice = ownedAgentSlice(stateDir);
112 const layerScope = yield* Effect.scope;
113
114 const resolve = (command: string, env: NodeJS.ProcessEnv) =>
115 resolveCommandPath(command, { env }).pipe(
116 Effect.provideService(FileSystem.FileSystem, fileSystem),
117 Effect.provideService(Path.Path, path),
118 Effect.provideService(HostProcess.Platform, platform),
119 );
120
121 const run = (command: string, args: ReadonlyArray<string>) =>
122 runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option);
123
124 const readCgroupFile = (controlGroup: string, file: string) =>
125 fileSystem
126 .readFileString(path.join("/sys/fs/cgroup", controlGroup, file))
127 .pipe(Effect.orElseSucceed(() => ""));
128
129 // Old agents keep their scopes when the server is killed, and the restarted
130 // server resumes their sessions. Stop them first, so two copies of one
131 // session never work in the same worktree.
132 const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* (
133 systemctl: string,
134 ) {
135 if (hasOtherLiveOwner) return;
136 const shown = yield* run(systemctl, [
137 "--user",
138 "show",
139 ownedSlice,
140 "--property=ActiveState,ControlGroup",
141 ]);
142 if (shown._tag === "None") return;
143 const values = parseKeyValues(shown.value.stdout);
144 if (values.get("ActiveState") !== "active") return;
145 const controlGroup = values.get("ControlGroup") ?? "";
146 const events = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
147 if (parseCgroupCounter(events, "populated") === 0) return;
148 yield* Effect.logWarning("Stopping agents left by a previous server", { slice: ownedSlice });
149 const stopped = yield* runner
150 .run({ command: systemctl, args: ["--user", "stop", ownedSlice], timeout: "30 seconds" })
151 .pipe(Effect.option);
152 if (stopped._tag === "None" || stopped.value.code !== 0) {
153 yield* Effect.logWarning("Could not stop agents left by a previous server", {
154 slice: ownedSlice,
155 stderr: stopped._tag === "Some" ? stopped.value.stderr.trim() : undefined,
156 });
157 }
158 });
159
160 const systemd = yield* Effect.cached(
161 Effect.gen(function* () {
162 const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;
163 if (runtimeDir === undefined || runtimeDir.length === 0) return undefined;
164 const systemdRun = yield* resolve("systemd-run", hostEnvironment);
165 const systemctl = yield* resolve("systemctl", hostEnvironment);
166 if (systemdRun === null || systemctl === null) return undefined;
167 yield* stopLeftoverAgents(systemctl);
168 const probe = (options: ReadonlyArray<string>) =>
169 run(systemdRun, [
170 "--user",
171 "--scope",
172 "--quiet",
173 "--collect",
174 `--slice=${AGENT_SLICE}`,
175 ...options,
176 "--",
177 "true",
178 ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0));
179 // Scopes take OOMPolicy from systemd 253; older managers still get scopes.
180 const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]);
181 if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) {
182 yield* Effect.logInfo("Agent scopes are off: no systemd user manager");
183 return undefined;
184 }
185 const setProperties = Effect.fn("agentScope.setProperties")(function* (
186 slice: string,
187 properties: ReadonlyArray<string>,
188 ) {
189 const result = yield* run(systemctl, [
190 "--user",
191 "set-property",
192 "--runtime",
193 slice,
194 ...properties,
195 ]);
196 if (result._tag === "Some" && result.value.code === 0) return true;
197 yield* Effect.logWarning("Could not set resource limits on an agents slice", {
198 slice,
199 properties,
200 stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined,
201 });
202 return false;
203 });
204 // VMs and containers can resize RAM live, so each launch re-checks the total
205 // the limits came from. A failed update is retried by the next launch.
206 // An unreadable meminfo keeps the throttle, as before swap was checked.
207 const readHasSwap = fileSystem.readFileString("/proc/meminfo").pipe(
208 Effect.map((text) => /^SwapTotal:\s*0 kB$/m.exec(text) === null),
209 Effect.orElseSucceed(() => true),
210 );
211 let appliedTotal: number | undefined;
212 let appliedHasSwap: boolean | undefined;
213 const memoryLimitsLock = yield* Semaphore.make(1);
214 const applyMemoryLimits = memoryLimitsLock.withPermits(1)(
215 Effect.gen(function* () {
216 const total = NodeOS.totalmem();
217 const hasSwap = yield* readHasSwap;
218 if (total === appliedTotal && hasSwap === appliedHasSwap) return;
219 const limits = agentSliceMemoryLimits(total, hasSwap);
220 const applied = yield* setProperties(AGENT_SLICE, [
221 `MemoryHigh=${limits.high}`,
222 `MemoryMax=${limits.max}`,
223 ]);
224 if (!applied) return;
225 if (appliedTotal !== undefined) {
226 yield* Effect.logInfo("Agent memory limits follow a memory change", {
227 slice: AGENT_SLICE,
228 previousTotal: appliedTotal,
229 total,
230 ...limits,
231 });
232 }
233 appliedTotal = total;
234 appliedHasSwap = hasSwap;
235 }),
236 );
237 yield* applyMemoryLimits;
238 yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES);
239 yield* Effect.logInfo("Agent scopes are on", {
240 slice: AGENT_SLICE,
241 ...agentSliceMemoryLimits(NodeOS.totalmem(), yield* readHasSwap),
242 oomPolicy: supportsOomPolicy,
243 });
244 return {
245 systemdRun,
246 systemctl,
247 runtimeDir,
248 supportsOomPolicy,
249 applyMemoryLimits,
250 };
251 }),
252 );
253
254 // Layer acquisition is the startup barrier: recovery must not terminalize or
255 // continue old runs while a previous server's providers can still execute.
256 yield* systemd;
257
258 // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.
259 const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>();
260
261 const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) {
262 const shown = yield* run(systemctl, [
263 "--user",
264 "show",
265 unit,
266 "--property=LoadState,ActiveState,Result,ControlGroup",
267 ]);
268 if (shown._tag === "None" || shown.value.code !== 0) return undefined;
269 const values = parseKeyValues(shown.value.stdout);
270 const controlGroup = values.get("ControlGroup") ?? "";
271 const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, "memory.events") : "";
272 const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : "";
273 const populated = parseCgroupCounter(cgroupEvents, "populated");
274 return {
275 loadState: values.get("LoadState") ?? "",
276 activeState: values.get("ActiveState") ?? "",
277 result: values.get("Result") ?? "",
278 oomKills: parseCgroupCounter(memoryEvents, "oom_kill"),
279 populated: populated === undefined ? undefined : populated === 1,
280 } satisfies ScopeState;
281 });
282
283 const clearFailed = Effect.fn("agentScope.clearFailed")(function* (
284 systemctl: string,
285 unit: string,
286 ) {
287 for (let attempt = 0; attempt < CLEAR_FAILED_ATTEMPTS; attempt++) {
288 const state = yield* readScope(systemctl, unit);
289 if (state === undefined) return;
290 if (state.activeState === "failed") {
291 yield* run(systemctl, ["--user", "reset-failed", unit]);
292 return;
293 }
294 if (state.activeState !== "deactivating") return;
295 yield* Effect.sleep(CLEAR_FAILED_INTERVAL);
296 }
297 });
298
299 const forget = (systemctl: string, unit: string) =>
300 clearFailed(systemctl, unit).pipe(Effect.forkIn(layerScope), Effect.asVoid);
301
302 const evictOldest = Effect.fn("agentScope.evictOldest")(function* (systemctl: string) {
303 const oldest = scopeByThread.entries().next();
304 if (oldest.done === true) return;
305 const [threadId, tracked] = oldest.value;
306 const state = yield* readScope(systemctl, tracked.unit);
307 if (scopeByThread.get(threadId) !== tracked) return;
308 scopeByThread.delete(threadId);
309 const status = state === undefined ? "running" : classifyScope(state);
310 if (status === "running" || status === "stopping") {
311 scopeByThread.set(threadId, tracked);
312 return;
313 }
314 yield* forget(systemctl, tracked.unit);
315 });
316
317 const track = Effect.fn("agentScope.track")(function* (
318 systemctl: string,
319 threadId: string,
320 unit: string | undefined,
321 shared = false,
322 ) {
323 const previous = scopeByThread.get(threadId);
324 scopeByThread.delete(threadId);
325 if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared });
326 if (previous !== undefined) yield* forget(systemctl, previous.unit);
327 if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);
328 });
329
330 const oomKilled: AgentScopeShape["oomKilled"] = Effect.fn("agentScope.oomKilled")(
331 function* (threadId) {
332 const tracked = scopeByThread.get(threadId);
333 if (tracked === undefined) return false;
334 if (tracked.oomKilled) return true;
335 const scope = yield* systemd;
336 if (scope === undefined) return false;
337 for (let attempt = 0; attempt < STOPPING_POLL_ATTEMPTS; attempt++) {
338 const state = yield* readScope(scope.systemctl, tracked.unit);
339 if (state === undefined) return false;
340 const status = classifyScope(state);
341 if (status === "running") return false;
342 if (status === "stopping") {
343 yield* Effect.sleep(STOPPING_POLL_INTERVAL);
344 continue;
345 }
346 yield* forget(scope.systemctl, tracked.unit);
347 if (status === "gone") return false;
348 yield* Effect.logWarning("Agent scope was killed: out of memory", {
349 threadId,
350 unit: tracked.unit,
351 });
352 tracked.oomKilled = true;
353 return true;
354 }
355 return false;
356 },
357 );
358
359 const stop = Effect.fn("AgentScope.stop")(function* (unit: string) {
360 if (!unit.startsWith("t3code-") || !unit.endsWith(".scope")) return;
361 const scope = yield* systemd;
362 if (scope === undefined) return;
363 const state = yield* readScope(scope.systemctl, unit);
364 if (
365 state === undefined ||
366 state.loadState !== "loaded" ||
367 state.activeState !== "active" ||
368 state.result === "oom-kill" ||
369 state.populated === false
370 )
371 return;
372 // The kernel counts OOM kills before signaling the victim. Let OOMPolicy
373 // record the result before a manual stop can overwrite it; an unreadable
374 // counter cannot rule out that race. Older managers need us to stop it.
375 if (scope.supportsOomPolicy && (state.oomKills === undefined || state.oomKills > 0)) return;
376 const stopped = yield* run(scope.systemctl, ["--user", "stop", "--no-block", unit]);
377 if (stopped._tag === "None" || stopped.value.code !== 0) {
378 yield* Effect.logWarning("Could not stop an exited agent's scope", {
379 unit,
380 exitCode: stopped._tag === "Some" ? stopped.value.code : undefined,
381 });
382 }
383 });
384
385 const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) {
386 const tracked = scopeByThread.get(threadId);
387 if (tracked === undefined || tracked.shared) return false;
388 const scope = yield* systemd;
389 if (scope === undefined) return false;
390 const shown = yield* run(scope.systemctl, [
391 "--user",
392 "show",
393 tracked.unit,
394 "--property=ActiveState,ControlGroup",
395 ]);
396 if (shown._tag === "None" || shown.value.code !== 0) return false;
397 const values = parseKeyValues(shown.value.stdout);
398 if (values.get("ActiveState") !== "active") return false;
399 const controlGroup = values.get("ControlGroup") ?? "";
400 // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback.
401 const killedByCgroup =
402 controlGroup.length > 0 &&
403 (yield* fileSystem
404 .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1")
405 .pipe(
406 Effect.as(true),
407 Effect.orElseSucceed(() => false),
408 ));
409 if (killedByCgroup) return true;
410 const killed = yield* run(scope.systemctl, [
411 "--user",
412 "kill",
413 "--signal=SIGKILL",
414 tracked.unit,
415 ]);
416 return killed._tag === "Some" && killed.value.code === 0;
417 });
418
419 const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
420 if (!scopeByThread.has(threadId)) return;
421 const scope = yield* systemd;
422 if (scope !== undefined) yield* track(scope.systemctl, threadId, undefined);
423 });
424
425 return {
426 wrap: Effect.fn("AgentScope.wrap")(function* ({
427 command,
428 args,
429 name,
430 threadId,
431 env,
432 survivesRestart,
433 shared,
434 }) {
435 const resolved = yield* resolve(command, env ?? hostEnvironment);
436 if (resolved === null) return { command, args };
437 const scope = yield* systemd;
438 if (scope === undefined) return agentScopeCommand({ command: resolved, args });
439 yield* scope.applyMemoryLimits;
440 const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`;
441 // A continuing shared scope cannot attribute a tool's OOM to one thread.
442 // Clear the launch thread's previous scope instead.
443 const continuing = shared === true && scope.supportsOomPolicy;
444 const owned = threadId !== undefined && !continuing;
445 if (threadId !== undefined)
446 yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true);
447 const systemdScope: AgentSystemdScope = {
448 systemdRun: scope.systemdRun,
449 runtimeDir: scope.runtimeDir,
450 slice: survivesRestart === true ? AGENT_SLICE : ownedSlice,
451 unit,
452 // Without an owning thread nothing reads the result, so let systemd
453 // unload the unit even when it fails.
454 options: [
455 ...SCOPE_OPTIONS,
456 ...(scope.supportsOomPolicy
457 ? [`--property=OOMPolicy=${continuing ? "continue" : "stop"}`]
458 : []),
459 ...(owned ? [] : ["--collect"]),
460 ],
461 };
462 return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit };
463 }),
464 oomKilled,
465 stop,
466 kill,
467 clear,
468 } satisfies AgentScopeShape;
469 });
470
471 export const layer = Layer.effect(AgentScope, make).pipe(
472 Layer.provide(ProcessRunner.layer),
473 Layer.provideMerge(ProviderProcesses.layer),
474 );