@devries-cameron20 / t3code-1 / commits / 9ff435b060
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): pause service restart when disk is full (#25924)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

gyurix committed Nov 14, 2026 at 11:24 UTC 9ff435b060f3a73579ac599c80c00616ce760800
2 files changed +280 -2
apps/server/src/serviceLauncher.test.ts
+206
@@ -201,6 +201,212 @@ const fakeServiceUnit = (
201 });
202
203 it.layer(NodeServices.layer)("service state persistence", (it) => {
204 + it.effect.each([
205 + "startup",
206 + "log write fails",
207 + "home probe error",
208 + "database probe error",
209 + "pending update",
210 + "update handoff",
211 + "trial exit",
212 + "failed trial start",
213 + "stop",
214 + ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) =>
215 + Effect.gen(function* () {
216 + const fs = yield* FileSystem.FileSystem;
217 + const path = yield* Path.Path;
218 + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" });
219 + const statePath = path.join(root, "runtime", "service-state.json");
220 + const databasePath = path.join(root, "userdata", "statev2.sqlite");
221 + let backupPath = path.join(root, "runtime", "db-backup", "disk-update");
222 + const pending = {
223 + id: "disk-update",
224 + fromVersion: "1.0.0",
225 + targetVersion: "1.1.0",
226 + dbPath: databasePath,
227 + status: "pending" as const,
228 + };
229 + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
230 + yield* fs.writeFileString(databasePath, "before trial");
231 + const childSource = `
232 +const fs = require("node:fs");
233 +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
234 +if (context.update?.status === "pending") {
235 + if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version");
236 + if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") {
237 + throw new Error("missing original backup");
238 + }
239 + if (${JSON.stringify(scenario)} === "trial exit") {
240 + fs.writeFileSync(context.update.dbPath, "migrated");
241 + process.on("message", () => process.exit(1));
242 + process.send("migrated");
243 + } else {
244 + process.send({ type: "prepared", updateId: context.update.id });
245 + process.on("message", (message) => {
246 + if (message.type === "committed") process.send("ready");
247 + });
248 + }
249 +} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) {
250 + process.on("message", (message) => {
251 + if (message.type === "update-accepted") process.send("exiting");
252 + if (message === "exit") process.exit(0);
253 + });
254 + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
255 +} else {
256 + process.send("ready");
257 +}
258 +setInterval(() => {}, 1_000);
259 +`;
260 + for (const version of ["1.0.0", "1.1.0"]) {
261 + yield* writeFakeRuntime(
262 + fs,
263 + path,
264 + path.join(root, "runtime", "versions", version),
265 + childSource,
266 + );
267 + }
268 + yield* Effect.promise(() =>
269 + writeServiceState(statePath, {
270 + protocol: SERVICE_LAUNCHER_PROTOCOL,
271 + activeVersion: "1.0.0",
272 + ...([
273 + "startup",
274 + "log write fails",
275 + "home probe error",
276 + "database probe error",
277 + "update handoff",
278 + "stop",
279 + ].includes(scenario)
280 + ? {}
281 + : { update: pending }),
282 + }),
283 + );
284 + let low = [
285 + "startup",
286 + "log write fails",
287 + "home probe error",
288 + "database probe error",
289 + "pending update",
290 + "stop",
291 + ].includes(scenario);
292 + let recovered = false;
293 + const blocked = Promise.withResolvers<void>();
294 + const ready = Promise.withResolvers<void>();
295 + const probing = Promise.withResolvers<void>();
296 + const capacityResult = Promise.withResolvers<void>();
297 + const actualStatfs = NodeFSP.statfs;
298 + const stats = yield* Effect.promise(() => actualStatfs(root));
299 + yield* Effect.acquireRelease(
300 + Effect.sync(() => {
301 + vi.useFakeTimers({
302 + toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"],
303 + });
304 + const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => {
305 + if (scenario === "stop" && directory === root) {
306 + probing.resolve();
307 + await capacityResult.promise;
308 + }
309 + if (
310 + (scenario === "home probe error" && directory === root) ||
311 + (scenario === "database probe error" && directory === path.dirname(databasePath))
312 + ) {
313 + throw new Error("statfs unavailable");
314 + }
315 + if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath))
316 + low = true;
317 + return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 };
318 + });
319 + const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
320 + if (String(chunk).includes("Waiting for free space")) {
321 + blocked.resolve();
322 + if (scenario === "log write fails") {
323 + throw Object.assign(new Error("no space left"), { code: "ENOSPC" });
324 + }
325 + }
326 + return true;
327 + });
328 + spawnHook.current = (child) =>
329 + child.on("message", (message) => {
330 + if (message === "ready") ready.resolve();
331 + if (message === "migrated" || message === "exiting") {
332 + low = true;
333 + child.send("exit");
334 + }
335 + });
336 + return { capacity, log };
337 + }),
338 + ({ capacity, log }) =>
339 + Effect.sync(() => {
340 + capacity.mockRestore();
341 + log.mockRestore();
342 + spawnHook.current = undefined;
343 + vi.useRealTimers();
344 + }),
345 + );
346 + const beforeSpawns = spawnedChildren.length;
347 + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
348 + const running = launcher.run();
349 + yield* Effect.addFinalizer(() =>
350 + Effect.promise(async () => {
351 + await launcher.stop("SIGTERM");
352 + await running;
353 + }),
354 + );
355 + if (scenario === "stop") {
356 + yield* Effect.promise(() => probing.promise);
357 + const stopping = launcher.stop("SIGTERM");
358 + capacityResult.resolve();
359 + yield* Effect.promise(() => stopping);
360 + yield* Effect.promise(() => running);
361 + assert.equal(spawnedChildren.length, beforeSpawns);
362 + assert.equal(vi.getTimerCount(), 0);
363 + return;
364 + }
365 + yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running]));
366 + assert.equal(
367 + spawnedChildren.length - beforeSpawns,
368 + ["trial exit", "update handoff"].includes(scenario) ? 1 : 0,
369 + );
370 + if (scenario === "update handoff") {
371 + const state = yield* Effect.promise(() => readServiceState(statePath));
372 + backupPath = path.join(root, "runtime", "db-backup", state.update!.id);
373 + }
374 + assert.equal(
375 + yield* fs.exists(backupPath),
376 + scenario === "trial exit" || scenario === "failed trial start",
377 + );
378 + if (scenario === "trial exit")
379 + assert.equal(yield* fs.readFileString(databasePath), "migrated");
380 + if (scenario === "failed trial start") {
381 + yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true });
382 + }
383 + low = false;
384 + recovered = true;
385 + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000));
386 + yield* Effect.promise(() => Promise.race([ready.promise, running]));
387 + const state = yield* Effect.promise(() => readServiceState(statePath));
388 + assert.equal(
389 + state.activeVersion,
390 + ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0",
391 + );
392 + assert.equal(
393 + state.update?.status,
394 + {
395 + startup: undefined,
396 + "log write fails": undefined,
397 + "home probe error": undefined,
398 + "database probe error": undefined,
399 + "pending update": "committed",
400 + "update handoff": "committed",
401 + "trial exit": "rolled-back",
402 + "failed trial start": "failed",
403 + }[scenario],
404 + );
405 + assert.equal(yield* fs.readFileString(databasePath), "before trial");
406 + assert.isFalse(yield* fs.exists(backupPath));
407 + }),
408 + );
409 +
410 it.effect(
411 "loads optional service.env literally without changing inherited or managed variables",
412 () =>
apps/server/src/serviceLauncher.ts
+74 -2
@@ -50,6 +50,9 @@ const HANDOFF_DELAY_MS = 2_000;
50 */
51 const PREPARED_TIMEOUT_MS = 30 * 60_000;
52 const TERMINATE_GRACE_MS = 5_000;
53 +// Keep SQLite startup and journaling from turning a full filesystem into a restart loop.
54 +const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2;
55 +const DISK_SPACE_RETRY_MS = 30_000;
56 // A replacement started by a handoff waits up to two minutes for the stopping launcher.
57 const LOCK_RETRY_MS = 500;
58 const LOCK_ATTEMPTS = 240;
@@ -928,9 +931,12 @@ export class Launcher {
931 return terminateChild(child, signal, this.#stopChildOverIpc);
932 }
933
934 + // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space.
935 #log(line: string): void {
932 - if (this.#output === undefined) process.stderr.write(line);
933 - else NodeFS.writeSync(this.#output, line);
936 + try {
937 + if (this.#output === undefined) process.stderr.write(line);
938 + else NodeFS.writeSync(this.#output, line);
939 + } catch {}
940 }
941
942 #enqueue(transition: () => Promise<void>): void {
@@ -983,6 +989,53 @@ export class Launcher {
989 this.#timer = undefined;
990 }
991
992 + async #serviceStartDiskSpaceBlocker(): Promise<string | undefined> {
993 + const databasePath =
994 + this.#state.update?.status === "pending"
995 + ? this.#state.update.dbPath
996 + : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite");
997 + const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath);
998 + const freeBytes = async (directory: string): Promise<number> => {
999 + try {
1000 + return await filesystemBackupSpaceProbe.readFreeBytes(directory);
1001 + } catch (cause) {
1002 + const parent = NodePath.dirname(directory);
1003 + if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) {
1004 + return freeBytes(parent);
1005 + }
1006 + return Number.POSITIVE_INFINITY;
1007 + }
1008 + };
1009 + const available = await Promise.all([
1010 + freeBytes(this.#baseDir),
1011 + freeBytes(NodePath.dirname(resolvedDatabasePath)),
1012 + ]);
1013 + const free = Math.min(...available);
1014 + return free < MIN_SERVICE_START_FREE_BYTES
1015 + ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.`
1016 + : undefined;
1017 + }
1018 +
1019 + async #deferForDiskSpace(retry: () => Promise<void>): Promise<boolean> {
1020 + const reason = await this.#serviceStartDiskSpaceBlocker();
1021 + if (reason === undefined) return false;
1022 + // A stop during the probe must not arm a retry that outlives it.
1023 + if (this.#stopRequested) return true;
1024 + this.#clearTimer();
1025 + this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`);
1026 + this.#timer = setInterval(
1027 + () =>
1028 + this.#enqueue(async () => {
1029 + if (this.#child !== null || this.#stopping) return;
1030 + if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return;
1031 + this.#clearTimer();
1032 + await retry();
1033 + }),
1034 + DISK_SPACE_RETRY_MS,
1035 + );
1036 + return true;
1037 + }
1038 +
1039 async #selectState(next: ServiceState): Promise<void> {
1040 const release = await acquireServiceSelectionLock(this.#statePath);
1041 try {
@@ -1057,6 +1110,7 @@ export class Launcher {
1110 }
1111
1112 async #startTrial(pending: PendingServiceUpdate): Promise<void> {
1113 + if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return;
1114 try {
1115 await backupDatabaseOnce(this.#baseDir, pending, this.#stopSignal.signal);
1116 } catch {
@@ -1075,6 +1129,14 @@ export class Launcher {
1129
1130 async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
1131 if (this.#stopping) return;
1132 + if (
1133 + await this.#deferForDiskSpace(() =>
1134 + role === "trial" && update?.status === "pending"
1135 + ? this.#startTrial(update)
1136 + : this.#startChild(version, role, update),
1137 + )
1138 + )
1139 + return;
1140 if (!(await runtimeExists(this.#baseDir, version))) {
1141 throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
1142 }
@@ -1333,6 +1395,12 @@ export class Launcher {
1395 await this.#startTrial(pending);
1396 return;
1397 }
1398 + if (
1399 + await this.#deferForDiskSpace(() =>
1400 + this.#startChild(child.version, "active", this.#state.update),
1401 + )
1402 + )
1403 + return;
1404 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1405 }
1406
@@ -1398,6 +1466,8 @@ export class Launcher {
1466 this.#child = null;
1467 await this.#terminate(child.process);
1468 }
1469 + if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason)))
1470 + return;
1471 await restoreDatabaseBackup(this.#baseDir, pending);
1472 const outcome = terminalUpdate({ pending, status, reason });
1473 const next: ServiceState = {
@@ -1432,6 +1502,8 @@ export async function main(options: LauncherOptions = {}): Promise<void> {
1502 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1503 const windows = process.platform === "win32";
1504 let logTimer: NodeJS.Timeout | undefined;
1505 + // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal.
1506 + process.stderr.on("error", () => undefined);
1507 try {
1508 // What the service manager started the launcher with. It locates the unit; service.env
1509 // changes only what the server gets.