The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 16:00 UTC
fix(server): pause service restart when disk is full (#25924)
Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>
gyurix committed
Nov 14, 2026 at 11:24 UTC
9ff435b060f3a73579ac599c80c00616ce760800
2 files changed
+280
-2
apps/server/src/serviceLauncher.test.ts
+206
@@ -201,6 +201,212 @@ const fakeServiceUnit = (
201
});
202
203
it.layer(NodeServices.layer)("service state persistence", (it) => {
204
+ it.effect.each([
205
+ "startup",
206
+ "log write fails",
207
+ "home probe error",
208
+ "database probe error",
209
+ "pending update",
210
+ "update handoff",
211
+ "trial exit",
212
+ "failed trial start",
213
+ "stop",
214
+ ] as const)("waits for disk recovery during %s without losing the update lifecycle", (scenario) =>
215
+ Effect.gen(function* () {
216
+ const fs = yield* FileSystem.FileSystem;
217
+ const path = yield* Path.Path;
218
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-disk-" });
219
+ const statePath = path.join(root, "runtime", "service-state.json");
220
+ const databasePath = path.join(root, "userdata", "statev2.sqlite");
221
+ let backupPath = path.join(root, "runtime", "db-backup", "disk-update");
222
+ const pending = {
223
+ id: "disk-update",
224
+ fromVersion: "1.0.0",
225
+ targetVersion: "1.1.0",
226
+ dbPath: databasePath,
227
+ status: "pending" as const,
228
+ };
229
+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
230
+ yield* fs.writeFileString(databasePath, "before trial");
231
+ const childSource = `
232
+const fs = require("node:fs");
233
+const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
234
+if (context.update?.status === "pending") {
235
+ if (context.childVersion !== context.update.targetVersion) throw new Error("wrong trial version");
236
+ if (fs.readFileSync(${JSON.stringify(path.dirname(backupPath))} + "/" + context.update.id + "/database", "utf8") !== "before trial") {
237
+ throw new Error("missing original backup");
238
+ }
239
+ if (${JSON.stringify(scenario)} === "trial exit") {
240
+ fs.writeFileSync(context.update.dbPath, "migrated");
241
+ process.on("message", () => process.exit(1));
242
+ process.send("migrated");
243
+ } else {
244
+ process.send({ type: "prepared", updateId: context.update.id });
245
+ process.on("message", (message) => {
246
+ if (message.type === "committed") process.send("ready");
247
+ });
248
+ }
249
+} else if (${JSON.stringify(scenario)} === "update handoff" && context.update === undefined) {
250
+ process.on("message", (message) => {
251
+ if (message.type === "update-accepted") process.send("exiting");
252
+ if (message === "exit") process.exit(0);
253
+ });
254
+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} });
255
+} else {
256
+ process.send("ready");
257
+}
258
+setInterval(() => {}, 1_000);
259
+`;
260
+ for (const version of ["1.0.0", "1.1.0"]) {
261
+ yield* writeFakeRuntime(
262
+ fs,
263
+ path,
264
+ path.join(root, "runtime", "versions", version),
265
+ childSource,
266
+ );
267
+ }
268
+ yield* Effect.promise(() =>
269
+ writeServiceState(statePath, {
270
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
271
+ activeVersion: "1.0.0",
272
+ ...([
273
+ "startup",
274
+ "log write fails",
275
+ "home probe error",
276
+ "database probe error",
277
+ "update handoff",
278
+ "stop",
279
+ ].includes(scenario)
280
+ ? {}
281
+ : { update: pending }),
282
+ }),
283
+ );
284
+ let low = [
285
+ "startup",
286
+ "log write fails",
287
+ "home probe error",
288
+ "database probe error",
289
+ "pending update",
290
+ "stop",
291
+ ].includes(scenario);
292
+ let recovered = false;
293
+ const blocked = Promise.withResolvers<void>();
294
+ const ready = Promise.withResolvers<void>();
295
+ const probing = Promise.withResolvers<void>();
296
+ const capacityResult = Promise.withResolvers<void>();
297
+ const actualStatfs = NodeFSP.statfs;
298
+ const stats = yield* Effect.promise(() => actualStatfs(root));
299
+ yield* Effect.acquireRelease(
300
+ Effect.sync(() => {
301
+ vi.useFakeTimers({
302
+ toFake: ["setInterval", "clearInterval", "setTimeout", "clearTimeout"],
303
+ });
304
+ const capacity = vi.spyOn(NodeFSP, "statfs").mockImplementation(async (directory) => {
305
+ if (scenario === "stop" && directory === root) {
306
+ probing.resolve();
307
+ await capacityResult.promise;
308
+ }
309
+ if (
310
+ (scenario === "home probe error" && directory === root) ||
311
+ (scenario === "database probe error" && directory === path.dirname(databasePath))
312
+ ) {
313
+ throw new Error("statfs unavailable");
314
+ }
315
+ if (scenario === "failed trial start" && !recovered && NodeFS.existsSync(backupPath))
316
+ low = true;
317
+ return { ...stats, bavail: low ? 0 : 1024 ** 4, bsize: 1 };
318
+ });
319
+ const log = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
320
+ if (String(chunk).includes("Waiting for free space")) {
321
+ blocked.resolve();
322
+ if (scenario === "log write fails") {
323
+ throw Object.assign(new Error("no space left"), { code: "ENOSPC" });
324
+ }
325
+ }
326
+ return true;
327
+ });
328
+ spawnHook.current = (child) =>
329
+ child.on("message", (message) => {
330
+ if (message === "ready") ready.resolve();
331
+ if (message === "migrated" || message === "exiting") {
332
+ low = true;
333
+ child.send("exit");
334
+ }
335
+ });
336
+ return { capacity, log };
337
+ }),
338
+ ({ capacity, log }) =>
339
+ Effect.sync(() => {
340
+ capacity.mockRestore();
341
+ log.mockRestore();
342
+ spawnHook.current = undefined;
343
+ vi.useRealTimers();
344
+ }),
345
+ );
346
+ const beforeSpawns = spawnedChildren.length;
347
+ const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
348
+ const running = launcher.run();
349
+ yield* Effect.addFinalizer(() =>
350
+ Effect.promise(async () => {
351
+ await launcher.stop("SIGTERM");
352
+ await running;
353
+ }),
354
+ );
355
+ if (scenario === "stop") {
356
+ yield* Effect.promise(() => probing.promise);
357
+ const stopping = launcher.stop("SIGTERM");
358
+ capacityResult.resolve();
359
+ yield* Effect.promise(() => stopping);
360
+ yield* Effect.promise(() => running);
361
+ assert.equal(spawnedChildren.length, beforeSpawns);
362
+ assert.equal(vi.getTimerCount(), 0);
363
+ return;
364
+ }
365
+ yield* Effect.promise(() => Promise.race([blocked.promise, ready.promise, running]));
366
+ assert.equal(
367
+ spawnedChildren.length - beforeSpawns,
368
+ ["trial exit", "update handoff"].includes(scenario) ? 1 : 0,
369
+ );
370
+ if (scenario === "update handoff") {
371
+ const state = yield* Effect.promise(() => readServiceState(statePath));
372
+ backupPath = path.join(root, "runtime", "db-backup", state.update!.id);
373
+ }
374
+ assert.equal(
375
+ yield* fs.exists(backupPath),
376
+ scenario === "trial exit" || scenario === "failed trial start",
377
+ );
378
+ if (scenario === "trial exit")
379
+ assert.equal(yield* fs.readFileString(databasePath), "migrated");
380
+ if (scenario === "failed trial start") {
381
+ yield* fs.remove(path.join(root, "runtime", "versions", "1.1.0"), { recursive: true });
382
+ }
383
+ low = false;
384
+ recovered = true;
385
+ yield* Effect.promise(() => vi.advanceTimersByTimeAsync(30_000));
386
+ yield* Effect.promise(() => Promise.race([ready.promise, running]));
387
+ const state = yield* Effect.promise(() => readServiceState(statePath));
388
+ assert.equal(
389
+ state.activeVersion,
390
+ ["pending update", "update handoff"].includes(scenario) ? "1.1.0" : "1.0.0",
391
+ );
392
+ assert.equal(
393
+ state.update?.status,
394
+ {
395
+ startup: undefined,
396
+ "log write fails": undefined,
397
+ "home probe error": undefined,
398
+ "database probe error": undefined,
399
+ "pending update": "committed",
400
+ "update handoff": "committed",
401
+ "trial exit": "rolled-back",
402
+ "failed trial start": "failed",
403
+ }[scenario],
404
+ );
405
+ assert.equal(yield* fs.readFileString(databasePath), "before trial");
406
+ assert.isFalse(yield* fs.exists(backupPath));
407
+ }),
408
+ );
409
+
410
it.effect(
411
"loads optional service.env literally without changing inherited or managed variables",
412
() =>
apps/server/src/serviceLauncher.ts
+74
-2
@@ -50,6 +50,9 @@ const HANDOFF_DELAY_MS = 2_000;
50
*/
51
const PREPARED_TIMEOUT_MS = 30 * 60_000;
52
const TERMINATE_GRACE_MS = 5_000;
53
+// Keep SQLite startup and journaling from turning a full filesystem into a restart loop.
54
+const MIN_SERVICE_START_FREE_BYTES = 64 * 1024 ** 2;
55
+const DISK_SPACE_RETRY_MS = 30_000;
56
// A replacement started by a handoff waits up to two minutes for the stopping launcher.
57
const LOCK_RETRY_MS = 500;
58
const LOCK_ATTEMPTS = 240;
@@ -928,9 +931,12 @@ export class Launcher {
931
return terminateChild(child, signal, this.#stopChildOverIpc);
932
}
933
934
+ // Diagnostics are best-effort: a full log filesystem must not stop the launcher waiting for space.
935
#log(line: string): void {
932
- if (this.#output === undefined) process.stderr.write(line);
933
- else NodeFS.writeSync(this.#output, line);
936
+ try {
937
+ if (this.#output === undefined) process.stderr.write(line);
938
+ else NodeFS.writeSync(this.#output, line);
939
+ } catch {}
940
}
941
942
#enqueue(transition: () => Promise<void>): void {
@@ -983,6 +989,53 @@ export class Launcher {
989
this.#timer = undefined;
990
}
991
992
+ async #serviceStartDiskSpaceBlocker(): Promise<string | undefined> {
993
+ const databasePath =
994
+ this.#state.update?.status === "pending"
995
+ ? this.#state.update.dbPath
996
+ : NodePath.join(this.#baseDir, "userdata", "statev2.sqlite");
997
+ const resolvedDatabasePath = await restoreTarget(databasePath).catch(() => databasePath);
998
+ const freeBytes = async (directory: string): Promise<number> => {
999
+ try {
1000
+ return await filesystemBackupSpaceProbe.readFreeBytes(directory);
1001
+ } catch (cause) {
1002
+ const parent = NodePath.dirname(directory);
1003
+ if ((cause as NodeJS.ErrnoException).code === "ENOENT" && parent !== directory) {
1004
+ return freeBytes(parent);
1005
+ }
1006
+ return Number.POSITIVE_INFINITY;
1007
+ }
1008
+ };
1009
+ const available = await Promise.all([
1010
+ freeBytes(this.#baseDir),
1011
+ freeBytes(NodePath.dirname(resolvedDatabasePath)),
1012
+ ]);
1013
+ const free = Math.min(...available);
1014
+ return free < MIN_SERVICE_START_FREE_BYTES
1015
+ ? `T3 Code needs at least ${MIN_SERVICE_START_FREE_BYTES} bytes free before starting the server; ${free} bytes available.`
1016
+ : undefined;
1017
+ }
1018
+
1019
+ async #deferForDiskSpace(retry: () => Promise<void>): Promise<boolean> {
1020
+ const reason = await this.#serviceStartDiskSpaceBlocker();
1021
+ if (reason === undefined) return false;
1022
+ // A stop during the probe must not arm a retry that outlives it.
1023
+ if (this.#stopRequested) return true;
1024
+ this.#clearTimer();
1025
+ this.#log(`[service-launcher] ${reason} Waiting for free space before retrying.\n`);
1026
+ this.#timer = setInterval(
1027
+ () =>
1028
+ this.#enqueue(async () => {
1029
+ if (this.#child !== null || this.#stopping) return;
1030
+ if ((await this.#serviceStartDiskSpaceBlocker()) !== undefined) return;
1031
+ this.#clearTimer();
1032
+ await retry();
1033
+ }),
1034
+ DISK_SPACE_RETRY_MS,
1035
+ );
1036
+ return true;
1037
+ }
1038
+
1039
async #selectState(next: ServiceState): Promise<void> {
1040
const release = await acquireServiceSelectionLock(this.#statePath);
1041
try {
@@ -1057,6 +1110,7 @@ export class Launcher {
1110
}
1111
1112
async #startTrial(pending: PendingServiceUpdate): Promise<void> {
1113
+ if (await this.#deferForDiskSpace(() => this.#startTrial(pending))) return;
1114
try {
1115
await backupDatabaseOnce(this.#baseDir, pending, this.#stopSignal.signal);
1116
} catch {
@@ -1075,6 +1129,14 @@ export class Launcher {
1129
1130
async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise<void> {
1131
if (this.#stopping) return;
1132
+ if (
1133
+ await this.#deferForDiskSpace(() =>
1134
+ role === "trial" && update?.status === "pending"
1135
+ ? this.#startTrial(update)
1136
+ : this.#startChild(version, role, update),
1137
+ )
1138
+ )
1139
+ return;
1140
if (!(await runtimeExists(this.#baseDir, version))) {
1141
throw new Error(`Selected t3@${version} runtime is missing or incomplete.`);
1142
}
@@ -1333,6 +1395,12 @@ export class Launcher {
1395
await this.#startTrial(pending);
1396
return;
1397
}
1398
+ if (
1399
+ await this.#deferForDiskSpace(() =>
1400
+ this.#startChild(child.version, "active", this.#state.update),
1401
+ )
1402
+ )
1403
+ return;
1404
throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
1405
}
1406
@@ -1398,6 +1466,8 @@ export class Launcher {
1466
this.#child = null;
1467
await this.#terminate(child.process);
1468
}
1469
+ if (await this.#deferForDiskSpace(() => this.#returnToPrevious(pending, status, reason)))
1470
+ return;
1471
await restoreDatabaseBackup(this.#baseDir, pending);
1472
const outcome = terminalUpdate({ pending, status, reason });
1473
const next: ServiceState = {
@@ -1432,6 +1502,8 @@ export async function main(options: LauncherOptions = {}): Promise<void> {
1502
// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
1503
const windows = process.platform === "win32";
1504
let logTimer: NodeJS.Timeout | undefined;
1505
+ // A failed stderr write (ENOSPC on a redirected log) is emitted asynchronously; without a listener it is fatal.
1506
+ process.stderr.on("error", () => undefined);
1507
try {
1508
// What the service manager started the launcher with. It locates the unit; service.env
1509
// changes only what the server gets.