The best way to control Claude Code, Codex and more from any computer or phone
fix(server): updates copy the database while the server still runs when it cannot be cloned (#33058)
Co-authored-by: Ajraba04 <143622128+Ajraba04@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Bin Jamshid committed
Nov 11, 2026 at 10:14 UTC
034f3ba918c986da6abe9cdb4dc9715b7150dc1a
2 files changed
+216
-3
apps/server/src/serviceLauncher.test.ts
+121
-1
@@ -2,6 +2,7 @@
2
import type * as NodeChildProcess from "node:child_process";
3
// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4
import * as NodeFS from "node:fs";
5
+import * as NodeSqlite from "node:sqlite";
6
7
import * as NodeServices from "@effect/platform-node/NodeServices";
8
import { assert, it } from "@effect/vitest";
@@ -464,6 +465,125 @@ if (context.update?.status === "pending") {
465
}),
466
);
467
468
+ it.effect.each(["wal", "delete"] as const)(
469
+ "backs up a %s-mode database (online when WAL) and restores its final state",
470
+ (journalMode) =>
471
+ Effect.gen(function* () {
472
+ const fs = yield* FileSystem.FileSystem;
473
+ const path = yield* Path.Path;
474
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" });
475
+ const statePath = path.join(root, "runtime", "service-state.json");
476
+ const databasePath = path.join(root, "userdata", "state.sqlite");
477
+ const observationsPath = path.join(root, "observations.json");
478
+ yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
479
+ // SQLite keeps -wal beside the symlink's target, so the backup must restore it there.
480
+ yield* fs.symlink("actual.sqlite", databasePath);
481
+ const seed = new NodeSqlite.DatabaseSync(databasePath);
482
+ seed.exec(`
483
+ PRAGMA journal_mode = ${journalMode};
484
+ CREATE TABLE events (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);
485
+ WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 500)
486
+ INSERT INTO events (payload) SELECT 'before the update ' || i FROM n;
487
+ `);
488
+ seed.close();
489
+
490
+ const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode });
491
+ const childSource = `
492
+import { readdirSync, writeFileSync } from "node:fs";
493
+import { DatabaseSync } from "node:sqlite";
494
+const paths = ${encoded};
495
+const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
496
+if (context.update?.status === "pending") {
497
+ writeFileSync(paths.observationsPath + ".trial", JSON.stringify(
498
+ readdirSync(paths.root + "/runtime/db-backup/" + context.update.id).sort(),
499
+ ));
500
+ const trial = new DatabaseSync(context.update.dbPath);
501
+ trial.exec("PRAGMA journal_mode = " + paths.journalMode + "; INSERT INTO events (payload) VALUES ('trial');");
502
+ trial.close();
503
+ process.exit(1);
504
+} else if (context.update === undefined) {
505
+ const db = new DatabaseSync(paths.databasePath);
506
+ db.exec("PRAGMA busy_timeout = 5000;");
507
+ db.exec("PRAGMA journal_mode = " + paths.journalMode + ";");
508
+ const insert = db.prepare("INSERT INTO events (payload) VALUES (?)");
509
+ const writer = setInterval(() => insert.run("written while serving"), 1);
510
+ process.once("SIGTERM", () => {
511
+ clearInterval(writer);
512
+ insert.run("last write before stopping");
513
+ const backupRoot = paths.root + "/runtime/db-backup";
514
+ writeFileSync(paths.observationsPath, JSON.stringify({
515
+ backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)),
516
+ rows: db.prepare("SELECT count(*) AS n FROM events").get().n,
517
+ }));
518
+ db.close();
519
+ process.exit(0);
520
+ });
521
+ process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
522
+} else {
523
+ process.exit(0);
524
+}
525
+`;
526
+ for (const version of ["1.0.0", "1.1.0"]) {
527
+ yield* writeFakeRuntime(
528
+ fs,
529
+ path,
530
+ path.join(root, "runtime", "versions", version),
531
+ childSource,
532
+ );
533
+ }
534
+ yield* Effect.promise(() =>
535
+ writeServiceState(statePath, {
536
+ protocol: SERVICE_LAUNCHER_PROTOCOL,
537
+ activeVersion: "1.0.0",
538
+ }),
539
+ );
540
+
541
+ const launcher = new Launcher(
542
+ root,
543
+ yield* Effect.promise(() => readServiceState(statePath)),
544
+ {
545
+ backupSpaceProbe: { canClone: async () => false },
546
+ },
547
+ );
548
+ yield* Effect.promise(() =>
549
+ launcher.run().then(
550
+ () => Promise.reject(new Error("launcher unexpectedly completed")),
551
+ () => Promise.resolve(),
552
+ ),
553
+ );
554
+
555
+ const state = yield* Effect.promise(() => readServiceState(statePath));
556
+ assert.equal(state.update?.status, "rolled-back");
557
+ const observed = JSON.parse(yield* fs.readFileString(observationsPath));
558
+ // A WAL database's main file is copied before the server is asked to stop; others fall back offline.
559
+ assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []);
560
+ assert.isAbove(observed.rows, 501);
561
+ // Only the online copy holds no -shm: the offline fallback copies the one the server left.
562
+ assert.deepEqual(
563
+ JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)),
564
+ journalMode === "wal" ? ["database", "database-wal"] : ["database"],
565
+ );
566
+ const restored = new NodeSqlite.DatabaseSync(databasePath);
567
+ try {
568
+ assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok");
569
+ assert.equal(
570
+ restored.prepare("SELECT count(*) AS n FROM events").get()?.n,
571
+ observed.rows,
572
+ );
573
+ assert.equal(
574
+ restored
575
+ .prepare(
576
+ "SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'",
577
+ )
578
+ .get()?.n,
579
+ 1,
580
+ );
581
+ } finally {
582
+ restored.close();
583
+ }
584
+ }),
585
+ );
586
+
587
const requestRejectedUpdate = (backupSpaceProbe: Partial<BackupSpaceProbe>) =>
588
Effect.gen(function* () {
589
const fs = yield* FileSystem.FileSystem;
@@ -670,7 +790,7 @@ setInterval(() => {}, 1_000);
790
assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free");
791
const blocker = yield* check(8 * 1024 ** 3, false);
792
assert.include(blocker ?? "", "Not enough free disk space");
673
- assert.include(blocker ?? "", "10.5 GB needed, 8.0 GB free");
793
+ assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free");
794
assert.include(blocker ?? "", "The server was not stopped");
795
}),
796
);
apps/server/src/serviceLauncher.ts
+95
-2
@@ -162,6 +162,8 @@ const GIB = 1024 ** 3;
162
// Cloned backups still need headroom for trial writes and migrations to diverge.
163
const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
164
const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
165
+// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
166
+const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
167
168
async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
169
const stats: Array<NodeFS.Stats> = [];
@@ -242,7 +244,7 @@ export async function databaseBackupBlocker(
244
await removeAbandonedBackupCopies(backupRoot);
245
const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
246
const size = files.reduce((total, file) => total + file.size, 0);
245
- const fullCopyBytes = size * 1.05;
247
+ const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
248
if (free >= fullCopyBytes) return undefined;
249
const clonedBytes = Math.max(
250
CLONED_BACKUP_MIN_HEADROOM_BYTES,
@@ -254,6 +256,83 @@ export async function databaseBackupBlocker(
256
return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
257
}
258
259
+// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
260
+async function copyWithoutOpening(source: string, destination: string): Promise<void> {
261
+ // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
262
+ if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
263
+ if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
264
+}
265
+
266
+// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
267
+// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
268
+// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
269
+// a write-heavy server's new writes wait, uncommitted, until the server stops.
270
+async function startOnlineDatabaseBackup(
271
+ baseDir: string,
272
+ pending: PendingServiceUpdate,
273
+ probe: Partial<BackupSpaceProbe> = {},
274
+): Promise<(() => Promise<void>) | undefined> {
275
+ const backupDir = databaseBackupDir(baseDir, pending.id);
276
+ if (await pathExists(backupDir)) return undefined;
277
+ const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
278
+ if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
279
+ return undefined;
280
+ const NodeSqlite = await import("node:sqlite").catch(() => undefined);
281
+ if (NodeSqlite === undefined) return undefined;
282
+
283
+ const stagingDir = `${backupDir}.staging`;
284
+ const discardStaging = () =>
285
+ NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
286
+ const fallBack = async (cause: unknown) => {
287
+ await discardStaging();
288
+ process.stderr.write(
289
+ `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
290
+ );
291
+ return undefined;
292
+ };
293
+ let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
294
+ // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
295
+ let dbPath = pending.dbPath;
296
+ const release = () => {
297
+ try {
298
+ snapshot?.close();
299
+ } catch {}
300
+ snapshot = undefined;
301
+ };
302
+ try {
303
+ await discardStaging();
304
+ await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
305
+ dbPath = await NodeFSP.realpath(pending.dbPath);
306
+ snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
307
+ const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
308
+ if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
309
+ snapshot.exec("BEGIN");
310
+ snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
311
+ const mainCopy = databaseBackupFile(stagingDir, "");
312
+ await copyWithoutOpening(dbPath, mainCopy);
313
+ await syncFile(mainCopy);
314
+ } catch (cause) {
315
+ release();
316
+ return fallBack(cause);
317
+ }
318
+ return async () => {
319
+ try {
320
+ const wal = `${dbPath}-wal`;
321
+ if (!(await pathExists(wal)))
322
+ throw new Error("the WAL disappeared while the snapshot was open.");
323
+ const walCopy = databaseBackupFile(stagingDir, "-wal");
324
+ await copyWithoutOpening(wal, walCopy);
325
+ await syncFile(walCopy);
326
+ release();
327
+ await NodeFSP.rename(stagingDir, backupDir);
328
+ await syncDirectory(NodePath.dirname(backupDir));
329
+ } catch (cause) {
330
+ release();
331
+ await fallBack(cause);
332
+ }
333
+ };
334
+}
335
+
336
// Preserve completed backups; restarted trials may already have changed the live database.
337
async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
338
const backupDir = databaseBackupDir(baseDir, pending.id);
@@ -330,8 +409,13 @@ async function restoreDatabaseBackup(
409
410
await markDatabaseRestorePending(backupDir);
411
const directories = new Set([NodePath.dirname(pending.dbPath)]);
412
+ // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
413
+ const mainTarget = await restoreTarget(pending.dbPath);
414
for (const suffix of DB_FILE_SUFFIXES) {
334
- const livePath = `${pending.dbPath}${suffix}`;
415
+ const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
416
+ if (livePath !== `${pending.dbPath}${suffix}`) {
417
+ await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
418
+ }
419
const source = databaseBackupFile(backupDir, suffix);
420
if (await pathExists(source)) {
421
// Restore into the symlink target; cloning would replace the link.
@@ -708,8 +792,17 @@ export class Launcher {
792
return;
793
}
794
this.#timer = undefined;
795
+ // Anything thrown outside its own fallback still leaves the offline backup to run.
796
+ const finishOnlineBackup = await startOnlineDatabaseBackup(
797
+ this.#baseDir,
798
+ pending,
799
+ this.#backupSpaceProbe,
800
+ ).catch(() => undefined);
801
this.#child = null;
802
await terminateChild(child.process);
803
+ await finishOnlineBackup?.();
804
+ // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
805
+ if (this.#stopRequested) return;
806
await this.#startTrial(pending);
807
}
808