@devries-cameron20 / t3code Mirror · read-only / commits / 034f3ba918
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 23:30 UTC

fix(server): updates copy the database while the server still runs when it cannot be cloned (#33058)

Co-authored-by: Ajraba04 <143622128+Ajraba04@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Bin Jamshid committed Nov 11, 2026 at 10:14 UTC 034f3ba918c986da6abe9cdb4dc9715b7150dc1a
2 files changed +216 -3
apps/server/src/serviceLauncher.test.ts
+121 -1
@@ -2,6 +2,7 @@
2 import type * as NodeChildProcess from "node:child_process";
3 // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
4 import * as NodeFS from "node:fs";
5 +import * as NodeSqlite from "node:sqlite";
6
7 import * as NodeServices from "@effect/platform-node/NodeServices";
8 import { assert, it } from "@effect/vitest";
@@ -464,6 +465,125 @@ if (context.update?.status === "pending") {
465 }),
466 );
467
468 + it.effect.each(["wal", "delete"] as const)(
469 + "backs up a %s-mode database (online when WAL) and restores its final state",
470 + (journalMode) =>
471 + Effect.gen(function* () {
472 + const fs = yield* FileSystem.FileSystem;
473 + const path = yield* Path.Path;
474 + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" });
475 + const statePath = path.join(root, "runtime", "service-state.json");
476 + const databasePath = path.join(root, "userdata", "state.sqlite");
477 + const observationsPath = path.join(root, "observations.json");
478 + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
479 + // SQLite keeps -wal beside the symlink's target, so the backup must restore it there.
480 + yield* fs.symlink("actual.sqlite", databasePath);
481 + const seed = new NodeSqlite.DatabaseSync(databasePath);
482 + seed.exec(`
483 + PRAGMA journal_mode = ${journalMode};
484 + CREATE TABLE events (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);
485 + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 500)
486 + INSERT INTO events (payload) SELECT 'before the update ' || i FROM n;
487 + `);
488 + seed.close();
489 +
490 + const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode });
491 + const childSource = `
492 +import { readdirSync, writeFileSync } from "node:fs";
493 +import { DatabaseSync } from "node:sqlite";
494 +const paths = ${encoded};
495 +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
496 +if (context.update?.status === "pending") {
497 + writeFileSync(paths.observationsPath + ".trial", JSON.stringify(
498 + readdirSync(paths.root + "/runtime/db-backup/" + context.update.id).sort(),
499 + ));
500 + const trial = new DatabaseSync(context.update.dbPath);
501 + trial.exec("PRAGMA journal_mode = " + paths.journalMode + "; INSERT INTO events (payload) VALUES ('trial');");
502 + trial.close();
503 + process.exit(1);
504 +} else if (context.update === undefined) {
505 + const db = new DatabaseSync(paths.databasePath);
506 + db.exec("PRAGMA busy_timeout = 5000;");
507 + db.exec("PRAGMA journal_mode = " + paths.journalMode + ";");
508 + const insert = db.prepare("INSERT INTO events (payload) VALUES (?)");
509 + const writer = setInterval(() => insert.run("written while serving"), 1);
510 + process.once("SIGTERM", () => {
511 + clearInterval(writer);
512 + insert.run("last write before stopping");
513 + const backupRoot = paths.root + "/runtime/db-backup";
514 + writeFileSync(paths.observationsPath, JSON.stringify({
515 + backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)),
516 + rows: db.prepare("SELECT count(*) AS n FROM events").get().n,
517 + }));
518 + db.close();
519 + process.exit(0);
520 + });
521 + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
522 +} else {
523 + process.exit(0);
524 +}
525 +`;
526 + for (const version of ["1.0.0", "1.1.0"]) {
527 + yield* writeFakeRuntime(
528 + fs,
529 + path,
530 + path.join(root, "runtime", "versions", version),
531 + childSource,
532 + );
533 + }
534 + yield* Effect.promise(() =>
535 + writeServiceState(statePath, {
536 + protocol: SERVICE_LAUNCHER_PROTOCOL,
537 + activeVersion: "1.0.0",
538 + }),
539 + );
540 +
541 + const launcher = new Launcher(
542 + root,
543 + yield* Effect.promise(() => readServiceState(statePath)),
544 + {
545 + backupSpaceProbe: { canClone: async () => false },
546 + },
547 + );
548 + yield* Effect.promise(() =>
549 + launcher.run().then(
550 + () => Promise.reject(new Error("launcher unexpectedly completed")),
551 + () => Promise.resolve(),
552 + ),
553 + );
554 +
555 + const state = yield* Effect.promise(() => readServiceState(statePath));
556 + assert.equal(state.update?.status, "rolled-back");
557 + const observed = JSON.parse(yield* fs.readFileString(observationsPath));
558 + // A WAL database's main file is copied before the server is asked to stop; others fall back offline.
559 + assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []);
560 + assert.isAbove(observed.rows, 501);
561 + // Only the online copy holds no -shm: the offline fallback copies the one the server left.
562 + assert.deepEqual(
563 + JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)),
564 + journalMode === "wal" ? ["database", "database-wal"] : ["database"],
565 + );
566 + const restored = new NodeSqlite.DatabaseSync(databasePath);
567 + try {
568 + assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok");
569 + assert.equal(
570 + restored.prepare("SELECT count(*) AS n FROM events").get()?.n,
571 + observed.rows,
572 + );
573 + assert.equal(
574 + restored
575 + .prepare(
576 + "SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'",
577 + )
578 + .get()?.n,
579 + 1,
580 + );
581 + } finally {
582 + restored.close();
583 + }
584 + }),
585 + );
586 +
587 const requestRejectedUpdate = (backupSpaceProbe: Partial<BackupSpaceProbe>) =>
588 Effect.gen(function* () {
589 const fs = yield* FileSystem.FileSystem;
@@ -670,7 +790,7 @@ setInterval(() => {}, 1_000);
790 assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free");
791 const blocker = yield* check(8 * 1024 ** 3, false);
792 assert.include(blocker ?? "", "Not enough free disk space");
673 - assert.include(blocker ?? "", "10.5 GB needed, 8.0 GB free");
793 + assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free");
794 assert.include(blocker ?? "", "The server was not stopped");
795 }),
796 );
apps/server/src/serviceLauncher.ts
+95 -2
@@ -162,6 +162,8 @@ const GIB = 1024 ** 3;
162 // Cloned backups still need headroom for trial writes and migrations to diverge.
163 const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
164 const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
165 +// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
166 +const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
167
168 async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
169 const stats: Array<NodeFS.Stats> = [];
@@ -242,7 +244,7 @@ export async function databaseBackupBlocker(
244 await removeAbandonedBackupCopies(backupRoot);
245 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
246 const size = files.reduce((total, file) => total + file.size, 0);
245 - const fullCopyBytes = size * 1.05;
247 + const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
248 if (free >= fullCopyBytes) return undefined;
249 const clonedBytes = Math.max(
250 CLONED_BACKUP_MIN_HEADROOM_BYTES,
@@ -254,6 +256,83 @@ export async function databaseBackupBlocker(
256 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
257 }
258
259 +// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
260 +async function copyWithoutOpening(source: string, destination: string): Promise<void> {
261 + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
262 + if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
263 + if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
264 +}
265 +
266 +// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
267 +// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
268 +// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
269 +// a write-heavy server's new writes wait, uncommitted, until the server stops.
270 +async function startOnlineDatabaseBackup(
271 + baseDir: string,
272 + pending: PendingServiceUpdate,
273 + probe: Partial<BackupSpaceProbe> = {},
274 +): Promise<(() => Promise<void>) | undefined> {
275 + const backupDir = databaseBackupDir(baseDir, pending.id);
276 + if (await pathExists(backupDir)) return undefined;
277 + const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
278 + if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
279 + return undefined;
280 + const NodeSqlite = await import("node:sqlite").catch(() => undefined);
281 + if (NodeSqlite === undefined) return undefined;
282 +
283 + const stagingDir = `${backupDir}.staging`;
284 + const discardStaging = () =>
285 + NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
286 + const fallBack = async (cause: unknown) => {
287 + await discardStaging();
288 + process.stderr.write(
289 + `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
290 + );
291 + return undefined;
292 + };
293 + let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
294 + // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
295 + let dbPath = pending.dbPath;
296 + const release = () => {
297 + try {
298 + snapshot?.close();
299 + } catch {}
300 + snapshot = undefined;
301 + };
302 + try {
303 + await discardStaging();
304 + await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
305 + dbPath = await NodeFSP.realpath(pending.dbPath);
306 + snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
307 + const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
308 + if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
309 + snapshot.exec("BEGIN");
310 + snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
311 + const mainCopy = databaseBackupFile(stagingDir, "");
312 + await copyWithoutOpening(dbPath, mainCopy);
313 + await syncFile(mainCopy);
314 + } catch (cause) {
315 + release();
316 + return fallBack(cause);
317 + }
318 + return async () => {
319 + try {
320 + const wal = `${dbPath}-wal`;
321 + if (!(await pathExists(wal)))
322 + throw new Error("the WAL disappeared while the snapshot was open.");
323 + const walCopy = databaseBackupFile(stagingDir, "-wal");
324 + await copyWithoutOpening(wal, walCopy);
325 + await syncFile(walCopy);
326 + release();
327 + await NodeFSP.rename(stagingDir, backupDir);
328 + await syncDirectory(NodePath.dirname(backupDir));
329 + } catch (cause) {
330 + release();
331 + await fallBack(cause);
332 + }
333 + };
334 +}
335 +
336 // Preserve completed backups; restarted trials may already have changed the live database.
337 async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
338 const backupDir = databaseBackupDir(baseDir, pending.id);
@@ -330,8 +409,13 @@ async function restoreDatabaseBackup(
409
410 await markDatabaseRestorePending(backupDir);
411 const directories = new Set([NodePath.dirname(pending.dbPath)]);
412 + // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
413 + const mainTarget = await restoreTarget(pending.dbPath);
414 for (const suffix of DB_FILE_SUFFIXES) {
334 - const livePath = `${pending.dbPath}${suffix}`;
415 + const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
416 + if (livePath !== `${pending.dbPath}${suffix}`) {
417 + await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
418 + }
419 const source = databaseBackupFile(backupDir, suffix);
420 if (await pathExists(source)) {
421 // Restore into the symlink target; cloning would replace the link.
@@ -708,8 +792,17 @@ export class Launcher {
792 return;
793 }
794 this.#timer = undefined;
795 + // Anything thrown outside its own fallback still leaves the offline backup to run.
796 + const finishOnlineBackup = await startOnlineDatabaseBackup(
797 + this.#baseDir,
798 + pending,
799 + this.#backupSpaceProbe,
800 + ).catch(() => undefined);
801 this.#child = null;
802 await terminateChild(child.process);
803 + await finishOnlineBackup?.();
804 + // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
805 + if (this.#stopRequested) return;
806 await this.#startTrial(pending);
807 }
808