@devries-cameron20 / t3code Mirror · read-only / commits / 4ca5b9cdb8
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 19:45 UTC

fix(server): long startup migrations log progress and resume after a restart (#35486)

Julius Marminge committed Nov 12, 2026 at 11:25 UTC 4ca5b9cdb8f5f8a0e463faba0d5c4b4f6bb322c0
5 files changed +152 -12
apps/server/src/persistence/Migrations.test.ts
+16
@@ -151,6 +151,22 @@ describe("runMigrations", () => {
151 }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
152 );
153
154 + it.effect("keeps the migrations that finished before a later one failed", () =>
155 + Effect.gen(function* () {
156 + const sql = yield* SqlClient.SqlClient;
157 + yield* runMigrations({ toMigrationInclusive: 81 });
158 + yield* sql`CREATE TABLE orchestration_v2_projection_turn_item_counts (thread_id TEXT)`;
159 +
160 + const exit = yield* Effect.exit(runMigrations());
161 +
162 + assert.isTrue(Exit.isFailure(exit));
163 + assert.deepStrictEqual(
164 + yield* history,
165 + migrationManifest.filter(([id]) => id <= 84),
166 + );
167 + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
168 + );
169 +
170 it.effect("reapplies peer link migrations whose ledger rows are missing", () =>
171 Effect.gen(function* () {
172 const sql = yield* SqlClient.SqlClient;
apps/server/src/persistence/Migrations.ts
+29 -5
@@ -268,16 +268,40 @@ export const runMigrations = Effect.fn("runMigrations")(function* ({
268 if (includesV2) {
269 yield* assertV2MigrationsCanRun(toMigrationInclusive);
270 }
271 - const executedMigrations = [
272 - ...previewMigrations,
273 - ...(yield* run({ loader: makeMigrationLoader(toMigrationInclusive) })),
274 - ];
271 + const sql = yield* SqlClient.SqlClient;
272 + const ledger = yield* sql`
273 + SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'effect_sql_migrations'
274 + `;
275 + const [latest] =
276 + ledger.length === 0
277 + ? []
278 + : yield* sql<{
279 + readonly id: number | null;
280 + }>`SELECT MAX(migration_id) AS id FROM effect_sql_migrations`;
281 + const latestRecordedId = latest?.id ?? 0;
282 + const pending = migrationEntries.filter(
283 + ([id]) =>
284 + id > latestRecordedId && (toMigrationInclusive === undefined || id <= toMigrationInclusive),
285 + );
286 + // One transaction per migration: an index build on a large database can take minutes, and a
287 + // restart then resumes after the last finished migration instead of rolling all of them back.
288 + const executedMigrations: Array<readonly [id: number, name: string]> = [...previewMigrations];
289 + for (const [index, [id, name]] of pending.entries()) {
290 + if (latestRecordedId > 0) {
291 + yield* Effect.log("Upgrading database").pipe(
292 + Effect.annotateLogs({
293 + migration: `${id}_${name}`,
294 + progress: `${index + 1} of ${pending.length}`,
295 + }),
296 + );
297 + }
298 + executedMigrations.push(...(yield* run({ loader: makeMigrationLoader(id) })));
299 + }
300 const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`);
301 yield* migrations.length === 0
302 ? Effect.logDebug("Database schema is current")
303 : Effect.log("Migrations ran successfully").pipe(Effect.annotateLogs({ migrations }));
304
280 - const sql = yield* SqlClient.SqlClient;
305 const recorded = yield* sql<{
306 readonly migration_id: number;
307 readonly name: string;
apps/server/src/persistence/Sqlite.test.ts
+39
@@ -24,6 +24,7 @@ import { migrationManifest, runMigrations } from "./Migrations.ts";
24 import * as SqlitePersistence from "./Sqlite.ts";
25
26 const workers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
27 +const workerData = vi.hoisted((): Array<unknown> => []);
28
29 vi.mock("node:worker_threads", async (importOriginal) => {
30 const actual = await importOriginal<typeof NodeWorkerThreads>();
@@ -31,6 +32,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
32 constructor(...args: ConstructorParameters<typeof actual.Worker>) {
33 super(...args);
34 workers.push(this);
35 + workerData.push(args[1]?.workerData);
36 }
37 }
38 return { ...actual, Worker: RecordedWorker };
@@ -192,6 +194,43 @@ it.effect("drops planner statistics a database already has", () => {
194 );
195 });
196
197 +it.effect("upgrades an existing database on a read-write worker connection", () => {
198 + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-upgrade-"));
199 + const dbPath = NodePath.join(tempDir, "state.sqlite");
200 +
201 + return Effect.gen(function* () {
202 + yield* Effect.gen(function* () {
203 + const sql = yield* SqlClient.SqlClient;
204 + yield* sql`PRAGMA journal_mode = WAL;`;
205 + yield* runMigrations({ toMigrationInclusive: 81 });
206 + }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: dbPath })));
207 + const startedBefore = workerData.length;
208 +
209 + yield* Effect.gen(function* () {
210 + const sql = yield* SqlClient.SqlClient;
211 + const rows = yield* sql<{ readonly migration_id: number; readonly name: string }>`
212 + SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id
213 + `;
214 + assert.deepStrictEqual(
215 + rows.map((row) => [row.migration_id, row.name] as const),
216 + migrationManifest,
217 + );
218 + }).pipe(
219 + Effect.provide(
220 + SqlitePersistence.layerFromPath(dbPath).pipe(Layer.provide(NodeServices.layer)),
221 + ),
222 + );
223 + assert.deepInclude(workerData.slice(startedBefore), {
224 + filename: dbPath,
225 + readOnly: false,
226 + allowExtension: false,
227 + prepareCacheSize: 200,
228 + });
229 + }).pipe(
230 + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
231 + );
232 +});
233 +
234 it.effect("names a database file that is not SQLite", () => {
235 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-notadb-"));
236 const dbPath = NodePath.join(tempDir, "state.sqlite");
apps/server/src/persistence/Sqlite.ts
+26 -4
@@ -64,6 +64,26 @@ const runMigrationsAsOwner = Effect.fn("SqlitePersistence.runMigrationsAsOwner")
64 yield* runMigrations();
65 }, Effect.scoped);
66
67 +// Index builds over large tables take minutes; on the main connection they block the event loop,
68 +// so requests hang instead of getting the listener's 503. A fresh database stays on the main
69 +// connection, which must create the first table for its auto_vacuum setting to apply.
70 +const runMigrationsOffMainThread = Effect.fn("SqlitePersistence.runMigrationsOffMainThread")(
71 + function* (dbPath: string) {
72 + const sql = yield* SqlClient.SqlClient;
73 + const tables = yield* sql<{
74 + readonly count: number;
75 + }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
76 + if (!tables[0]?.count || !(yield* hasPendingMigrations())) return yield* runMigrations();
77 + return yield* Effect.gen(function* () {
78 + const worker = yield* SqlClient.SqlClient;
79 + yield* worker`PRAGMA foreign_keys = ON;`;
80 + yield* worker`PRAGMA synchronous = NORMAL;`;
81 + yield* worker`PRAGMA temp_store = MEMORY;`;
82 + return yield* runMigrations();
83 + }).pipe(Effect.provide(NodeSqliteClient.layerWorker(dbPath)));
84 + },
85 +);
86 +
87 const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
88 Layer.effectDiscard(
89 Effect.gen(function* () {
@@ -100,14 +120,14 @@ const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
120
121 const layerSetup = makeLayerSetup(runMigrations());
122
103 -const makeLayerFromPath = <E, R>(setup: Layer.Layer<never, E, R>) =>
123 +const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<never, E, R>) =>
124 Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string) {
125 const fs = yield* FileSystem.FileSystem;
126 const path = yield* Path.Path;
127 yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
128
129 return Layer.provideMerge(
110 - setup,
130 + makeSetup(dbPath),
131 NodeSqliteClient.layer({
132 filename: dbPath,
133 readerWorker: true,
@@ -132,7 +152,9 @@ const makeLayerFromPath = <E, R>(setup: Layer.Layer<never, E, R>) =>
152 );
153 }, Layer.unwrap);
154
135 -export const layerFromPath = makeLayerFromPath(layerSetup);
155 +export const layerFromPath = makeLayerFromPath((dbPath) =>
156 + makeLayerSetup(runMigrationsOffMainThread(dbPath)),
157 +);
158
159 export const layerMemory = Layer.provideMerge(
160 layerSetup,
@@ -154,5 +176,5 @@ export const layerConfig = makeLayerConfig(layerFromPath);
176
177 /** For CLI commands: pending migrations run only while no server owns the T3 home. */
178 export const layerCliConfig = makeLayerConfig(
157 - makeLayerFromPath(makeLayerSetup(runMigrationsAsOwner())),
179 + makeLayerFromPath(() => makeLayerSetup(runMigrationsAsOwner())),
180 );
packages/shared/src/nodeSqliteClient.ts
+42 -3
@@ -201,7 +201,7 @@ let db;
201 try {
202 // A read-only open cannot be undone by SQL, unlike PRAGMA query_only.
203 const next = new DatabaseSync(workerData.filename, {
204 - readOnly: true,
204 + readOnly: workerData.readOnly,
205 allowExtension: workerData.allowExtension,
206 });
207 next.exec("PRAGMA busy_timeout = 5000;");
@@ -300,6 +300,7 @@ const READER_POOL_SIZE = 4;
300 const makeReader = Effect.fnUntraced(function* (
301 options: SqliteClientConfig,
302 pool: { unavailable: boolean },
303 + readOnly = true,
304 ) {
305 let current:
306 | {
@@ -315,6 +316,7 @@ const makeReader = Effect.fnUntraced(function* (
316 eval: true,
317 workerData: {
318 filename: options.filename,
319 + readOnly,
320 allowExtension: false,
321 prepareCacheSize: 200,
322 },
@@ -358,7 +360,10 @@ const makeReader = Effect.fnUntraced(function* (
360
361 const request = (input: ReaderRequest) =>
362 Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
361 - const endsTransaction = endsReadTransaction(input.sql);
363 + // A COMMIT that never reached a writer must fail; only its ROLLBACK is moot.
364 + const endsTransaction = readOnly
365 + ? endsReadTransaction(input.sql)
366 + : /^\s*ROLLBACK\b/i.test(input.sql);
367 if (stoppedMessage !== undefined) {
368 return endsTransaction
369 ? Effect.succeed([])
@@ -411,7 +416,7 @@ const makeReader = Effect.fnUntraced(function* (
416 },
417 executeUnprepared(sql, params, rowTransform) {
418 // Read snapshots need deferred BEGIN; readonly connections reject IMMEDIATE and EXCLUSIVE.
414 - const statement = /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql;
419 + const statement = readOnly && /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql;
420 const effect = request({
421 sql: statement,
422 params: params ?? [],
@@ -1288,5 +1293,39 @@ const make = Effect.fn("makeWithDatabase")(function* (
1293 });
1294 });
1295
1296 +const makeWorker = Effect.fn("makeWorkerClient")(function* (filename: string) {
1297 + yield* checkNodeSqliteCompat();
1298 + const acquire = yield* makeReader({ filename }, { unavailable: false }, false);
1299 + const connection = yield* acquire;
1300 + if (connection === undefined) {
1301 + return yield* failed(
1302 + new Error("database worker is unavailable"),
1303 + readerFailureMessages.open,
1304 + "open",
1305 + );
1306 + }
1307 + const { acquirer, borrower, onCommitFailure, transactionAcquirer } = Client.makeSqliteAcquirers({
1308 + connection: Effect.succeed(connection),
1309 + semaphore: yield* Semaphore.make(1),
1310 + });
1311 + return yield* Client.make({
1312 + driver: "node:sqlite",
1313 + acquirer,
1314 + borrower,
1315 + compiler: Statement.makeCompilerSqlite(),
1316 + transactionAcquirer,
1317 + onCommitFailure,
1318 + beginTransaction: "BEGIN IMMEDIATE",
1319 + spanAttributes: [[ATTR_DB_SYSTEM_NAME, "sqlite"]],
1320 + });
1321 +});
1322 +
1323 +/**
1324 + * A read-write client whose one connection runs in a worker thread, so long statements such as
1325 + * index builds leave the event loop free. Requires a file database already in WAL mode.
1326 + */
1327 +export const layerWorker = (filename: string): Layer.Layer<Client.SqlClient, SqlError> =>
1328 + Layer.effect(Client.SqlClient, makeWorker(filename)).pipe(Layer.provide(Reactivity.layer));
1329 +
1330 export const layer = (config: SqliteClientConfig): Layer.Layer<Client.SqlClient, SqlError> =>
1331 Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer));