The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 19:45 UTC
fix(server): long startup migrations log progress and resume after a restart (#35486)
Julius Marminge committed
Nov 12, 2026 at 11:25 UTC
4ca5b9cdb8f5f8a0e463faba0d5c4b4f6bb322c0
5 files changed
+152
-12
apps/server/src/persistence/Migrations.test.ts
+16
@@ -151,6 +151,22 @@ describe("runMigrations", () => {
151
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
152
);
153
154
+ it.effect("keeps the migrations that finished before a later one failed", () =>
155
+ Effect.gen(function* () {
156
+ const sql = yield* SqlClient.SqlClient;
157
+ yield* runMigrations({ toMigrationInclusive: 81 });
158
+ yield* sql`CREATE TABLE orchestration_v2_projection_turn_item_counts (thread_id TEXT)`;
159
+
160
+ const exit = yield* Effect.exit(runMigrations());
161
+
162
+ assert.isTrue(Exit.isFailure(exit));
163
+ assert.deepStrictEqual(
164
+ yield* history,
165
+ migrationManifest.filter(([id]) => id <= 84),
166
+ );
167
+ }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
168
+ );
169
+
170
it.effect("reapplies peer link migrations whose ledger rows are missing", () =>
171
Effect.gen(function* () {
172
const sql = yield* SqlClient.SqlClient;
apps/server/src/persistence/Migrations.ts
+29
-5
@@ -268,16 +268,40 @@ export const runMigrations = Effect.fn("runMigrations")(function* ({
268
if (includesV2) {
269
yield* assertV2MigrationsCanRun(toMigrationInclusive);
270
}
271
- const executedMigrations = [
272
- ...previewMigrations,
273
- ...(yield* run({ loader: makeMigrationLoader(toMigrationInclusive) })),
274
- ];
271
+ const sql = yield* SqlClient.SqlClient;
272
+ const ledger = yield* sql`
273
+ SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'effect_sql_migrations'
274
+ `;
275
+ const [latest] =
276
+ ledger.length === 0
277
+ ? []
278
+ : yield* sql<{
279
+ readonly id: number | null;
280
+ }>`SELECT MAX(migration_id) AS id FROM effect_sql_migrations`;
281
+ const latestRecordedId = latest?.id ?? 0;
282
+ const pending = migrationEntries.filter(
283
+ ([id]) =>
284
+ id > latestRecordedId && (toMigrationInclusive === undefined || id <= toMigrationInclusive),
285
+ );
286
+ // One transaction per migration: an index build on a large database can take minutes, and a
287
+ // restart then resumes after the last finished migration instead of rolling all of them back.
288
+ const executedMigrations: Array<readonly [id: number, name: string]> = [...previewMigrations];
289
+ for (const [index, [id, name]] of pending.entries()) {
290
+ if (latestRecordedId > 0) {
291
+ yield* Effect.log("Upgrading database").pipe(
292
+ Effect.annotateLogs({
293
+ migration: `${id}_${name}`,
294
+ progress: `${index + 1} of ${pending.length}`,
295
+ }),
296
+ );
297
+ }
298
+ executedMigrations.push(...(yield* run({ loader: makeMigrationLoader(id) })));
299
+ }
300
const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`);
301
yield* migrations.length === 0
302
? Effect.logDebug("Database schema is current")
303
: Effect.log("Migrations ran successfully").pipe(Effect.annotateLogs({ migrations }));
304
280
- const sql = yield* SqlClient.SqlClient;
305
const recorded = yield* sql<{
306
readonly migration_id: number;
307
readonly name: string;
apps/server/src/persistence/Sqlite.test.ts
+39
@@ -24,6 +24,7 @@ import { migrationManifest, runMigrations } from "./Migrations.ts";
24
import * as SqlitePersistence from "./Sqlite.ts";
25
26
const workers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
27
+const workerData = vi.hoisted((): Array<unknown> => []);
28
29
vi.mock("node:worker_threads", async (importOriginal) => {
30
const actual = await importOriginal<typeof NodeWorkerThreads>();
@@ -31,6 +32,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
32
constructor(...args: ConstructorParameters<typeof actual.Worker>) {
33
super(...args);
34
workers.push(this);
35
+ workerData.push(args[1]?.workerData);
36
}
37
}
38
return { ...actual, Worker: RecordedWorker };
@@ -192,6 +194,43 @@ it.effect("drops planner statistics a database already has", () => {
194
);
195
});
196
197
+it.effect("upgrades an existing database on a read-write worker connection", () => {
198
+ const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-upgrade-"));
199
+ const dbPath = NodePath.join(tempDir, "state.sqlite");
200
+
201
+ return Effect.gen(function* () {
202
+ yield* Effect.gen(function* () {
203
+ const sql = yield* SqlClient.SqlClient;
204
+ yield* sql`PRAGMA journal_mode = WAL;`;
205
+ yield* runMigrations({ toMigrationInclusive: 81 });
206
+ }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: dbPath })));
207
+ const startedBefore = workerData.length;
208
+
209
+ yield* Effect.gen(function* () {
210
+ const sql = yield* SqlClient.SqlClient;
211
+ const rows = yield* sql<{ readonly migration_id: number; readonly name: string }>`
212
+ SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id
213
+ `;
214
+ assert.deepStrictEqual(
215
+ rows.map((row) => [row.migration_id, row.name] as const),
216
+ migrationManifest,
217
+ );
218
+ }).pipe(
219
+ Effect.provide(
220
+ SqlitePersistence.layerFromPath(dbPath).pipe(Layer.provide(NodeServices.layer)),
221
+ ),
222
+ );
223
+ assert.deepInclude(workerData.slice(startedBefore), {
224
+ filename: dbPath,
225
+ readOnly: false,
226
+ allowExtension: false,
227
+ prepareCacheSize: 200,
228
+ });
229
+ }).pipe(
230
+ Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
231
+ );
232
+});
233
+
234
it.effect("names a database file that is not SQLite", () => {
235
const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-notadb-"));
236
const dbPath = NodePath.join(tempDir, "state.sqlite");
apps/server/src/persistence/Sqlite.ts
+26
-4
@@ -64,6 +64,26 @@ const runMigrationsAsOwner = Effect.fn("SqlitePersistence.runMigrationsAsOwner")
64
yield* runMigrations();
65
}, Effect.scoped);
66
67
+// Index builds over large tables take minutes; on the main connection they block the event loop,
68
+// so requests hang instead of getting the listener's 503. A fresh database stays on the main
69
+// connection, which must create the first table for its auto_vacuum setting to apply.
70
+const runMigrationsOffMainThread = Effect.fn("SqlitePersistence.runMigrationsOffMainThread")(
71
+ function* (dbPath: string) {
72
+ const sql = yield* SqlClient.SqlClient;
73
+ const tables = yield* sql<{
74
+ readonly count: number;
75
+ }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
76
+ if (!tables[0]?.count || !(yield* hasPendingMigrations())) return yield* runMigrations();
77
+ return yield* Effect.gen(function* () {
78
+ const worker = yield* SqlClient.SqlClient;
79
+ yield* worker`PRAGMA foreign_keys = ON;`;
80
+ yield* worker`PRAGMA synchronous = NORMAL;`;
81
+ yield* worker`PRAGMA temp_store = MEMORY;`;
82
+ return yield* runMigrations();
83
+ }).pipe(Effect.provide(NodeSqliteClient.layerWorker(dbPath)));
84
+ },
85
+);
86
+
87
const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
88
Layer.effectDiscard(
89
Effect.gen(function* () {
@@ -100,14 +120,14 @@ const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
120
121
const layerSetup = makeLayerSetup(runMigrations());
122
103
-const makeLayerFromPath = <E, R>(setup: Layer.Layer<never, E, R>) =>
123
+const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<never, E, R>) =>
124
Effect.fn("makeSqlitePersistenceLive")(function* (dbPath: string) {
125
const fs = yield* FileSystem.FileSystem;
126
const path = yield* Path.Path;
127
yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true });
128
129
return Layer.provideMerge(
110
- setup,
130
+ makeSetup(dbPath),
131
NodeSqliteClient.layer({
132
filename: dbPath,
133
readerWorker: true,
@@ -132,7 +152,9 @@ const makeLayerFromPath = <E, R>(setup: Layer.Layer<never, E, R>) =>
152
);
153
}, Layer.unwrap);
154
135
-export const layerFromPath = makeLayerFromPath(layerSetup);
155
+export const layerFromPath = makeLayerFromPath((dbPath) =>
156
+ makeLayerSetup(runMigrationsOffMainThread(dbPath)),
157
+);
158
159
export const layerMemory = Layer.provideMerge(
160
layerSetup,
@@ -154,5 +176,5 @@ export const layerConfig = makeLayerConfig(layerFromPath);
176
177
/** For CLI commands: pending migrations run only while no server owns the T3 home. */
178
export const layerCliConfig = makeLayerConfig(
157
- makeLayerFromPath(makeLayerSetup(runMigrationsAsOwner())),
179
+ makeLayerFromPath(() => makeLayerSetup(runMigrationsAsOwner())),
180
);
packages/shared/src/nodeSqliteClient.ts
+42
-3
@@ -201,7 +201,7 @@ let db;
201
try {
202
// A read-only open cannot be undone by SQL, unlike PRAGMA query_only.
203
const next = new DatabaseSync(workerData.filename, {
204
- readOnly: true,
204
+ readOnly: workerData.readOnly,
205
allowExtension: workerData.allowExtension,
206
});
207
next.exec("PRAGMA busy_timeout = 5000;");
@@ -300,6 +300,7 @@ const READER_POOL_SIZE = 4;
300
const makeReader = Effect.fnUntraced(function* (
301
options: SqliteClientConfig,
302
pool: { unavailable: boolean },
303
+ readOnly = true,
304
) {
305
let current:
306
| {
@@ -315,6 +316,7 @@ const makeReader = Effect.fnUntraced(function* (
316
eval: true,
317
workerData: {
318
filename: options.filename,
319
+ readOnly,
320
allowExtension: false,
321
prepareCacheSize: 200,
322
},
@@ -358,7 +360,10 @@ const makeReader = Effect.fnUntraced(function* (
360
361
const request = (input: ReaderRequest) =>
362
Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
361
- const endsTransaction = endsReadTransaction(input.sql);
363
+ // A COMMIT that never reached a writer must fail; only its ROLLBACK is moot.
364
+ const endsTransaction = readOnly
365
+ ? endsReadTransaction(input.sql)
366
+ : /^\s*ROLLBACK\b/i.test(input.sql);
367
if (stoppedMessage !== undefined) {
368
return endsTransaction
369
? Effect.succeed([])
@@ -411,7 +416,7 @@ const makeReader = Effect.fnUntraced(function* (
416
},
417
executeUnprepared(sql, params, rowTransform) {
418
// Read snapshots need deferred BEGIN; readonly connections reject IMMEDIATE and EXCLUSIVE.
414
- const statement = /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql;
419
+ const statement = readOnly && /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql;
420
const effect = request({
421
sql: statement,
422
params: params ?? [],
@@ -1288,5 +1293,39 @@ const make = Effect.fn("makeWithDatabase")(function* (
1293
});
1294
});
1295
1296
+const makeWorker = Effect.fn("makeWorkerClient")(function* (filename: string) {
1297
+ yield* checkNodeSqliteCompat();
1298
+ const acquire = yield* makeReader({ filename }, { unavailable: false }, false);
1299
+ const connection = yield* acquire;
1300
+ if (connection === undefined) {
1301
+ return yield* failed(
1302
+ new Error("database worker is unavailable"),
1303
+ readerFailureMessages.open,
1304
+ "open",
1305
+ );
1306
+ }
1307
+ const { acquirer, borrower, onCommitFailure, transactionAcquirer } = Client.makeSqliteAcquirers({
1308
+ connection: Effect.succeed(connection),
1309
+ semaphore: yield* Semaphore.make(1),
1310
+ });
1311
+ return yield* Client.make({
1312
+ driver: "node:sqlite",
1313
+ acquirer,
1314
+ borrower,
1315
+ compiler: Statement.makeCompilerSqlite(),
1316
+ transactionAcquirer,
1317
+ onCommitFailure,
1318
+ beginTransaction: "BEGIN IMMEDIATE",
1319
+ spanAttributes: [[ATTR_DB_SYSTEM_NAME, "sqlite"]],
1320
+ });
1321
+});
1322
+
1323
+/**
1324
+ * A read-write client whose one connection runs in a worker thread, so long statements such as
1325
+ * index builds leave the event loop free. Requires a file database already in WAL mode.
1326
+ */
1327
+export const layerWorker = (filename: string): Layer.Layer<Client.SqlClient, SqlError> =>
1328
+ Layer.effect(Client.SqlClient, makeWorker(filename)).pipe(Layer.provide(Reactivity.layer));
1329
+
1330
export const layer = (config: SqliteClientConfig): Layer.Layer<Client.SqlClient, SqlError> =>
1331
Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer));