@devries-cameron20 / t3code Mirror · read-only / commits / 1c44b29395
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 18:30 UTC

fix(server): migrate-dev-db refuses unimported legacy threads (#23191)

Julius Marminge committed Nov 12, 2026 at 19:31 UTC 1c44b29395e23bd1eb3a437443893b777b0e0a6d
2 files changed +143 -41
apps/server/scripts/migrate-dev-db.test.ts
+64 -4
@@ -9,9 +9,9 @@ import { runMigrations } from "../src/persistence/Migrations.ts";
9 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
10 import { runMigrateDevDb } from "./migrate-dev-db.ts";
11
12 -const withDatabase = <A, E>(
12 +const withDatabase = <A, E, R>(
13 databasePath: string,
14 - effect: Effect.Effect<A, E, SqlClient.SqlClient>,
14 + effect: Effect.Effect<A, E, R | SqlClient.SqlClient>,
15 ) => effect.pipe(Effect.provide(NodeSqliteClient.layer({ filename: databasePath })));
16
17 const insertPeerLink = (sql: SqlClient.SqlClient) => sql`INSERT INTO peer_environment_links
@@ -111,9 +111,14 @@ const createFixtureSource = Effect.fn("createMigrateDevDbFixtureSource")(functio
111 (run_id, thread_id, ordinal, provider, status, requested_at, payload_json)
112 VALUES (${`run-${threadId}`}, ${threadId}, 1, 'codex', ${runStatus}, '2026-08-01', '{}')`;
113 yield* sql`INSERT INTO orchestration_events
114 - (event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json)
115 - VALUES (${`event-${threadId}`}, 'thread', ${threadId}, 0, 'thread.created', '2026-08-01', 'user', '{}', '{}')`;
114 + (event_id, aggregate_kind, stream_id, stream_version, event_type, occurred_at, actor_kind, payload_json, metadata_json, application_event_version)
115 + VALUES (${`event-${threadId}`}, 'thread', ${threadId}, 0, 'thread.created', '2026-08-01', 'user', '{}', '{}', 2)`;
116 }
117 + // A V1 thread the server already imported must not trip the legacy guard.
118 + yield* sql`INSERT INTO projection_threads
119 + (thread_id, project_id, title, model_selection_json, runtime_mode, created_at, updated_at)
120 + VALUES ('stopped-thread', 'project-kept', 'stopped-thread',
121 + '{"instanceId":"codex","model":"gpt-5.4"}', 'full-access', '2026-08-01', '2026-08-01')`;
122 yield* sql`INSERT INTO orchestration_v2_projection_provider_sessions
123 (provider_session_id, thread_id, provider, status, updated_at, payload_json)
124 VALUES ('session-shared', 'running-thread', 'codex', 'stopped', '2026-08-01', '{}')`;
@@ -324,6 +329,61 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => {
329 }),
330 );
331
332 + // The server imports V1 threads at boot, not in a migration, so a source can
333 + // reach the V2 schema with its legacy threads still unimported.
334 + it.effect.each([
335 + { name: "a pre-V2 source", migrateToLatest: false },
336 + { name: "a V2 source whose legacy threads were never imported", migrateToLatest: true },
337 + ])("refuses $name", ({ migrateToLatest }) =>
338 + Effect.gen(function* () {
339 + const fs = yield* FileSystem.FileSystem;
340 + const path = yield* Path.Path;
341 + const sourceDir = yield* fs.makeTempDirectoryScoped({ prefix: "migrate-dev-db-v1-data-" });
342 + const destDir = yield* fs.makeTempDirectoryScoped({
343 + prefix: "migrate-dev-db-v1-data-dest-",
344 + });
345 + const stateDir = path.join(sourceDir, "userdata");
346 + const source = path.join(stateDir, "statev2.sqlite");
347 + yield* fs.makeDirectory(stateDir, { recursive: true });
348 + yield* withDatabase(
349 + source,
350 + Effect.gen(function* () {
351 + const sql = yield* SqlClient.SqlClient;
352 + yield* runMigrations({ toMigrationInclusive: 54 });
353 + yield* sql`INSERT INTO projection_projects
354 + (project_id, title, workspace_root, scripts_json, created_at, updated_at)
355 + VALUES ('project-legacy', 'Legacy', '/tmp/legacy', '[]', '2026-08-01', '2026-08-01')`;
356 + yield* sql`INSERT INTO projection_threads
357 + (thread_id, project_id, title, model_selection_json, runtime_mode, created_at, updated_at)
358 + VALUES ('thread-legacy', 'project-legacy', 'Legacy thread',
359 + '{"instanceId":"codex","model":"gpt-5.4"}', 'full-access', '2026-08-01', '2026-08-01')`;
360 + if (migrateToLatest) yield* runMigrations();
361 + }),
362 + );
363 +
364 + // A live destination connection keeps uncheckpointed rows in the WAL;
365 + // the rejected run must leave those bytes alone.
366 + const destination = yield* createFixtureSource(destDir);
367 + yield* withDatabase(
368 + destination,
369 + Effect.gen(function* () {
370 + const sql = yield* SqlClient.SqlClient;
371 + yield* sql.unsafe("PRAGMA journal_mode = WAL").unprepared;
372 + yield* sql`UPDATE projection_projects SET title = 'Edited' WHERE project_id = 'project-kept'`;
373 + const walBefore = yield* fs.readFile(`${destination}-wal`);
374 + assert.isAbove(walBefore.length, 0);
375 +
376 + const error = yield* runMigrateDevDb(
377 + { baseDir: destDir, source, projects: 5, threadsPerProject: 10 },
378 + { sharedHome: sourceDir },
379 + ).pipe(Effect.flip);
380 + assert.equal(error._tag, "MigrateDevDbLegacySourceError");
381 + assert.deepStrictEqual(yield* fs.readFile(`${destination}-wal`), walBefore);
382 + }),
383 + );
384 + }),
385 + );
386 +
387 it.effect("seeds from a source that recorded another migration below V2", () =>
388 Effect.gen(function* () {
389 const fs = yield* FileSystem.FileSystem;
apps/server/scripts/migrate-dev-db.ts
+79 -37
@@ -60,6 +60,18 @@ export class MigrateDevDbSourceIsDestinationError extends Schema.TaggedError<Mig
60 }
61 }
62
63 +export class MigrateDevDbLegacySourceError extends Schema.TaggedError<MigrateDevDbLegacySourceError>()(
64 + "MigrateDevDbLegacySourceError",
65 + {
66 + sourcePath: Schema.String,
67 + legacyThreads: Schema.Number,
68 + },
69 +) {
70 + override get message(): string {
71 + return `Source database '${this.sourcePath}' holds ${this.legacyThreads} legacy threads the server has not imported into V2 yet, which this command cannot carry over. Start a server once on a T3 home holding a copy of it so it imports them, then pass the userdata/statev2.sqlite that server leaves behind as --source.`;
72 + }
73 +}
74 +
75 export class MigrateDevDbServerRunningError extends Schema.TaggedError<MigrateDevDbServerRunningError>()(
76 "MigrateDevDbServerRunningError",
77 {
@@ -210,47 +222,71 @@ const rootIsVisible = (sql: SqlClient.SqlClient) => sql`
222 AND json_extract(root.payload_json, '$.deletedAt') IS NULL
223 AND json_extract(root.payload_json, '$.archivedAt') IS NULL`;
224
213 -// A single read transaction keeps the copy consistent with concurrent server writes.
225 +// One read transaction keeps the copy, and its legacy-thread guard, consistent with concurrent
226 +// server writes. The server imports V1 threads at boot, and the copy keeps only threads with V2
227 +// rows, so unimported ones would vanish silently (see LegacyV1ThreadImporter.pendingThreadCount).
228 const copySourceSlice = Effect.fn("copyDevDbSourceSlice")(function* (sourcePath: string) {
229 const sql = yield* SqlClient.SqlClient;
230 yield* sql`ATTACH DATABASE ${`${NodeURL.pathToFileURL(sourcePath).href}?mode=ro`} AS src`;
217 - const schema = yield* sql<{ type: string; name: string; sql: string }>`
218 - SELECT type, name, sql FROM src.sqlite_master
219 - WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'`;
220 - const threadColumns = yield* sql<{ name: string; notnull: number }>`
221 - SELECT m.name, c."notnull" FROM src.sqlite_master m, pragma_table_info(m.name, 'src') c
222 - WHERE m.type = 'table' AND c.name = 'thread_id'`;
223 - const threadIdNullable = new Map(threadColumns.map((row) => [row.name, row.notnull === 0]));
224 - // FTS5 makes its own shadow tables and contentless rows cannot be copied:
225 - // create the virtual table, skip its shadow tables and rows, and rebuild the
226 - // index once the messages and triggers are in.
227 - const virtualTables = schema
228 - .filter((entry) => entry.type === "table" && /^CREATE VIRTUAL TABLE/i.test(entry.sql))
229 - .map((entry) => entry.name);
230 - const isShadowTable = (name: string) =>
231 - virtualTables.some((virtual) =>
232 - FTS5_SHADOW_SUFFIXES.some((suffix) => name === `${virtual}${suffix}`),
233 - );
234 - const tables = schema.filter((entry) => entry.type === "table" && !isShadowTable(entry.name));
235 - const filtered = tables.some((table) => table.name === "orchestration_v2_projection_threads");
236 - // A row that points into a cleared table cannot be copied, so whatever references one is
237 - // cleared with it, however far down the chain a newer migration added it.
238 - const references = yield* sql<{ name: string; parent: string }>`
239 - SELECT m.name, f."table" AS parent FROM src.sqlite_master m, pragma_foreign_key_list(m.name, 'src') f
240 - WHERE m.type = 'table'`;
241 - const cleared = new Set(CLEARED_TABLES);
242 - for (let grew = true; grew;) {
243 - grew = false;
244 - for (const { name, parent } of references) {
245 - if (cleared.has(parent) && !cleared.has(name)) {
246 - cleared.add(name);
247 - grew = true;
248 - }
249 - }
250 - }
231
232 yield* sql.withTransaction(
233 Effect.gen(function* () {
234 + const schema = yield* sql<{ type: string; name: string; sql: string }>`
235 + SELECT type, name, sql FROM src.sqlite_master
236 + WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%'`;
237 + const threadColumns = yield* sql<{ name: string; notnull: number }>`
238 + SELECT m.name, c."notnull" FROM src.sqlite_master m, pragma_table_info(m.name, 'src') c
239 + WHERE m.type = 'table' AND c.name = 'thread_id'`;
240 + const threadIdNullable = new Map(threadColumns.map((row) => [row.name, row.notnull === 0]));
241 + // FTS5 makes its own shadow tables and contentless rows cannot be copied:
242 + // create the virtual table, skip its shadow tables and rows, and rebuild the
243 + // index once the messages and triggers are in.
244 + const virtualTables = schema
245 + .filter((entry) => entry.type === "table" && /^CREATE VIRTUAL TABLE/i.test(entry.sql))
246 + .map((entry) => entry.name);
247 + const isShadowTable = (name: string) =>
248 + virtualTables.some((virtual) =>
249 + FTS5_SHADOW_SUFFIXES.some((suffix) => name === `${virtual}${suffix}`),
250 + );
251 + const tables = schema.filter((entry) => entry.type === "table" && !isShadowTable(entry.name));
252 + const filtered = tables.some((table) => table.name === "orchestration_v2_projection_threads");
253 + // A row that points into a cleared table cannot be copied, so whatever references one is
254 + // cleared with it, however far down the chain a newer migration added it.
255 + const references = yield* sql<{ name: string; parent: string }>`
256 + SELECT m.name, f."table" AS parent FROM src.sqlite_master m, pragma_foreign_key_list(m.name, 'src') f
257 + WHERE m.type = 'table'`;
258 + const cleared = new Set(CLEARED_TABLES);
259 + for (let grew = true; grew;) {
260 + grew = false;
261 + for (const { name, parent } of references) {
262 + if (cleared.has(parent) && !cleared.has(name)) {
263 + cleared.add(name);
264 + grew = true;
265 + }
266 + }
267 + }
268 +
269 + if (tables.some((table) => table.name === "projection_threads")) {
270 + // The V2 tables, application_event_version, and the partial index arrive in
271 + // the same migration. Without the hint SQLite scans each thread's stream.
272 + const [pending] = filtered
273 + ? yield* sql<{ count: number }>`
274 + SELECT COUNT(*) AS count FROM src.projection_threads AS thread
275 + WHERE NOT EXISTS (
276 + SELECT 1
277 + FROM src.orchestration_events AS event INDEXED BY orchestration_events_v2_created_threads_idx
278 + WHERE event.application_event_version = 2
279 + AND event.aggregate_kind = 'thread'
280 + AND event.stream_id = thread.thread_id
281 + AND event.event_type = 'thread.created'
282 + )`
283 + : yield* sql<{ count: number }>`SELECT COUNT(*) AS count FROM src.projection_threads`;
284 + const legacyThreads = Number(pending?.count ?? 0);
285 + if (legacyThreads > 0) {
286 + return yield* new MigrateDevDbLegacySourceError({ sourcePath, legacyThreads });
287 + }
288 + }
289 +
290 for (const table of tables) yield* sql.unsafe(table.sql).unprepared;
291 if (filtered) {
292 yield* sql`CREATE TEMP TABLE copied_threads (thread_id TEXT PRIMARY KEY)`;
@@ -496,7 +532,6 @@ export const runMigrateDevDb = Effect.fn("runMigrateDevDb")(function* (
532 }
533
534 yield* fs.makeDirectory(stateDir, { recursive: true });
499 - yield* ensureNotInUse(databasePath);
535
536 const wrapPhase =
537 (phase: MigrateDevDbPhaseError["phase"], phaseDatabasePath: string) =>
@@ -512,8 +547,15 @@ export const runMigrateDevDb = Effect.fn("runMigrateDevDb")(function* (
547 yield* Console.log(`Copying visible threads from ${sourcePath} (read-only)...`);
548 yield* copySourceSlice(sourcePath).pipe(
549 Effect.provide(NodeSqliteClient.layer({ filename: snapshotPath })),
515 - wrapPhase("snapshot", sourcePath),
550 + Effect.catchTags({
551 + SqlError: (cause) =>
552 + Effect.fail(
553 + new MigrateDevDbPhaseError({ phase: "snapshot", databasePath: sourcePath, cause }),
554 + ),
555 + }),
556 );
557 + // Checkpoints the destination, so it runs after the copy's legacy guard.
558 + yield* ensureNotInUse(databasePath);
559
560 // Migrate first so prune queries can use columns missing from older sources.
561 yield* Console.log("Running migrations on the snapshot...");