@devries-cameron20 / t3code-1 / commits / c0561c5c3d
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(shared): the SQLite reader pool comes back after a failed start (#30818)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Navid Nadali committed Nov 15, 2026 at 00:20 UTC c0561c5c3d75f5fa2017d4ba9d75124ad63fe897
2 files changed +250 -5
packages/shared/src/nodeSqliteClient.test.ts
+196
@@ -4,12 +4,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
4 import { assert, it } from "@effect/vitest";
5 import * as Deferred from "effect/Deferred";
6 import * as Cause from "effect/Cause";
7 +import * as Duration from "effect/Duration";
8 import * as Effect from "effect/Effect";
9 import * as Exit from "effect/Exit";
10 import * as Fiber from "effect/Fiber";
11 import * as FileSystem from "effect/FileSystem";
12 import * as Layer from "effect/Layer";
13 +import * as Logger from "effect/Logger";
14 import * as Path from "effect/Path";
15 +import * as Queue from "effect/Queue";
16 import * as SqlClient from "effect/sql/SqlClient";
17 import * as SqlError from "effect/sql/SqlError";
18 import * as Statement from "effect/sql/Statement";
@@ -22,6 +25,7 @@ import { vi } from "vite-plus/test";
25 import * as SqliteClient from "./nodeSqliteClient.ts";
26
27 const startedWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
28 +const workerExits = vi.hoisted(() => new WeakMap<NodeWorkerThreads.Worker, Promise<void>>());
29 const silentWorkers = vi.hoisted(() => ({ on: false, readyOnTerminate: false }));
30 const failingWorkers = vi.hoisted(() => ({ message: undefined as string | undefined }));
31 const checkpointWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
@@ -38,6 +42,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
42 : args) as ConstructorParameters<typeof actual.Worker>),
43 );
44 startedWorkers.push(this);
45 + workerExits.set(this, new Promise((resolve) => this.once("exit", () => resolve())));
46 if (args[1]?.workerData?.frames !== undefined) checkpointWorkers.push(this);
47 }
48 override terminate() {
@@ -620,6 +625,197 @@ it.effect("uses the writer when the reader cannot open the database", () =>
625 ),
626 );
627
628 +const withLogLines = <A, E, R>(
629 + use: (lines: Queue.Queue<ReadonlyArray<unknown>>) => Effect.Effect<A, E, R>,
630 +) =>
631 + Effect.gen(function* () {
632 + const lines = yield* Queue.unbounded<ReadonlyArray<unknown>>();
633 + const logger = Logger.make(({ logLevel, message }) => {
634 + Queue.offerUnsafe(lines, [logLevel, ...(message as ReadonlyArray<unknown>)]);
635 + });
636 + return yield* use(lines).pipe(
637 + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })),
638 + );
639 + });
640 +
641 +const readConnection = (sql: SqlClient.SqlClient) =>
642 + Effect.gen(function* () {
643 + const spans: Array<Tracer.Span> = [];
644 + const observe: Statement.Transformer = (statement, _sql, _fiber, span) =>
645 + Effect.sync(() => {
646 + spans.push(span);
647 + return statement;
648 + });
649 + const count = yield* countEntries(sql).pipe(
650 + SqliteClient.readOnly,
651 + Effect.provideService(Statement.CurrentTransformer, observe),
652 + Effect.provideService(Statement.SpanPropagationEnabled, true),
653 + );
654 + return { count, connection: spans.map((span) => span.attributes.get("db.connection")) };
655 + });
656 +
657 +const withWorkers = <A, E, R>(
658 + mode: { readonly failWith?: string; readonly silent?: boolean },
659 + effect: Effect.Effect<A, E, R>,
660 +) =>
661 + Effect.acquireUseRelease(
662 + Effect.sync(() => {
663 + failingWorkers.message = mode.failWith;
664 + silentWorkers.on = mode.silent ?? false;
665 + }),
666 + () => effect,
667 + () =>
668 + Effect.sync(() => {
669 + failingWorkers.message = undefined;
670 + silentWorkers.on = false;
671 + }),
672 + );
673 +
674 +const unavailableWarning = "SQLite reader worker unavailable; reads use the writer";
675 +
676 +const notWal = "database uses journal_mode delete, not wal";
677 +
678 +it.effect.each([
679 + {
680 + failure: "a crash",
681 + workers: { failWith: "reader start failed" },
682 + startup: Duration.zero,
683 + reason: "Database reader failed: reader start failed",
684 + },
685 + {
686 + failure: "a startup timeout",
687 + workers: { silent: true },
688 + startup: Duration.seconds(10),
689 + reason: "Database reader did not open the database within 10000 ms",
690 + },
691 +])(
692 + "reads on the writer after $failure until a retry starts the reader again",
693 + ({ workers, startup, reason }) =>
694 + withLogLines((lines) =>
695 + withReaderClient((sql) =>
696 + Effect.gen(function* () {
697 + const failed = yield* withWorkers(
698 + workers,
699 + Effect.gen(function* () {
700 + const read = yield* Effect.forkChild(readConnection(sql), {
701 + startImmediately: true,
702 + });
703 + yield* TestClock.adjust(startup);
704 + return yield* Fiber.join(read);
705 + }),
706 + );
707 + assert.deepEqual(failed, { count: 1, connection: ["writer"] });
708 + const startedBefore = startedWorkers.length;
709 + assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["writer"] });
710 + assert.equal(startedWorkers.length, startedBefore);
711 + assert.deepEqual(yield* Queue.take(lines), ["Warn", unavailableWarning, { reason }]);
712 +
713 + yield* TestClock.adjust("1 second");
714 + assert.equal(startedWorkers.length, startedBefore + 1);
715 + assert.deepEqual(yield* Queue.take(lines), [
716 + "Info",
717 + "SQLite reader worker available again; reads use the reader workers",
718 + ]);
719 + assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["reader"] });
720 + assert.equal(Queue.sizeUnsafe(lines), 0);
721 + }),
722 + ),
723 + ),
724 +);
725 +
726 +it.effect("backs off a reader that never starts without logging each retry", () =>
727 + withLogLines((lines) =>
728 + withReaderClient((sql) =>
729 + withWorkers(
730 + { silent: true },
731 + Effect.gen(function* () {
732 + const read = yield* Effect.forkChild(readConnection(sql), { startImmediately: true });
733 + yield* TestClock.adjust("10 seconds");
734 + assert.deepEqual(yield* Fiber.join(read), { count: 1, connection: ["writer"] });
735 +
736 + const startedBefore = startedWorkers.length;
737 + // Each retry waits 10 s for startup; the gaps between them double from 1 s up to a minute.
738 + yield* TestClock.adjust("10 minutes");
739 + assert.equal(startedWorkers.length - startedBefore, 12);
740 + assert.equal(Queue.sizeUnsafe(lines), 1);
741 + }),
742 + ),
743 + ),
744 + ),
745 +);
746 +
747 +it.effect.each([
748 + { phase: "the retry waits", beforeClose: Duration.zero, retries: 0 },
749 + { phase: "a retried reader starts", beforeClose: Duration.seconds(1), retries: 1 },
750 +])("stops retrying the reader when the client closes while $phase", ({ beforeClose, retries }) =>
751 + withWorkers(
752 + { silent: true },
753 + Effect.gen(function* () {
754 + const startedAtClose = yield* withReaderClient((sql) =>
755 + Effect.gen(function* () {
756 + const read = yield* Effect.forkChild(readConnection(sql), { startImmediately: true });
757 + yield* TestClock.adjust("10 seconds");
758 + assert.deepEqual(yield* Fiber.join(read), { count: 1, connection: ["writer"] });
759 +
760 + const startedBefore = startedWorkers.length;
761 + yield* TestClock.adjust(beforeClose);
762 + assert.equal(startedWorkers.length - startedBefore, retries);
763 + return startedWorkers.length;
764 + }),
765 + );
766 + yield* TestClock.adjust("10 minutes");
767 + assert.equal(startedWorkers.length, startedAtClose);
768 + }),
769 + ),
770 +);
771 +
772 +it.effect.each([
773 + {
774 + when: "the database is not in WAL mode",
775 + journalMode: "DELETE",
776 + workers: {},
777 + starts: 1,
778 + logged: [["Warn", unavailableWarning, { reason: notWal }]],
779 + },
780 + {
781 + when: "a retry finds the database out of WAL mode",
782 + journalMode: "WAL",
783 + workers: { failWith: "reader start failed" },
784 + starts: 2,
785 + logged: [
786 + ["Warn", unavailableWarning, { reason: "Database reader failed: reader start failed" }],
787 + [
788 + "Info",
789 + "SQLite reader worker retries stopped permanently; reads use the writer",
790 + { reason: notWal },
791 + ],
792 + ],
793 + },
794 +])(
795 + "keeps readers off without further retries when $when",
796 + ({ journalMode, workers, starts, logged }) =>
797 + withLogLines((lines) =>
798 + withFileClient((sql) =>
799 + Effect.gen(function* () {
800 + yield* sql.unsafe(`PRAGMA journal_mode = ${journalMode}`);
801 + yield* sql`CREATE TABLE entries(name TEXT NOT NULL)`;
802 + yield* sql`INSERT INTO entries VALUES ('kept')`;
803 + const startedBefore = startedWorkers.length;
804 +
805 + const failed = yield* withWorkers(workers, readConnection(sql));
806 + assert.deepEqual(failed, { count: 1, connection: ["writer"] });
807 + yield* sql`PRAGMA journal_mode = DELETE`;
808 + yield* TestClock.adjust("1 second");
809 + yield* Effect.promise(() => workerExits.get(startedWorkers.at(-1)!)!);
810 + yield* TestClock.adjust("1 hour");
811 + assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["writer"] });
812 + assert.equal(startedWorkers.length, startedBefore + starts);
813 + assert.deepEqual(yield* Queue.clear(lines), logged);
814 + }),
815 + ),
816 + ),
817 +);
818 +
819 it.effect("fails with a typed SqlError when the database file is not SQLite", () =>
820 Effect.gen(function* () {
821 const fs = yield* FileSystem.FileSystem;
packages/shared/src/nodeSqliteClient.ts
+54 -5
@@ -193,6 +193,9 @@ const hasLargeBinding = (params: ReadonlyArray<unknown>): boolean => {
193 }
194 };
195
196 +// A reader refuses a database outside WAL mode with this code; retrying cannot change that.
197 +const NOT_WAL = "ERR_SQLITE_NOT_WAL";
198 +
199 // Inline worker source also runs in bundles and single-executable builds without module files.
200 const workerSource = `
201 const { parentPort, workerData } = require("node:worker_threads");
@@ -214,7 +217,9 @@ try {
217 // Reads the header, so a file that is not a database fails here rather than on first use.
218 const mode = String(next.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
219 // Without WAL a reader's shared lock blocks the writer's commits, so refuse.
217 - if (workerData.requireWal && mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
220 + if (workerData.requireWal && mode !== "wal") {
221 + throw Object.assign(new Error("database uses journal_mode " + mode + ", not wal"), { code: ${JSON.stringify(NOT_WAL)} });
222 + }
223 db = next;
224 parentPort.postMessage({ type: "ready" });
225 } catch (error) {
@@ -345,6 +350,8 @@ const endsReadTransaction = (sql: string) => /^\s*ROLLBACK\b|^\s*COMMIT\b/i.test
350 // WAL readers run concurrently; one worker serialized every read behind the slowest transaction.
351 const READER_POOL_SIZE = 4;
352 const READER_STARTUP_TIMEOUT = Duration.seconds(10);
353 +const READER_RETRY_DELAY = Duration.seconds(1);
354 +const READER_RETRY_MAX_DELAY = Duration.minutes(1);
355
356 // Bind transactions to one worker; fail after its exit instead of resuming outside the transaction on a replacement.
357 const makeWorkerConnection = Effect.fnUntraced(function* (options: {
@@ -566,9 +573,44 @@ const makeWorkerConnection = Effect.fnUntraced(function* (options: {
573 });
574 });
575
569 -// A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
576 +const startFailure = (error: SqlError) => {
577 + const cause = error.reason.cause;
578 + return {
579 + reason: cause instanceof Error ? cause.message : error.message,
580 + permanent: Predicate.hasProperty(cause, "code") && cause.code === NOT_WAL,
581 + };
582 +};
583 +
584 +// A reader that cannot start turns off the whole pool, so routing never picks a dead slot, until
585 +// one background retry starts it again.
586 const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavailable: boolean }) {
587 const acquire = yield* makeWorkerConnection({ filename, role: "reader", readOnly: true });
588 + const scope = yield* Effect.scope;
589 + const context = yield* Effect.context<never>();
590 +
591 + const recover = (delay: Duration.Duration): Effect.Effect<void> =>
592 + Effect.sleep(delay).pipe(
593 + Effect.andThen(acquire),
594 + Effect.matchEffect({
595 + onSuccess: () =>
596 + Effect.suspend(() => {
597 + pool.unavailable = false;
598 + return Effect.logInfo(
599 + "SQLite reader worker available again; reads use the reader workers",
600 + );
601 + }),
602 + onFailure: (error) => {
603 + const { reason, permanent } = startFailure(error);
604 + return permanent
605 + ? Effect.logInfo(
606 + "SQLite reader worker retries stopped permanently; reads use the writer",
607 + { reason },
608 + )
609 + : recover(Duration.min(Duration.times(delay, 2), READER_RETRY_MAX_DELAY));
610 + },
611 + }),
612 + );
613 +
614 return Effect.suspend((): Effect.Effect<Connection | undefined> =>
615 pool.unavailable
616 ? Effect.undefined
@@ -577,10 +619,17 @@ const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavai
619 Effect.catch((error) => {
620 if (pool.unavailable) return Effect.undefined;
621 pool.unavailable = true;
580 - const cause = error.reason.cause;
622 + const { reason, permanent } = startFailure(error);
623 + // Pool state belongs to the client: never log or retry with the failed read's services.
624 return Effect.logWarning("SQLite reader worker unavailable; reads use the writer", {
582 - reason: cause instanceof Error ? cause.message : error.message,
583 - }).pipe(Effect.as(undefined));
625 + reason,
626 + }).pipe(
627 + Effect.andThen(
628 + permanent ? Effect.void : Effect.forkIn(recover(READER_RETRY_DELAY), scope),
629 + ),
630 + Effect.updateContext((_: Context.Context<never>) => context),
631 + Effect.as(undefined),
632 + );
633 }),
634 ),
635 );