The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 13:30 UTC
fix(shared): the SQLite reader pool comes back after a failed start (#30818)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Navid Nadali committed
Nov 15, 2026 at 00:20 UTC
c0561c5c3d75f5fa2017d4ba9d75124ad63fe897
2 files changed
+250
-5
packages/shared/src/nodeSqliteClient.test.ts
+196
@@ -4,12 +4,15 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
4
import { assert, it } from "@effect/vitest";
5
import * as Deferred from "effect/Deferred";
6
import * as Cause from "effect/Cause";
7
+import * as Duration from "effect/Duration";
8
import * as Effect from "effect/Effect";
9
import * as Exit from "effect/Exit";
10
import * as Fiber from "effect/Fiber";
11
import * as FileSystem from "effect/FileSystem";
12
import * as Layer from "effect/Layer";
13
+import * as Logger from "effect/Logger";
14
import * as Path from "effect/Path";
15
+import * as Queue from "effect/Queue";
16
import * as SqlClient from "effect/sql/SqlClient";
17
import * as SqlError from "effect/sql/SqlError";
18
import * as Statement from "effect/sql/Statement";
@@ -22,6 +25,7 @@ import { vi } from "vite-plus/test";
25
import * as SqliteClient from "./nodeSqliteClient.ts";
26
27
const startedWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
28
+const workerExits = vi.hoisted(() => new WeakMap<NodeWorkerThreads.Worker, Promise<void>>());
29
const silentWorkers = vi.hoisted(() => ({ on: false, readyOnTerminate: false }));
30
const failingWorkers = vi.hoisted(() => ({ message: undefined as string | undefined }));
31
const checkpointWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
@@ -38,6 +42,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
42
: args) as ConstructorParameters<typeof actual.Worker>),
43
);
44
startedWorkers.push(this);
45
+ workerExits.set(this, new Promise((resolve) => this.once("exit", () => resolve())));
46
if (args[1]?.workerData?.frames !== undefined) checkpointWorkers.push(this);
47
}
48
override terminate() {
@@ -620,6 +625,197 @@ it.effect("uses the writer when the reader cannot open the database", () =>
625
),
626
);
627
628
+const withLogLines = <A, E, R>(
629
+ use: (lines: Queue.Queue<ReadonlyArray<unknown>>) => Effect.Effect<A, E, R>,
630
+) =>
631
+ Effect.gen(function* () {
632
+ const lines = yield* Queue.unbounded<ReadonlyArray<unknown>>();
633
+ const logger = Logger.make(({ logLevel, message }) => {
634
+ Queue.offerUnsafe(lines, [logLevel, ...(message as ReadonlyArray<unknown>)]);
635
+ });
636
+ return yield* use(lines).pipe(
637
+ Effect.provide(Logger.layer([logger], { mergeWithExisting: false })),
638
+ );
639
+ });
640
+
641
+const readConnection = (sql: SqlClient.SqlClient) =>
642
+ Effect.gen(function* () {
643
+ const spans: Array<Tracer.Span> = [];
644
+ const observe: Statement.Transformer = (statement, _sql, _fiber, span) =>
645
+ Effect.sync(() => {
646
+ spans.push(span);
647
+ return statement;
648
+ });
649
+ const count = yield* countEntries(sql).pipe(
650
+ SqliteClient.readOnly,
651
+ Effect.provideService(Statement.CurrentTransformer, observe),
652
+ Effect.provideService(Statement.SpanPropagationEnabled, true),
653
+ );
654
+ return { count, connection: spans.map((span) => span.attributes.get("db.connection")) };
655
+ });
656
+
657
+const withWorkers = <A, E, R>(
658
+ mode: { readonly failWith?: string; readonly silent?: boolean },
659
+ effect: Effect.Effect<A, E, R>,
660
+) =>
661
+ Effect.acquireUseRelease(
662
+ Effect.sync(() => {
663
+ failingWorkers.message = mode.failWith;
664
+ silentWorkers.on = mode.silent ?? false;
665
+ }),
666
+ () => effect,
667
+ () =>
668
+ Effect.sync(() => {
669
+ failingWorkers.message = undefined;
670
+ silentWorkers.on = false;
671
+ }),
672
+ );
673
+
674
+const unavailableWarning = "SQLite reader worker unavailable; reads use the writer";
675
+
676
+const notWal = "database uses journal_mode delete, not wal";
677
+
678
+it.effect.each([
679
+ {
680
+ failure: "a crash",
681
+ workers: { failWith: "reader start failed" },
682
+ startup: Duration.zero,
683
+ reason: "Database reader failed: reader start failed",
684
+ },
685
+ {
686
+ failure: "a startup timeout",
687
+ workers: { silent: true },
688
+ startup: Duration.seconds(10),
689
+ reason: "Database reader did not open the database within 10000 ms",
690
+ },
691
+])(
692
+ "reads on the writer after $failure until a retry starts the reader again",
693
+ ({ workers, startup, reason }) =>
694
+ withLogLines((lines) =>
695
+ withReaderClient((sql) =>
696
+ Effect.gen(function* () {
697
+ const failed = yield* withWorkers(
698
+ workers,
699
+ Effect.gen(function* () {
700
+ const read = yield* Effect.forkChild(readConnection(sql), {
701
+ startImmediately: true,
702
+ });
703
+ yield* TestClock.adjust(startup);
704
+ return yield* Fiber.join(read);
705
+ }),
706
+ );
707
+ assert.deepEqual(failed, { count: 1, connection: ["writer"] });
708
+ const startedBefore = startedWorkers.length;
709
+ assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["writer"] });
710
+ assert.equal(startedWorkers.length, startedBefore);
711
+ assert.deepEqual(yield* Queue.take(lines), ["Warn", unavailableWarning, { reason }]);
712
+
713
+ yield* TestClock.adjust("1 second");
714
+ assert.equal(startedWorkers.length, startedBefore + 1);
715
+ assert.deepEqual(yield* Queue.take(lines), [
716
+ "Info",
717
+ "SQLite reader worker available again; reads use the reader workers",
718
+ ]);
719
+ assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["reader"] });
720
+ assert.equal(Queue.sizeUnsafe(lines), 0);
721
+ }),
722
+ ),
723
+ ),
724
+);
725
+
726
+it.effect("backs off a reader that never starts without logging each retry", () =>
727
+ withLogLines((lines) =>
728
+ withReaderClient((sql) =>
729
+ withWorkers(
730
+ { silent: true },
731
+ Effect.gen(function* () {
732
+ const read = yield* Effect.forkChild(readConnection(sql), { startImmediately: true });
733
+ yield* TestClock.adjust("10 seconds");
734
+ assert.deepEqual(yield* Fiber.join(read), { count: 1, connection: ["writer"] });
735
+
736
+ const startedBefore = startedWorkers.length;
737
+ // Each retry waits 10 s for startup; the gaps between them double from 1 s up to a minute.
738
+ yield* TestClock.adjust("10 minutes");
739
+ assert.equal(startedWorkers.length - startedBefore, 12);
740
+ assert.equal(Queue.sizeUnsafe(lines), 1);
741
+ }),
742
+ ),
743
+ ),
744
+ ),
745
+);
746
+
747
+it.effect.each([
748
+ { phase: "the retry waits", beforeClose: Duration.zero, retries: 0 },
749
+ { phase: "a retried reader starts", beforeClose: Duration.seconds(1), retries: 1 },
750
+])("stops retrying the reader when the client closes while $phase", ({ beforeClose, retries }) =>
751
+ withWorkers(
752
+ { silent: true },
753
+ Effect.gen(function* () {
754
+ const startedAtClose = yield* withReaderClient((sql) =>
755
+ Effect.gen(function* () {
756
+ const read = yield* Effect.forkChild(readConnection(sql), { startImmediately: true });
757
+ yield* TestClock.adjust("10 seconds");
758
+ assert.deepEqual(yield* Fiber.join(read), { count: 1, connection: ["writer"] });
759
+
760
+ const startedBefore = startedWorkers.length;
761
+ yield* TestClock.adjust(beforeClose);
762
+ assert.equal(startedWorkers.length - startedBefore, retries);
763
+ return startedWorkers.length;
764
+ }),
765
+ );
766
+ yield* TestClock.adjust("10 minutes");
767
+ assert.equal(startedWorkers.length, startedAtClose);
768
+ }),
769
+ ),
770
+);
771
+
772
+it.effect.each([
773
+ {
774
+ when: "the database is not in WAL mode",
775
+ journalMode: "DELETE",
776
+ workers: {},
777
+ starts: 1,
778
+ logged: [["Warn", unavailableWarning, { reason: notWal }]],
779
+ },
780
+ {
781
+ when: "a retry finds the database out of WAL mode",
782
+ journalMode: "WAL",
783
+ workers: { failWith: "reader start failed" },
784
+ starts: 2,
785
+ logged: [
786
+ ["Warn", unavailableWarning, { reason: "Database reader failed: reader start failed" }],
787
+ [
788
+ "Info",
789
+ "SQLite reader worker retries stopped permanently; reads use the writer",
790
+ { reason: notWal },
791
+ ],
792
+ ],
793
+ },
794
+])(
795
+ "keeps readers off without further retries when $when",
796
+ ({ journalMode, workers, starts, logged }) =>
797
+ withLogLines((lines) =>
798
+ withFileClient((sql) =>
799
+ Effect.gen(function* () {
800
+ yield* sql.unsafe(`PRAGMA journal_mode = ${journalMode}`);
801
+ yield* sql`CREATE TABLE entries(name TEXT NOT NULL)`;
802
+ yield* sql`INSERT INTO entries VALUES ('kept')`;
803
+ const startedBefore = startedWorkers.length;
804
+
805
+ const failed = yield* withWorkers(workers, readConnection(sql));
806
+ assert.deepEqual(failed, { count: 1, connection: ["writer"] });
807
+ yield* sql`PRAGMA journal_mode = DELETE`;
808
+ yield* TestClock.adjust("1 second");
809
+ yield* Effect.promise(() => workerExits.get(startedWorkers.at(-1)!)!);
810
+ yield* TestClock.adjust("1 hour");
811
+ assert.deepEqual(yield* readConnection(sql), { count: 1, connection: ["writer"] });
812
+ assert.equal(startedWorkers.length, startedBefore + starts);
813
+ assert.deepEqual(yield* Queue.clear(lines), logged);
814
+ }),
815
+ ),
816
+ ),
817
+);
818
+
819
it.effect("fails with a typed SqlError when the database file is not SQLite", () =>
820
Effect.gen(function* () {
821
const fs = yield* FileSystem.FileSystem;
packages/shared/src/nodeSqliteClient.ts
+54
-5
@@ -193,6 +193,9 @@ const hasLargeBinding = (params: ReadonlyArray<unknown>): boolean => {
193
}
194
};
195
196
+// A reader refuses a database outside WAL mode with this code; retrying cannot change that.
197
+const NOT_WAL = "ERR_SQLITE_NOT_WAL";
198
+
199
// Inline worker source also runs in bundles and single-executable builds without module files.
200
const workerSource = `
201
const { parentPort, workerData } = require("node:worker_threads");
@@ -214,7 +217,9 @@ try {
217
// Reads the header, so a file that is not a database fails here rather than on first use.
218
const mode = String(next.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
219
// Without WAL a reader's shared lock blocks the writer's commits, so refuse.
217
- if (workerData.requireWal && mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
220
+ if (workerData.requireWal && mode !== "wal") {
221
+ throw Object.assign(new Error("database uses journal_mode " + mode + ", not wal"), { code: ${JSON.stringify(NOT_WAL)} });
222
+ }
223
db = next;
224
parentPort.postMessage({ type: "ready" });
225
} catch (error) {
@@ -345,6 +350,8 @@ const endsReadTransaction = (sql: string) => /^\s*ROLLBACK\b|^\s*COMMIT\b/i.test
350
// WAL readers run concurrently; one worker serialized every read behind the slowest transaction.
351
const READER_POOL_SIZE = 4;
352
const READER_STARTUP_TIMEOUT = Duration.seconds(10);
353
+const READER_RETRY_DELAY = Duration.seconds(1);
354
+const READER_RETRY_MAX_DELAY = Duration.minutes(1);
355
356
// Bind transactions to one worker; fail after its exit instead of resuming outside the transaction on a replacement.
357
const makeWorkerConnection = Effect.fnUntraced(function* (options: {
@@ -566,9 +573,44 @@ const makeWorkerConnection = Effect.fnUntraced(function* (options: {
573
});
574
});
575
569
-// A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
576
+const startFailure = (error: SqlError) => {
577
+ const cause = error.reason.cause;
578
+ return {
579
+ reason: cause instanceof Error ? cause.message : error.message,
580
+ permanent: Predicate.hasProperty(cause, "code") && cause.code === NOT_WAL,
581
+ };
582
+};
583
+
584
+// A reader that cannot start turns off the whole pool, so routing never picks a dead slot, until
585
+// one background retry starts it again.
586
const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavailable: boolean }) {
587
const acquire = yield* makeWorkerConnection({ filename, role: "reader", readOnly: true });
588
+ const scope = yield* Effect.scope;
589
+ const context = yield* Effect.context<never>();
590
+
591
+ const recover = (delay: Duration.Duration): Effect.Effect<void> =>
592
+ Effect.sleep(delay).pipe(
593
+ Effect.andThen(acquire),
594
+ Effect.matchEffect({
595
+ onSuccess: () =>
596
+ Effect.suspend(() => {
597
+ pool.unavailable = false;
598
+ return Effect.logInfo(
599
+ "SQLite reader worker available again; reads use the reader workers",
600
+ );
601
+ }),
602
+ onFailure: (error) => {
603
+ const { reason, permanent } = startFailure(error);
604
+ return permanent
605
+ ? Effect.logInfo(
606
+ "SQLite reader worker retries stopped permanently; reads use the writer",
607
+ { reason },
608
+ )
609
+ : recover(Duration.min(Duration.times(delay, 2), READER_RETRY_MAX_DELAY));
610
+ },
611
+ }),
612
+ );
613
+
614
return Effect.suspend((): Effect.Effect<Connection | undefined> =>
615
pool.unavailable
616
? Effect.undefined
@@ -577,10 +619,17 @@ const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavai
619
Effect.catch((error) => {
620
if (pool.unavailable) return Effect.undefined;
621
pool.unavailable = true;
580
- const cause = error.reason.cause;
622
+ const { reason, permanent } = startFailure(error);
623
+ // Pool state belongs to the client: never log or retry with the failed read's services.
624
return Effect.logWarning("SQLite reader worker unavailable; reads use the writer", {
582
- reason: cause instanceof Error ? cause.message : error.message,
583
- }).pipe(Effect.as(undefined));
625
+ reason,
626
+ }).pipe(
627
+ Effect.andThen(
628
+ permanent ? Effect.void : Effect.forkIn(recover(READER_RETRY_DELAY), scope),
629
+ ),
630
+ Effect.updateContext((_: Context.Context<never>) => context),
631
+ Effect.as(undefined),
632
+ );
633
}),
634
),
635
);