@devries-cameron20 / t3code Mirror · read-only / commits / ce268642cc
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 18:30 UTC

fix(server): a slow database write no longer freezes the server (#35728)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

sharath committed Nov 13, 2026 at 05:39 UTC ce268642cc4472e0671c4b6b8e29d2a21e383602
7 files changed +418 -841
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+31 -31
@@ -1,4 +1,4 @@
1 -import * as NodeSqlite from "node:sqlite";
1 +import type * as NodeWorkerThreads from "node:worker_threads";
2 import * as NodeServices from "@effect/platform-node/NodeServices";
3 import { assert, it } from "@effect/vitest";
4 import {
@@ -73,6 +73,29 @@ import * as ProjectStore from "./ProjectStore.ts";
73 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
74 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
75
76 +const workerStatements = vi.hoisted(
77 + (): Array<{ readonly readOnly: boolean; readonly sql: string }> => [],
78 +);
79 +
80 +vi.mock("node:worker_threads", async (importOriginal) => {
81 + const actual = await importOriginal<typeof NodeWorkerThreads>();
82 + class RecordedWorker extends actual.Worker {
83 + readonly #readOnly: boolean;
84 + constructor(...args: ConstructorParameters<typeof actual.Worker>) {
85 + super(...args);
86 + this.#readOnly = args[1]?.workerData?.readOnly === true;
87 + }
88 + override postMessage(...args: Parameters<NodeWorkerThreads.Worker["postMessage"]>) {
89 + const [message] = args;
90 + if (typeof message === "object" && message !== null && typeof message.sql === "string") {
91 + workerStatements.push({ readOnly: this.#readOnly, sql: message.sql });
92 + }
93 + return super.postMessage(...args);
94 + }
95 + }
96 + return { ...actual, Worker: RecordedWorker };
97 +});
98 +
99 const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
100 const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
101
@@ -193,45 +216,22 @@ it.effect("discovers on the reader worker and deletes on the writer", () =>
216 yield* Effect.gen(function* () {
217 const sql = yield* SqlClient.SqlClient;
218 yield* seedCompaction();
196 - // Establish the reader before recording main-thread statements.
219 + // Establish the reader before recording statements.
220 yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
198 - const reads: string[] = [];
199 - const writes: string[] = [];
200 - const all = NodeSqlite.StatementSync.prototype.all;
201 - const run = NodeSqlite.StatementSync.prototype.run;
202 - yield* Effect.acquireRelease(
203 - Effect.sync(() => {
204 - const readSpy = vi
205 - .spyOn(NodeSqlite.StatementSync.prototype, "all")
206 - .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
207 - reads.push(this.sourceSQL);
208 - return all.apply(this, params);
209 - });
210 - const writeSpy = vi
211 - .spyOn(NodeSqlite.StatementSync.prototype, "run")
212 - .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
213 - writes.push(this.sourceSQL);
214 - return run.apply(this, params);
215 - });
216 - return { readSpy, writeSpy };
217 - }),
218 - ({ readSpy, writeSpy }) =>
219 - Effect.sync(() => {
220 - readSpy.mockRestore();
221 - writeSpy.mockRestore();
222 - }),
223 - );
221 + workerStatements.length = 0;
222 const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
223 const result = yield* maintenance.pruneEventStore();
224 assert.equal(result.deletedEventCount, 150);
225 assert.equal(result.deletedReceiptCount, 100);
226 + const writer = workerStatements.filter((statement) => !statement.readOnly);
227 // Only the in-transaction check for rows appended during the pass reads on the writer.
228 assert.deepEqual(
230 - reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")),
229 + writer.filter(({ sql }) => /\bSELECT\b/.test(sql) && !sql.includes("AS prompt_sequence")),
230 [],
231 );
233 - assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
234 - assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
232 + assert(workerStatements.some(({ readOnly, sql }) => readOnly && /\bSELECT\b/.test(sql)));
233 + assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_events")));
234 + assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_command_receipts")));
235 }).pipe(
236 Effect.provide(
237 ProjectionMaintenance.layer.pipe(
apps/server/src/persistence/Sqlite.test.ts
+1 -74
@@ -2,7 +2,6 @@
2 import * as NodeFS from "node:fs";
3 import * as NodeOS from "node:os";
4 import * as NodePath from "node:path";
5 -import * as NodeSqlite from "node:sqlite";
5 import type * as NodeWorkerThreads from "node:worker_threads";
6
7 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -23,7 +22,6 @@ import * as ServerOwnership from "../serverOwnership.ts";
22 import { migrationManifest, runMigrations } from "./Migrations.ts";
23 import * as SqlitePersistence from "./Sqlite.ts";
24
26 -const workers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
25 const workerData = vi.hoisted((): Array<unknown> => []);
26
27 vi.mock("node:worker_threads", async (importOriginal) => {
@@ -31,84 +29,12 @@ vi.mock("node:worker_threads", async (importOriginal) => {
29 class RecordedWorker extends actual.Worker {
30 constructor(...args: ConstructorParameters<typeof actual.Worker>) {
31 super(...args);
34 - workers.push(this);
32 workerData.push(args[1]?.workerData);
33 }
34 }
35 return { ...actual, Worker: RecordedWorker };
36 });
37
41 -it.effect("keeps writes moving past a 48 MiB WAL while a reader retains its snapshot", () => {
42 - const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-burst-"));
43 - const dbPath = NodePath.join(tempDir, "state.sqlite");
44 - const persistence = SqlitePersistence.layerFromPath(dbPath).pipe(
45 - Layer.provide(NodeServices.layer),
46 - );
47 -
48 - return Effect.gen(function* () {
49 - const sql = yield* SqlClient.SqlClient;
50 - const checkpointWorker = workers.at(-1)!;
51 - yield* sql`CREATE TABLE burst_probe(id INTEGER PRIMARY KEY, payload BLOB)`;
52 - yield* sql`INSERT INTO burst_probe VALUES (1, NULL)`;
53 - yield* sql`PRAGMA wal_checkpoint(TRUNCATE)`;
54 - const reader = yield* Effect.acquireRelease(
55 - Effect.sync(() => new NodeSqlite.DatabaseSync(dbPath, { readOnly: true })),
56 - (database) => Effect.sync(() => database.close()),
57 - );
58 - reader.exec("BEGIN");
59 - assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);
60 -
61 - yield* sql.withTransaction(
62 - sql`INSERT INTO burst_probe VALUES (2, zeroblob(${48 * 1024 * 1024}))`,
63 - );
64 - const state = yield* Effect.callback<{
65 - readonly log: number;
66 - readonly checkpointed: number;
67 - readonly blocked: boolean;
68 - }>((resume) => {
69 - const onMessage = (message: {
70 - readonly type?: string;
71 - readonly log?: number;
72 - readonly checkpointed?: number;
73 - readonly blocked?: boolean;
74 - }) => {
75 - if (
76 - message.type === "checkpoint" &&
77 - typeof message.log === "number" &&
78 - message.log > 10_000 &&
79 - typeof message.checkpointed === "number" &&
80 - typeof message.blocked === "boolean"
81 - ) {
82 - checkpointWorker.off("message", onMessage);
83 - resume(
84 - Effect.succeed({
85 - log: message.log,
86 - checkpointed: message.checkpointed,
87 - blocked: message.blocked,
88 - }),
89 - );
90 - }
91 - };
92 - checkpointWorker.on("message", onMessage);
93 - checkpointWorker.ref();
94 - // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
95 - checkpointWorker.postMessage("barrier");
96 - return Effect.sync(() => checkpointWorker.off("message", onMessage));
97 - });
98 - assert.isBelow(state.checkpointed, state.log);
99 - assert.isFalse(state.blocked);
100 -
101 - yield* sql.withTransaction(sql`INSERT INTO burst_probe VALUES (3, NULL)`);
102 - assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);
103 - reader.exec("COMMIT");
104 - assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 3);
105 - }).pipe(
106 - Effect.scoped,
107 - Effect.provide(persistence),
108 - Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
109 - );
110 -});
111 -
38 // Linux names an open, already-deleted file "<path> (deleted)" in /proc/self/fd.
39 // SQLite names its temp files "etilqs_*" and, without TMPDIR, puts them in
40 // /var/tmp while os.tmpdir() is /tmp, so match the name, not the directory.
@@ -223,6 +149,7 @@ it.effect("upgrades an existing database on a read-write worker connection", ()
149 assert.deepInclude(workerData.slice(startedBefore), {
150 filename: dbPath,
151 readOnly: false,
152 + requireWal: false,
153 allowExtension: false,
154 prepareCacheSize: 200,
155 });
apps/server/src/persistence/Sqlite.ts
+5 -27
@@ -64,26 +64,6 @@ const runMigrationsAsOwner = Effect.fn("SqlitePersistence.runMigrationsAsOwner")
64 yield* runMigrations();
65 }, Effect.scoped);
66
67 -// Index builds over large tables take minutes; on the main connection they block the event loop,
68 -// so requests hang instead of getting the listener's 503. A fresh database stays on the main
69 -// connection, which must create the first table for its auto_vacuum setting to apply.
70 -const runMigrationsOffMainThread = Effect.fn("SqlitePersistence.runMigrationsOffMainThread")(
71 - function* (dbPath: string) {
72 - const sql = yield* SqlClient.SqlClient;
73 - const tables = yield* sql<{
74 - readonly count: number;
75 - }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
76 - if (!tables[0]?.count || !(yield* hasPendingMigrations())) return yield* runMigrations();
77 - return yield* Effect.gen(function* () {
78 - const worker = yield* SqlClient.SqlClient;
79 - yield* worker`PRAGMA foreign_keys = ON;`;
80 - yield* worker`PRAGMA synchronous = NORMAL;`;
81 - yield* worker`PRAGMA temp_store = MEMORY;`;
82 - return yield* runMigrations();
83 - }).pipe(Effect.provide(NodeSqliteClient.layerWorker(dbPath)));
84 - },
85 -);
86 -
67 const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
68 Layer.effectDiscard(
69 Effect.gen(function* () {
@@ -97,12 +77,12 @@ const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
77 }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
78 if (tables[0]?.count === 0) yield* sql`PRAGMA auto_vacuum = INCREMENTAL;`;
79 yield* sql`PRAGMA journal_mode = WAL;`;
100 - // FULL fsyncs every event commit on the main thread; NORMAL syncs only at WAL checkpoints.
80 + // FULL fsyncs every event commit; NORMAL syncs only at WAL checkpoints.
81 // App crashes lose nothing; OS crashes or power loss can roll back post-checkpoint commits without corruption.
82 yield* sql`PRAGMA synchronous = NORMAL;`;
83 // macOS fsync misses drive caches; F_FULLFSYNC flushes them at checkpoints and is ignored elsewhere.
84 yield* sql`PRAGMA checkpoint_fullfsync = ON;`;
105 - // Keep statement journals and sort spills off the main thread's disk path.
85 + // Keep statement journals and sort spills off the disk.
86 // Large statements and open savepoints can increase peak memory use.
87 yield* sql`PRAGMA temp_store = MEMORY;`;
88 // PASSIVE checkpoints retain peak WAL size until the last connection closes.
@@ -131,8 +111,8 @@ const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<neve
111 NodeSqliteClient.layer({
112 filename: dbPath,
113 readerWorker: true,
134 - // Run checkpoint fsyncs off the event loop so slow disk flushes cannot stall clients.
135 - checkpointWorker: true,
114 + // A slow disk then delays writes instead of freezing every client.
115 + writerWorker: true,
116 spanAttributes: {
117 "db.name": path.basename(dbPath),
118 "service.name": "t3code-server",
@@ -152,9 +132,7 @@ const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<neve
132 );
133 }, Layer.unwrap);
134
155 -export const layerFromPath = makeLayerFromPath((dbPath) =>
156 - makeLayerSetup(runMigrationsOffMainThread(dbPath)),
157 -);
135 +export const layerFromPath = makeLayerFromPath(() => layerSetup);
136
137 export const layerMemory = Layer.provideMerge(
138 layerSetup,
apps/server/src/project/ProjectService.test.ts
+6
@@ -552,6 +552,12 @@ it.layer(layerTest)("ProjectService", (it) => {
552 title: "Delete retry",
553 workspaceRoot: "/work/delete-retry",
554 });
555 + // Enrichment landing between the two deletes would change the project the retry reads.
556 + yield* waitForProject(
557 + service,
558 + projectId,
559 + (project) => project.faviconPath !== null && project.repositoryIdentity !== null,
560 + );
561 const input = { commandId: CommandId.make("command:delete-retry:delete"), projectId };
562 const deleted = yield* service.delete(input);
563 assert.deepEqual(yield* service.delete(input), deleted);
docs/operations/observability.md
+3 -2
@@ -57,8 +57,9 @@ The `TraceRecord`, `EffectTraceRecord`, and `OtlpTraceRecord` schemas live in
57 `packages/shared/src/observability.ts`.
58
59 SQL statement spans (named `sqlite`) and `sql.transaction` spans carry `db.connection` and
60 -`db.lock_wait_ms`. `db.connection` is `writer` for the main connection, which runs on the event loop,
61 -or `reader` for the read-only connection in a worker thread. `db.lock_wait_ms` is the part of the
60 +`db.lock_wait_ms`. `db.connection` is `writer` for the read-write connection or `reader` for a
61 +read-only one. For a database file both run in worker threads; an in-memory database has only a
62 +writer, on the event loop. `db.lock_wait_ms` is the part of the
63 span spent waiting for that connection, behind other SQL. That wait does not block the event loop,
64 so subtract it before you blame a slow span on the query. Statements inside a transaction use its
65 connection and do not carry these attributes.
packages/shared/src/nodeSqliteClient.test.ts
+87 -89
@@ -23,6 +23,7 @@ import * as SqliteClient from "./nodeSqliteClient.ts";
23 const startedWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
24 const silentWorkers = vi.hoisted(() => ({ on: false, readyOnTerminate: false }));
25 const failingWorkers = vi.hoisted(() => ({ message: undefined as string | undefined }));
26 +const checkpointWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
27
28 vi.mock("node:worker_threads", async (importOriginal) => {
29 const actual = await importOriginal<typeof NodeWorkerThreads>();
@@ -36,6 +37,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
37 : args) as ConstructorParameters<typeof actual.Worker>),
38 );
39 startedWorkers.push(this);
40 + if (args[1]?.workerData?.frames !== undefined) checkpointWorkers.push(this);
41 }
42 override terminate() {
43 // A "ready" already queued when termination starts reaches the parent before "exit".
@@ -222,7 +224,7 @@ it.effect("waits for another process's lock while the database opens", () =>
224 yield* Effect.gen(function* () {
225 const sql = yield* SqlClient.SqlClient;
226 assert.deepEqual(yield* sql`SELECT COUNT(*) FROM entries`.values, [[0]]);
225 - }).pipe(Effect.provide(SqliteClient.layer({ filename, checkpointWorker: true })));
227 + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
228 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
229 );
230
@@ -244,6 +246,52 @@ it.effect("reports a transaction blocked by another writer as a lock timeout", (
246 }).pipe(Effect.provide(NodeServices.layer)),
247 );
248
249 +it.effect("lets the event loop run while a write waits for another connection's lock", () =>
250 + Effect.gen(function* () {
251 + const { filename, other } = yield* makeTempDatabase;
252 + yield* Effect.gen(function* () {
253 + const sql = yield* SqlClient.SqlClient;
254 + yield* sql`PRAGMA busy_timeout = 5000`;
255 + yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
256 + const write = yield* Effect.forkChild(
257 + sql`UPDATE counters SET value = value + 1 WHERE id = 1`,
258 + { startImmediately: true },
259 + );
260 + // A writer on the event loop sleeps in SQLite's busy handler here, so this commit
261 + // cannot run until that write has already timed out.
262 + yield* eventLoopTurn;
263 + yield* Effect.sync(() => other.exec("UPDATE counters SET value = 10 WHERE id = 1; COMMIT"));
264 + yield* Fiber.join(write);
265 + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[11]]);
266 + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
267 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
268 +);
269 +
270 +it.effect("finishes a write already sent to the writer before its caller is interrupted", () =>
271 + Effect.gen(function* () {
272 + const { filename, other } = yield* makeTempDatabase;
273 + yield* Effect.gen(function* () {
274 + const sql = yield* SqlClient.SqlClient;
275 + yield* sql`PRAGMA busy_timeout = 5000`;
276 + yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
277 + const write = yield* Effect.forkChild(
278 + sql`UPDATE counters SET value = value + 1 WHERE id = 1`,
279 + { startImmediately: true },
280 + );
281 + yield* eventLoopTurn;
282 + const interrupted = yield* Effect.forkChild(Fiber.interrupt(write), {
283 + startImmediately: true,
284 + });
285 + yield* eventLoopTurn;
286 + // The write is waiting in the worker for the lock, so the interrupt has to wait too.
287 + assert.isUndefined(interrupted.pollUnsafe());
288 + yield* Effect.sync(() => other.exec("COMMIT"));
289 + yield* Fiber.join(interrupted);
290 + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[1]]);
291 + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
292 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
293 +);
294 +
295 it.effect("lets the writer commit while a read-only transaction is open", () =>
296 withReaderClient((sql) =>
297 Effect.gen(function* () {
@@ -420,7 +468,7 @@ it.effect("fails with a typed SqlError when the database file is not SQLite", ()
468 const filename = path.join(directory, "state.sqlite");
469 yield* fs.writeFileString(filename, "not a db\n");
470
423 - const exit = yield* Layer.build(SqliteClient.layer({ filename, checkpointWorker: true })).pipe(
471 + const exit = yield* Layer.build(SqliteClient.layer({ filename, writerWorker: true })).pipe(
472 Effect.exit,
473 );
474
@@ -514,45 +562,13 @@ it.effect.each([
562 }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))),
563 );
564
517 -const PAYLOAD = new Uint8Array(1024 * 1024).fill(7);
518 -const WAL_SIZE_LIMIT = 32 * 1024 * 1024;
519 -
520 -// Fails instead of hanging when admission holds the write: the worker retries a blocked
521 -// checkpoint on its own, so two blocked reports for one WAL generation mean a held write.
522 -const insertUnlessHeld = Effect.fnUntraced(function* (
523 - sql: SqlClient.SqlClient,
524 - worker: NodeWorkerThreads.Worker,
525 - label: string,
526 -) {
527 - const stalled = yield* Deferred.make<void>();
528 - let seen: unknown;
529 - const onMessage = (message: { type?: unknown; blocked?: unknown; generation?: unknown }) => {
530 - if (message?.type !== "checkpoint" || message.blocked !== true) return;
531 - if (message.generation === seen) Deferred.doneUnsafe(stalled, Effect.void);
532 - seen = message.generation;
533 - };
534 - worker.on("message", onMessage);
535 - const held = yield* Effect.raceFirst(
536 - sql`INSERT INTO blobs VALUES (${PAYLOAD})`.pipe(Effect.as(false)),
537 - Deferred.await(stalled).pipe(Effect.as(true)),
538 - ).pipe(Effect.ensuring(Effect.sync(() => worker.off("message", onMessage))));
539 - if (held) assert.fail(`${label} is held by admission while an outside reader pins the WAL`);
540 -});
541 -
542 -// `outside` stands in for `sqlite3 statev2.sqlite` or a backup script reading the database.
543 -const withOutsideConnection = <A, E>(
544 - use: (options: {
545 - readonly sql: SqlClient.SqlClient;
546 - readonly worker: NodeWorkerThreads.Worker;
547 - readonly outside: NodeSqlite.DatabaseSync;
548 - readonly walSize: Effect.Effect<number>;
549 - }) => Effect.Effect<A, E, SqlClient.SqlClient>,
550 -) =>
565 +it.effect("keeps writes moving while an outside reader pins the WAL, then trims the WAL", () =>
566 Effect.gen(function* () {
567 const fs = yield* FileSystem.FileSystem;
568 const path = yield* Path.Path;
569 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sqlite-pinned-wal-" });
570 const filename = path.join(directory, "state.sqlite");
571 + // Stands in for `sqlite3 statev2.sqlite` or a backup script reading the database.
572 const outside = yield* Effect.acquireRelease(
573 Effect.sync(() => new NodeSqlite.DatabaseSync(filename)),
574 (db) =>
@@ -561,64 +577,46 @@ const withOutsideConnection = <A, E>(
577 db.close();
578 }),
579 );
564 - outside.exec(`
565 - PRAGMA journal_mode = WAL;
566 - PRAGMA wal_autocheckpoint = 0;
567 - CREATE TABLE blobs(data BLOB NOT NULL);
568 - `);
580 + outside.exec("PRAGMA journal_mode = WAL; CREATE TABLE blobs(data BLOB NOT NULL)");
581 const walSize = fs.stat(`${filename}-wal`).pipe(
582 Effect.map((info) => Number(info.size)),
583 Effect.orDie,
584 );
585 + const limit = 8 * 1024 * 1024;
586 + const payload = new Uint8Array(1024 * 1024).fill(7);
587 yield* Effect.gen(function* () {
588 const sql = yield* SqlClient.SqlClient;
575 - yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT}`);
576 - return yield* use({ sql, worker: startedWorkers.at(-1)!, outside, walSize });
577 - }).pipe(Effect.provide(SqliteClient.layer({ filename, checkpointWorker: true })));
578 - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer));
579 -
580 -const openReadTransaction = (db: NodeSqlite.DatabaseSync) => {
581 - db.exec("BEGIN");
582 - db.prepare("SELECT COUNT(*) FROM blobs").get();
583 -};
584 -
585 -it.effect("keeps admitting writes while an outside read transaction pins the WAL", () =>
586 - withOutsideConnection(({ sql, worker, outside }) =>
587 - Effect.gen(function* () {
588 - yield* sql`INSERT INTO blobs VALUES (${PAYLOAD})`;
589 - openReadTransaction(outside);
590 - // 140 MiB passes both the 32 MiB size limit and the 32,768-frame ceiling.
591 - for (let write = 1; write <= 140; write++) {
592 - yield* insertUnlessHeld(sql, worker, `write ${write}`);
593 - }
594 - const rows = yield* sql<{ count: number }>`SELECT COUNT(*) AS count FROM blobs`;
595 - assert.equal(rows[0]?.count, 141);
596 - }),
597 - ),
598 -);
599 -
600 -it.effect("keeps admitting writes while a reader stops TRUNCATE, then resets the WAL", () =>
601 - withOutsideConnection(({ sql, worker, outside, walSize }) =>
602 - Effect.gen(function* () {
603 - const insert = outside.prepare("INSERT INTO blobs VALUES (?)");
604 - for (let write = 1; write <= 40; write++) insert.run(PAYLOAD);
605 - // A snapshot of an unbackfilled WAL lets PASSIVE finish but stops TRUNCATE.
606 - openReadTransaction(outside);
607 - const stopped = yield* Deferred.make<void>();
608 - const reset = yield* Deferred.make<void>();
609 - worker.on("message", (message: { type?: unknown; truncateId?: unknown; log?: unknown }) => {
610 - if (message?.type !== "checkpoint" || message.truncateId === undefined) return;
611 - Deferred.doneUnsafe(message.log === 0 ? reset : stopped, Effect.void);
612 - });
613 - // Starts a checkpoint without appending a frame past the reader's snapshot.
614 - yield* sql`DELETE FROM blobs WHERE 0`;
615 - yield* Deferred.await(stopped);
616 - yield* insertUnlessHeld(sql, worker, "a write after the failed TRUNCATE");
617 - assert.isAbove(yield* walSize, WAL_SIZE_LIMIT);
589 + yield* sql.unsafe(`PRAGMA journal_size_limit = ${limit}`);
590 + yield* sql`INSERT INTO blobs VALUES (${payload})`;
591 + outside.exec("BEGIN");
592 + outside.prepare("SELECT COUNT(*) FROM blobs").get();
593 + for (let write = 0; write < 24; write++) yield* sql`INSERT INTO blobs VALUES (${payload})`;
594 + assert.isAbove(yield* walSize, limit);
595 outside.exec("COMMIT");
619 - // No further write arrives; the worker has to retry on its own.
620 - yield* Deferred.await(reset);
621 - assert.equal(yield* walSize, 0);
622 - }),
623 - ),
596 + const checkpointer = checkpointWorkers.at(-1)!;
597 + const caughtUp = Effect.callback<void>((resume) => {
598 + const onMessage = (message: { type?: unknown; log?: unknown; checkpointed?: unknown }) => {
599 + if (
600 + message?.type === "checked" &&
601 + typeof message.log === "number" &&
602 + message.log > 0 &&
603 + message.log === message.checkpointed
604 + ) {
605 + resume(Effect.void);
606 + }
607 + };
608 + checkpointer.on("message", onMessage);
609 + checkpointer.ref();
610 + return Effect.sync(() => {
611 + checkpointer.off("message", onMessage);
612 + checkpointer.unref();
613 + });
614 + });
615 + // The worker checkpoints the released frames; the next write restarts the WAL at the limit.
616 + yield* Effect.all([caughtUp, sql`DELETE FROM blobs WHERE rowid = 1`], { concurrency: 2 });
617 + yield* sql`DELETE FROM blobs WHERE rowid = 2`;
618 + assert.isAtMost(yield* walSize, limit);
619 + assert.deepEqual(yield* sql`SELECT COUNT(*) FROM blobs`.values, [[23]]);
620 + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
621 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
622 );
packages/shared/src/nodeSqliteClient.ts
+285 -618
@@ -35,8 +35,8 @@ export interface SqliteClientConfig {
35 readonly spanAttributes?: Record<string, unknown> | undefined;
36 /** Ignored for in-memory databases, which cannot share a second connection. */
37 readonly readerWorker?: boolean | undefined;
38 - /** Ignored for in-memory databases. */
39 - readonly checkpointWorker?: boolean | undefined;
38 + /** Runs the writer and WAL checkpoints off the event loop. Ignored for in-memory databases. */
39 + readonly writerWorker?: boolean | undefined;
40 }
41
42 // SqlClient masks transaction acquisition, so readOnly carries the caller's interruption policy.
@@ -194,9 +194,15 @@ const hasLargeBinding = (params: ReadonlyArray<unknown>): boolean => {
194 };
195
196 // Inline worker source also runs in bundles and single-executable builds without module files.
197 -const readerWorkerSource = `
197 +const workerSource = `
198 const { parentPort, workerData } = require("node:worker_threads");
199 const { DatabaseSync } = require("node:sqlite");
200 +const describe = (error) => ({
201 + message: String(error && error.message ? error.message : error),
202 + code: error && error.code,
203 + errcode: error && error.errcode,
204 + errstr: error && error.errstr,
205 +});
206 let db;
207 try {
208 // A read-only open cannot be undone by SQL, unlike PRAGMA query_only.
@@ -205,13 +211,14 @@ try {
211 allowExtension: workerData.allowExtension,
212 });
213 next.exec("PRAGMA busy_timeout = 5000;");
208 - // Without WAL a reader's shared lock blocks the writer's commits, so refuse.
214 + // Reads the header, so a file that is not a database fails here rather than on first use.
215 const mode = String(next.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
210 - if (mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
216 + // Without WAL a reader's shared lock blocks the writer's commits, so refuse.
217 + if (workerData.requireWal && mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
218 db = next;
219 parentPort.postMessage({ type: "ready" });
220 } catch (error) {
214 - parentPort.postMessage({ type: "unavailable", message: String(error && error.message ? error.message : error) });
221 + parentPort.postMessage({ type: "unavailable", ...describe(error) });
222 }
223 const statements = new Map();
224 const prepare = (sql, cached) => {
@@ -226,10 +233,19 @@ const prepare = (sql, cached) => {
233 statements.set(sql, statement);
234 return statement;
235 };
229 -parentPort.on("message", ({ id, sql, params, cached, values, raw, safeIntegers }) => {
236 +parentPort.on("message", (request) => {
237 + if (request.type === "close") {
238 + // Closing lets SQLite checkpoint and remove the WAL, which terminating would skip.
239 + try {
240 + if (db !== undefined) db.close();
241 + } catch {}
242 + parentPort.close();
243 + return;
244 + }
245 + const { id, sql, params, cached, values, raw, safeIntegers } = request;
246 let operation = "open";
247 try {
232 - if (db === undefined) throw new Error("database reader is unavailable");
248 + if (db === undefined) throw new Error("database connection is unavailable");
249 operation = "prepare";
250 const statement = prepare(sql, cached);
251 operation = "execute";
@@ -242,21 +258,12 @@ parentPort.on("message", ({ id, sql, params, cached, values, raw, safeIntegers }
258 parentPort.postMessage({ id, rows: raw ? result : [] });
259 }
260 } catch (error) {
245 - parentPort.postMessage({
246 - id,
247 - error: {
248 - operation,
249 - message: String(error && error.message ? error.message : error),
250 - code: error && error.code,
251 - errcode: error && error.errcode,
252 - errstr: error && error.errstr,
253 - },
254 - });
261 + parentPort.postMessage({ id, error: { operation, ...describe(error) } });
262 }
263 });
264 `;
265
259 -interface ReaderRequest {
266 +interface WorkerRequest {
267 readonly sql: string;
268 readonly params: ReadonlyArray<unknown>;
269 readonly cached: boolean;
@@ -264,24 +271,25 @@ interface ReaderRequest {
271 readonly raw: boolean;
272 }
273
267 -type ReaderStartup =
274 +interface WorkerFailure {
275 + readonly message: string;
276 + readonly code?: unknown;
277 + readonly errcode?: unknown;
278 + readonly errstr?: unknown;
279 +}
280 +
281 +type WorkerStartup =
282 | { readonly type: "ready" }
269 - | { readonly type: "unavailable"; readonly message: string };
283 + | ({ readonly type: "unavailable" } & WorkerFailure);
284
271 -type ReaderReply =
285 +type WorkerReply =
286 | { readonly id: number; readonly rows: ReadonlyArray<any> }
287 | {
288 readonly id: number;
275 - readonly error: {
276 - readonly operation: "open" | "prepare" | "execute";
277 - readonly message: string;
278 - readonly code?: unknown;
279 - readonly errcode?: unknown;
280 - readonly errstr?: unknown;
281 - };
289 + readonly error: WorkerFailure & { readonly operation: "open" | "prepare" | "execute" };
290 };
291
284 -const readerFailureMessages = {
292 +const workerFailureMessages = {
293 open: "Failed to open database",
294 prepare: "Failed to prepare statement",
295 execute: "Failed to execute statement",
@@ -290,48 +298,66 @@ const readerFailureMessages = {
298 const failed = (cause: unknown, message: string, operation: string) =>
299 new SqlError({ reason: classifyError(cause, message, operation) });
300
301 +const failedWith = (
302 + { message, code, errcode, errstr }: WorkerFailure,
303 + operation: keyof typeof workerFailureMessages,
304 +) =>
305 + failed(
306 + Object.assign(new Error(message), { code, errcode, errstr }),
307 + workerFailureMessages[operation],
308 + operation,
309 + );
310 +
311 // Read-transaction cleanup succeeds after worker exit because it can no longer change anything.
312 const endsReadTransaction = (sql: string) => /^\s*ROLLBACK\b|^\s*COMMIT\b/i.test(sql);
313
314 // WAL readers run concurrently; one worker serialized every read behind the slowest transaction.
315 const READER_POOL_SIZE = 4;
316 +const READER_STARTUP_TIMEOUT = Duration.seconds(10);
317
318 // Bind transactions to one worker; fail after its exit instead of resuming outside the transaction on a replacement.
300 -const makeReader = Effect.fnUntraced(function* (
301 - options: SqliteClientConfig,
302 - pool: { unavailable: boolean },
303 - readOnly = true,
304 -) {
319 +const makeWorkerConnection = Effect.fnUntraced(function* (options: {
320 + readonly filename: string;
321 + readonly role: "reader" | "writer";
322 + readonly readOnly: boolean;
323 + readonly onReply?: () => void;
324 +}) {
325 + const label = options.role === "reader" ? "Database reader" : "Database writer";
326 + const readOnly = options.readOnly;
327 let current:
328 | {
329 readonly worker: NodeWorkerThreads.Worker;
330 readonly connection: Connection;
309 - readonly started: Promise<ReaderStartup>;
331 + readonly started: Promise<WorkerStartup>;
332 }
333 | undefined;
334 let nextRequestId = 0;
335 + // A replacement writer would lose the pragmas callers set on it, such as foreign_keys.
336 + let writerStopped: string | undefined;
337
338 const start = () => {
315 - const worker = new NodeWorkerThreads.Worker(readerWorkerSource, {
339 + const worker = new NodeWorkerThreads.Worker(workerSource, {
340 eval: true,
341 workerData: {
342 filename: options.filename,
343 readOnly,
344 + requireWal: options.role === "reader",
345 allowExtension: false,
346 prepareCacheSize: 200,
347 },
348 });
324 - let announce: (startup: ReaderStartup) => void = () => {};
325 - const started = new Promise<ReaderStartup>((resolve) => {
349 + let announce: (startup: WorkerStartup) => void = () => {};
350 + const started = new Promise<WorkerStartup>((resolve) => {
351 announce = resolve;
352 });
353 let stoppedMessage: string | undefined;
354 const pending = new Map<
355 number,
331 - { readonly endsTransaction: boolean; readonly resume: (reply: ReaderReply) => void }
356 + { readonly endsTransaction: boolean; readonly resume: (reply: WorkerReply) => void }
357 >();
358 const stop = (message: string) => {
359 stoppedMessage = message;
360 + if (options.role === "writer") writerStopped = message;
361 announce({ type: "unavailable", message });
362 if (current?.worker === worker) current = undefined;
363 for (const [id, request] of pending) {
@@ -343,7 +369,7 @@ const makeReader = Effect.fnUntraced(function* (
369 }
370 pending.clear();
371 };
346 - worker.on("message", (reply: ReaderReply | ReaderStartup) => {
372 + worker.on("message", (reply: WorkerReply | WorkerStartup) => {
373 if ("type" in reply) {
374 announce(reply);
375 if (pending.size === 0) worker.unref();
@@ -353,12 +379,13 @@ const makeReader = Effect.fnUntraced(function* (
379 if (request === undefined) return;
380 pending.delete(reply.id);
381 if (pending.size === 0) worker.unref();
382 + options.onReply?.();
383 request.resume(reply);
384 });
358 - worker.on("error", (cause) => stop(`Database reader failed: ${cause.message}`));
359 - worker.on("exit", (code) => stop(`Database reader exited with code ${code}`));
385 + worker.on("error", (cause) => stop(`${label} failed: ${cause.message}`));
386 + worker.on("exit", (code) => stop(`${label} exited with code ${code}`));
387
361 - const request = (input: ReaderRequest) =>
388 + const request = (input: WorkerRequest) =>
389 Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
390 // A COMMIT that never reached a writer must fail; only its ROLLBACK is moot.
391 const endsTransaction = readOnly
@@ -367,37 +394,28 @@ const makeReader = Effect.fnUntraced(function* (
394 if (stoppedMessage !== undefined) {
395 return endsTransaction
396 ? Effect.succeed([])
370 - : Effect.fail(
371 - failed(new Error(stoppedMessage), readerFailureMessages.execute, "execute"),
372 - );
397 + : Effect.fail(failedWith({ message: stoppedMessage }, "execute"));
398 }
399 const safeIntegers = Boolean(Context.get(fiber.context, Client.SafeIntegers));
375 - return Effect.callback<ReadonlyArray<any>, SqlError>((resume) => {
400 + const reply = Effect.callback<ReadonlyArray<any>, SqlError>((resume) => {
401 const id = nextRequestId++;
402 pending.set(id, {
403 endsTransaction,
379 - resume: (reply) => {
380 - if ("rows" in reply) {
381 - resume(Effect.succeed(reply.rows));
382 - return;
383 - }
384 - const { operation, message, ...codes } = reply.error;
404 + resume: (reply) =>
405 resume(
386 - Effect.fail(
387 - failed(
388 - Object.assign(new Error(message), codes),
389 - readerFailureMessages[operation],
390 - operation,
391 - ),
392 - ),
393 - );
394 - },
406 + "rows" in reply
407 + ? Effect.succeed(reply.rows)
408 + : Effect.fail(failedWith(reply.error, reply.error.operation)),
409 + ),
410 });
411 if (pending.size === 1) worker.ref();
412 const cached = input.cached && !hasLargeBinding(input.params);
413 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
414 worker.postMessage({ id, safeIntegers, ...input, cached });
415 });
416 + // A posted write runs to the end, as it did on the event loop; the caller and its lock
417 + // must not move on while the worker is still applying it. Reads stay cancellable.
418 + return options.role === "writer" ? Effect.uninterruptible(reply) : reply;
419 });
420
421 const connection = identity<Connection>({
@@ -436,220 +454,154 @@ const makeReader = Effect.fnUntraced(function* (
454
455 yield* Effect.addFinalizer(() =>
456 Effect.promise(async () => {
439 - await current?.worker.terminate();
457 + const open = current;
458 + if (open === undefined) return;
459 + if (options.role === "writer") {
460 + await new Promise<void>((resolve) => {
461 + open.worker.once("exit", () => resolve());
462 + open.worker.ref();
463 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
464 + open.worker.postMessage({ type: "close" });
465 + });
466 + }
467 + await open.worker.terminate();
468 }),
469 );
470
443 - const startupTimeout = Duration.seconds(10);
444 -
445 - const turnOff = (reader: typeof current, reason: string) =>
446 - Effect.suspend(() => {
447 - if (reader !== undefined) {
448 - if (current === reader) current = undefined;
449 - void reader.worker.terminate();
450 - }
451 - if (pool.unavailable) return Effect.undefined;
452 - pool.unavailable = true;
453 - return Effect.logWarning("SQLite reader worker unavailable; reads use the main connection", {
454 - reason,
455 - }).pipe(Effect.as(undefined));
456 - });
471 + const retire = (worker: NodeWorkerThreads.Worker) => {
472 + if (current?.worker === worker) current = undefined;
473 + void worker.terminate();
474 + };
475
458 - return Effect.suspend((): Effect.Effect<Connection | undefined> => {
459 - if (pool.unavailable) return Effect.undefined;
460 - let reader: NonNullable<typeof current>;
476 + return Effect.suspend((): Effect.Effect<Connection, SqlError> => {
477 + if (current === undefined && writerStopped !== undefined) {
478 + return Effect.fail(failedWith({ message: writerStopped }, "execute"));
479 + }
480 + let entry: NonNullable<typeof current>;
481 try {
462 - reader = current ?? start();
482 + entry = current ?? start();
483 } catch (cause) {
464 - return turnOff(undefined, String(cause));
484 + return Effect.fail(failed(cause, workerFailureMessages.open, "open"));
485 }
486 + const started = Effect.promise(() => entry.started);
487 return Effect.flatMap(
488 Effect.withFiber((fiber) => {
489 const restore = Context.get(fiber.context, ReadOnly) ?? identity;
490 + // Readers fall back to the writer; the writer has no fallback, so a slow disk only delays it.
491 return restore(
470 - Effect.timeoutOption(
471 - Effect.promise(() => reader.started),
472 - startupTimeout,
473 - ),
492 + options.role === "reader"
493 + ? Effect.timeoutOption(started, READER_STARTUP_TIMEOUT)
494 + : Effect.asSome(started),
495 );
496 }),
476 - (startup): Effect.Effect<Connection | undefined> =>
477 - // A late ready event must not hand out a worker retired by another acquire timeout.
478 - pool.unavailable || current !== reader
479 - ? Effect.undefined
480 - : Option.isSome(startup) && startup.value.type === "ready"
481 - ? Effect.succeed(reader.connection)
482 - : turnOff(
483 - reader,
484 - Option.isSome(startup) && startup.value.type === "unavailable"
485 - ? startup.value.message
486 - : `worker did not open the database within ${Duration.toMillis(startupTimeout)} ms`,
487 - ),
497 + (startup): Effect.Effect<Connection, SqlError> => {
498 + const outcome = Option.getOrUndefined(startup);
499 + if (outcome?.type === "ready") {
500 + // A late ready event must not hand out a worker retired by another acquire timeout.
501 + return current === entry
502 + ? Effect.succeed(entry.connection)
503 + : Effect.fail(failedWith({ message: `${label} stopped while starting` }, "open"));
504 + }
505 + retire(entry.worker);
506 + return Effect.fail(
507 + failedWith(
508 + outcome ?? {
509 + message: `${label} did not open the database within ${Duration.toMillis(READER_STARTUP_TIMEOUT)} ms`,
510 + },
511 + "open",
512 + ),
513 + );
514 + },
515 );
516 });
517 });
518
519 +// A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
520 +const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavailable: boolean }) {
521 + const acquire = yield* makeWorkerConnection({ filename, role: "reader", readOnly: true });
522 + return Effect.suspend((): Effect.Effect<Connection | undefined> =>
523 + pool.unavailable
524 + ? Effect.undefined
525 + : acquire.pipe(
526 + Effect.map((connection) => (pool.unavailable ? undefined : connection)),
527 + Effect.catch((error) => {
528 + if (pool.unavailable) return Effect.undefined;
529 + pool.unavailable = true;
530 + const cause = error.reason.cause;
531 + return Effect.logWarning("SQLite reader worker unavailable; reads use the writer", {
532 + reason: cause instanceof Error ? cause.message : error.message,
533 + }).pipe(Effect.as(undefined));
534 + }),
535 + ),
536 + );
537 +});
538 +
539 +// A checkpoint copies WAL pages into the database and syncs it. On the writer's thread every
540 +// write would wait for that sync, so a second connection checkpoints beside the writes.
541 const CHECKPOINT_FRAMES = 1000;
542 const CHECKPOINT_IDLE_MS = 100;
494 -const CHECKPOINT_BACKPRESSURE_FRAMES = 32_768;
495 -
496 -const readSafeSqliteInteger = (value: unknown) => {
497 - const number = typeof value === "bigint" ? Number(value) : value;
498 - return typeof number === "number" && Number.isSafeInteger(number) ? number : undefined;
499 -};
543 +// If checkpoints fall behind constant writes, the writer checkpoints itself, delaying writes only.
544 +const WRITER_BACKSTOP_FRAMES = 10_000;
545
501 -// Evaluated source also works in bundles and single-executable builds.
546 const checkpointWorkerSource = `
547 const { parentPort, workerData } = require("node:worker_threads");
548 const { DatabaseSync } = require("node:sqlite");
505 -const { statSync } = require("node:fs");
549 let db;
550 let frames;
508 -let checkpoint;
509 -let truncate;
510 -let journalSizeLimit = -1;
511 -let generation = 0;
512 -let nextTruncateId = 0;
513 -let pendingTruncate;
514 -let pendingBarriers = 0;
551 +let passive;
552 let idle;
516 -const initialize = () => {
517 - if (db !== undefined) return;
518 - // The writer switches to WAL after constructing the client.
519 - db = new DatabaseSync(workerData.filename);
520 - db.exec("PRAGMA busy_timeout = 5000;");
521 - const mode = String(db.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
522 - if (mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
523 - // macOS requires F_FULLFSYNC to flush the drive cache.
524 - db.exec("PRAGMA checkpoint_fullfsync = ON;");
525 - frames = db.prepare("PRAGMA wal_checkpoint(NOOP)");
526 - checkpoint = db.prepare("PRAGMA wal_checkpoint(PASSIVE)");
527 - truncate = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)");
528 -};
529 -const atCeiling = (state) => state.log >= workerData.backpressureFrames;
530 -const exceedsSizeLimit = () =>
531 - journalSizeLimit >= 0 && statSync(workerData.filename + "-wal").size > journalSizeLimit;
532 -// Retry a WAL past its limits even with admission open, so it resets once a pinning reader closes.
533 -const schedule = (state, retry) => {
534 - if (retry || state.log - state.checkpointed >= workerData.frames) {
535 - idle = setTimeout(() => check(undefined, undefined, true), workerData.idleMs);
553 +const check = (force) => {
554 + if (db === undefined) {
555 + db = new DatabaseSync(workerData.filename);
556 + db.exec("PRAGMA busy_timeout = 5000;");
557 + // macOS requires F_FULLFSYNC to flush the drive cache.
558 + db.exec("PRAGMA checkpoint_fullfsync = ON;");
559 + frames = db.prepare("PRAGMA wal_checkpoint(NOOP)");
560 + passive = db.prepare("PRAGMA wal_checkpoint(PASSIVE)");
561 }
537 -};
538 -const check = (id, nextJournalSizeLimit, force, nextGeneration = generation) => {
539 - initialize();
540 - if (Number.isSafeInteger(nextGeneration)) generation = Math.max(generation, nextGeneration);
541 - if (Number.isSafeInteger(nextJournalSizeLimit)) journalSizeLimit = nextJournalSizeLimit;
542 - let state = frames.get();
543 - const oversized = exceedsSizeLimit();
544 - let pinned = false;
545 - if (force || state.log - state.checkpointed >= workerData.frames || oversized) {
546 - const result = checkpoint.get();
547 - // A reader's snapshot stopped the backfill; holding writes cannot let it catch up.
548 - pinned = result.checkpointed < result.log;
562 + const before = frames.get();
563 + let state = before;
564 + if (state.log - state.checkpointed >= workerData.frames || (force && state.log > state.checkpointed)) {
565 + passive.get();
566 state = frames.get();
567 }
551 - if (state.checkpointed === state.log && (atCeiling(state) || oversized)) {
552 - pendingTruncate = { truncateId: ++nextTruncateId, checkId: id, generation, state };
553 - parentPort.postMessage({
554 - type: "truncate-request",
555 - truncateId: pendingTruncate.truncateId,
556 - checkId: id,
557 - generation,
558 - ...state,
559 - });
560 - return;
568 + // The WAL restarts only once every frame is checkpointed, so catch up when writes pause.
569 + // A reader pinning the WAL stops progress; the next write retries instead of a timer loop.
570 + const progressed = !force || state.checkpointed > before.checkpointed;
571 + if (state.log > state.checkpointed && progressed) {
572 + idle = setTimeout(() => run(true), workerData.idleMs);
573 }
562 - const blocked = !pinned && (atCeiling(state) || oversized) && state.checkpointed < state.log;
563 - parentPort.postMessage({ type: "checkpoint", id, generation, ...state, blocked });
564 - schedule(state, atCeiling(state) || oversized);
574 + parentPort.postMessage({ type: "checked", log: state.log, checkpointed: state.checkpointed });
575 };
566 -const finishTruncate = (truncateId) => {
567 - if (pendingTruncate === undefined || pendingTruncate.truncateId !== truncateId) return;
568 - const pending = pendingTruncate;
569 - parentPort.postMessage({ type: "truncate-start", truncateId });
570 - // Admission is paused here; a pinned reader must not make truncation wait.
571 - db.exec("PRAGMA busy_timeout = 0;");
576 +const run = (force) => {
577 + clearTimeout(idle);
578 try {
573 - truncate.get();
574 - } finally {
575 - db.exec("PRAGMA busy_timeout = 5000;");
576 - }
577 - const state = frames.get();
578 - pendingTruncate = undefined;
579 - parentPort.postMessage({
580 - type: "checkpoint",
581 - id: pending.checkId,
582 - generation: pending.generation,
583 - truncateId,
584 - ...state,
585 - // With writes held, only a reader can stop the reset, and holding them longer would not
586 - // release it.
587 - blocked: false,
588 - });
589 - while (pendingBarriers > 0) {
590 - pendingBarriers--;
591 - parentPort.postMessage("barrier");
579 + check(force);
580 + } catch (error) {
581 + parentPort.postMessage({ type: "failed", message: String(error && error.message ? error.message : error) });
582 }
593 - schedule(state, atCeiling(state) || exceedsSizeLimit());
583 };
595 -parentPort.on("message", (request) => {
596 - clearTimeout(idle);
597 - if (request && request.type === "truncate") {
598 - finishTruncate(request.truncateId);
599 - return;
600 - }
601 - const isBarrier = request === "barrier";
602 - if (pendingTruncate !== undefined) {
603 - if (isBarrier) pendingBarriers++;
604 - else if (request && request.type === "check") {
605 - // Retire the request without labeling the pending snapshot with a newer generation.
606 - parentPort.postMessage({
607 - type: "checkpoint",
608 - id: request.id,
609 - generation: pendingTruncate.generation,
610 - ...pendingTruncate.state,
611 - blocked: true,
612 - });
613 - }
614 - return;
615 - }
616 - const input = request && typeof request === "object" ? request : {};
617 - check(input.id, input.journalSizeLimit, input.force, input.generation);
618 - if (isBarrier) {
619 - if (pendingTruncate !== undefined) pendingBarriers++;
620 - else parentPort.postMessage("barrier");
621 - }
622 -});
584 +parentPort.on("message", () => run(false));
585 `;
586
625 -const startCheckpointWorker = Effect.fnUntraced(function* (
626 - filename: string,
627 - writer: NodeSqlite.DatabaseSync,
628 -) {
629 - const warn = (reason: string) =>
630 - Effect.logWarning(
631 - "SQLite checkpoint worker unavailable; checkpoints run on the main connection",
632 - { reason },
633 - );
634 - if (typeof writer.isTransaction !== "boolean" || typeof writer.setAuthorizer !== "function") {
635 - yield* warn("Node SQLite lacks transaction-state or statement-authorizer support");
636 - return undefined;
587 +const sqliteVersion = () => {
588 + const db = new NodeSqlite.DatabaseSync(":memory:");
589 + try {
590 + return String(db.prepare("SELECT sqlite_version() AS version").get()?.version);
591 + } finally {
592 + db.close();
593 }
638 - // These are the writer's first statements, before callers set their own wait. A file that is
639 - // not a database first fails here.
640 - const { version, frames, sizeLimit } = yield* Effect.try({
641 - try: () => {
642 - writer.exec("PRAGMA busy_timeout = 5000;");
643 - return {
644 - version: String(writer.prepare("SELECT sqlite_version() AS version").get()?.version),
645 - frames: writer.prepare("PRAGMA wal_checkpoint(NOOP)"),
646 - sizeLimit: writer.prepare("PRAGMA journal_size_limit"),
647 - };
648 - },
649 - catch: (cause) => failed(cause, "Failed to open database", "open"),
650 - });
594 +};
595 +
596 +// Returns the writer's per-statement notification, or undefined when the writer checkpoints alone.
597 +const startCheckpointWorker = Effect.fnUntraced(function* (filename: string) {
598 + const warn = (reason: string) =>
599 + Effect.logWarning("SQLite checkpoint worker unavailable; the writer checkpoints its own WAL", {
600 + reason,
601 + });
602 + const version = sqliteVersion();
603 const [major = 0, minor = 0] = version.split(".").map(Number);
652 - // Older SQLite treats NOOP as PASSIVE, which would checkpoint on the writer.
604 + // Older SQLite treats NOOP as PASSIVE, so counting frames would checkpoint on every write.
605 if (major < 3 || (major === 3 && minor < 51)) {
606 yield* warn(`SQLite ${version} has no NOOP checkpoint`);
607 return undefined;
@@ -658,12 +610,7 @@ const startCheckpointWorker = Effect.fnUntraced(function* (
610 () =>
611 new NodeWorkerThreads.Worker(checkpointWorkerSource, {
612 eval: true,
661 - workerData: {
662 - filename,
663 - frames: CHECKPOINT_FRAMES,
664 - backpressureFrames: CHECKPOINT_BACKPRESSURE_FRAMES,
665 - idleMs: CHECKPOINT_IDLE_MS,
666 - },
613 + workerData: { filename, frames: CHECKPOINT_FRAMES, idleMs: CHECKPOINT_IDLE_MS },
614 }),
615 );
616 if (Result.isFailure(started)) {
@@ -671,199 +618,37 @@ const startCheckpointWorker = Effect.fnUntraced(function* (
618 return undefined;
619 }
620 const worker = started.success;
621 + worker.unref();
622 const runFork = Effect.runForkWith(yield* Effect.context<never>());
623 let stopped = false;
676 - let generation = 0;
677 - let nextId = 0;
678 - let inFlight: number | undefined;
679 - let checkAgain = false;
680 - let blocked = false;
681 - let log = 0;
682 - let checkpointed = 0;
683 - let unknown = false;
684 - let truncate: { id: number; acknowledged: boolean } | undefined;
685 - type Waiter = { resume: () => void; cancelled: boolean };
686 - const waiters: Array<Waiter> = [];
687 - let elected: Waiter | undefined;
688 - const gated = () =>
689 - !stopped && !writer.isTransaction && (unknown || blocked || truncate !== undefined);
690 - const wake = () => {
691 - if (!gated() && elected === undefined) {
692 - elected = waiters.shift();
693 - elected?.resume();
694 - }
695 - if (waiters.length === 0 && elected === undefined) worker.unref();
696 - };
697 - const release = (waiter: Waiter) => {
698 - waiter.cancelled = true;
699 - const index = waiters.indexOf(waiter);
700 - if (index !== -1) waiters.splice(index, 1);
701 - if (elected === waiter) elected = undefined;
702 - wake();
703 - };
704 - const handBack = (reason: string) => {
624 + const stop = (reason: string) => {
625 if (stopped) return;
626 stopped = true;
707 - truncate = undefined;
708 - try {
709 - writer.exec(`PRAGMA wal_autocheckpoint = ${CHECKPOINT_FRAMES};`);
710 - } catch {}
711 - wake();
627 + void worker.terminate();
628 runFork(warn(reason));
629 };
714 - const send = (message: unknown) => {
715 - try {
716 - // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
717 - worker.postMessage(message);
718 - } catch (cause) {
719 - handBack(String(cause));
720 - }
721 - };
722 - const acknowledge = () => {
723 - if (truncate === undefined || truncate.acknowledged || stopped || writer.isTransaction) return;
724 - truncate.acknowledged = true;
725 - send({ type: "truncate", truncateId: truncate.id });
726 - };
727 - const observe = () => {
728 - try {
729 - const state = frames.get();
730 - const nextLog = readSafeSqliteInteger(state?.log);
731 - const nextCheckpointed = readSafeSqliteInteger(state?.checkpointed);
732 - if (nextLog === undefined || nextCheckpointed === undefined)
733 - throw new Error("Invalid WAL frame counts");
734 - log = nextLog;
735 - checkpointed = nextCheckpointed;
736 - unknown = false;
737 - // Past the ceiling, hold writes until the worker judges this WAL; a reader can pin it there.
738 - if (log >= CHECKPOINT_BACKPRESSURE_FRAMES) blocked = true;
739 - } catch {
740 - unknown = true;
741 - }
742 - };
743 - const request = (force = false) => {
744 - if (stopped || writer.isTransaction) return;
745 - if (truncate !== undefined) {
746 - acknowledge();
747 - return;
748 - }
749 - if (inFlight !== undefined) {
750 - checkAgain = true;
751 - return;
752 - }
753 - if (!force && !gated() && log - checkpointed < CHECKPOINT_FRAMES) return;
754 - inFlight = ++nextId;
755 - try {
756 - send({
757 - type: "check",
758 - id: inFlight,
759 - generation,
760 - journalSizeLimit: readSafeSqliteInteger(sizeLimit.get()?.journal_size_limit),
761 - force: force || gated(),
762 - });
763 - } catch (cause) {
764 - handBack(String(cause));
765 - }
766 - };
767 - const check = () => {
768 - if (stopped || writer.isTransaction) return;
769 - generation++;
770 - observe();
771 - request();
772 - wake();
773 - };
774 - worker.on("error", (cause) => handBack(cause.message));
775 - worker.on("exit", (code) => handBack(`exited with code ${code}`));
776 - worker.on("message", (message: unknown) => {
777 - if (stopped || typeof message !== "object" || message === null || !("type" in message)) return;
778 - if (
779 - message.type === "truncate-request" &&
780 - "truncateId" in message &&
781 - typeof message.truncateId === "number"
782 - ) {
783 - truncate = { id: message.truncateId, acknowledged: false };
784 - blocked = true;
785 - acknowledge();
786 - return;
787 - }
788 - if (message.type !== "checkpoint") return;
789 - const completedTruncate =
790 - truncate !== undefined && "truncateId" in message && message.truncateId === truncate.id;
791 - if (completedTruncate) truncate = undefined;
792 - // An acknowledged truncate excludes new writer mutations, so its result is current.
793 - if (
794 - (completedTruncate || ("generation" in message && message.generation === generation)) &&
795 - "log" in message &&
796 - typeof message.log === "number" &&
797 - "checkpointed" in message &&
798 - typeof message.checkpointed === "number" &&
799 - "blocked" in message &&
800 - typeof message.blocked === "boolean"
801 - ) {
802 - log = message.log;
803 - checkpointed = message.checkpointed;
804 - blocked = message.blocked;
805 - unknown = false;
806 - }
807 - // Even a stale snapshot must retire its request so a newer check can run.
808 - if ("id" in message && message.id === inFlight) inFlight = undefined;
809 - if (checkAgain && inFlight === undefined) {
810 - checkAgain = false;
811 - request(true);
812 - }
813 - wake();
630 + worker.on("message", (message: { readonly type?: unknown; readonly message?: unknown }) => {
631 + if (message?.type === "failed") stop(String(message.message));
632 });
633 + worker.on("error", (cause) => stop(cause.message));
634 + worker.on("exit", (code) => stop(`exited with code ${code}`));
635 yield* Effect.addFinalizer(() =>
636 Effect.promise(() => {
637 stopped = true;
818 - truncate = undefined;
819 - wake();
638 return worker.terminate();
639 }),
640 );
823 - writer.exec("PRAGMA wal_autocheckpoint = 0;");
824 - observe();
825 - worker.unref();
826 -
827 - const runWithAdmission = <A, E>(
828 - execute: () => Exit.Exit<A, E>,
829 - admitted?: Waiter,
830 - ): Effect.Effect<A, E> =>
831 - Effect.suspend(() => {
832 - if (!gated()) {
833 - // Release the election after executing; its successor must see the new WAL state.
834 - try {
835 - return execute();
836 - } finally {
837 - if (admitted !== undefined) release(admitted);
838 - }
839 - }
840 - if (admitted !== undefined) release(admitted);
841 - const waiter: Waiter = { resume: () => {}, cancelled: false };
842 - const wait = Effect.uninterruptibleMask((restore) =>
843 - Effect.withFiber((fiber) =>
844 - (fiber.getRef(TransactionAcquire)?.restore ?? restore)(
845 - Effect.callback<void>(function (this: Scheduler.Scheduler, resume) {
846 - const dispatcher = this.makeDispatcher();
847 - waiter.resume = () => {
848 - dispatcher.scheduleTask(() => {
849 - if (!waiter.cancelled) resume(Effect.void);
850 - }, 0);
851 - };
852 - waiters.push(waiter);
853 - worker.ref();
854 - request(true);
855 - wake();
856 - return Effect.sync(() => release(waiter));
857 - }),
858 - ),
859 - ),
860 - );
861 - return wait.pipe(
862 - Effect.flatMap(() => runWithAdmission(execute, waiter)),
863 - Effect.onInterrupt(() => Effect.sync(() => release(waiter))),
864 - );
641 + let scheduled = false;
642 + return () => {
643 + if (stopped || scheduled) return;
644 + scheduled = true;
645 + // One check per event-loop turn is enough, however many statements finished in it.
646 + setImmediate(() => {
647 + scheduled = false;
648 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
649 + if (!stopped) worker.postMessage(0);
650 });
866 - return { check, runWithAdmission };
651 + };
652 });
653
654 const make = Effect.fn("makeWithDatabase")(function* (
@@ -897,78 +682,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
682 }),
683 }).pipe(Effect.orDie),
684 );
900 - // Started after the close finalizer, so the worker stops before the writer closes.
901 - const checkpoint =
902 - options.checkpointWorker && isFileDatabase
903 - ? yield* startCheckpointWorker(options.filename, db)
904 - : undefined;
905 -
685 const statementReaderCache = new WeakMap<NodeSqlite.StatementSync, boolean>();
907 - interface CheckpointClassification {
908 - backpressure: boolean;
909 - check: boolean;
910 - }
911 - const checkpointClassifications = new WeakMap<
912 - NodeSqlite.StatementSync,
913 - CheckpointClassification
914 - >();
915 - const mutatingPragmasWithValues = new Set([
916 - "application_id",
917 - "auto_vacuum",
918 - "journal_mode",
919 - "schema_version",
920 - "user_version",
921 - ]);
922 - const mutatesPragma = (name: string | null, value: string | null) => {
923 - const pragma = name?.toLowerCase();
924 - if (pragma === undefined) return false;
925 - if (pragma === "incremental_vacuum" || pragma === "optimize") return true;
926 - if (pragma === "wal_checkpoint") return value?.toLowerCase() !== "noop";
927 - return value !== null && mutatingPragmasWithValues.has(pragma);
928 - };
929 - let activeClassification: CheckpointClassification | undefined;
930 - const withClassification = <A>(
931 - classification: CheckpointClassification | undefined,
932 - f: () => A,
933 - ): A => {
934 - if (classification === undefined) return f();
935 - const previous = activeClassification;
936 - activeClassification = classification;
937 - try {
938 - return f();
939 - } finally {
940 - activeClassification = previous;
941 - }
942 - };
943 - if (checkpoint !== undefined) {
944 - // SQLite may reprepare an expired statement while it executes. Keep this
945 - // callback installed and collect only around that statement's prepare/run.
946 - db.setAuthorizer((actionCode, action, argument) => {
947 - const classification = activeClassification;
948 - if (classification === undefined) return NodeSqlite.constants.SQLITE_OK;
949 - const constants = NodeSqlite.constants;
950 - if (actionCode === constants.SQLITE_TRANSACTION) {
951 - classification.backpressure ||= action === "BEGIN";
952 - classification.check ||= action === "COMMIT" || action === "ROLLBACK";
953 - } else if (actionCode === constants.SQLITE_SAVEPOINT) {
954 - classification.backpressure ||= action === "BEGIN";
955 - classification.check = true;
956 - } else if (actionCode === constants.SQLITE_PRAGMA && mutatesPragma(action, argument)) {
957 - classification.backpressure = true;
958 - classification.check = true;
959 - } else if (
960 - actionCode !== constants.SQLITE_READ &&
961 - actionCode !== constants.SQLITE_SELECT &&
962 - actionCode !== constants.SQLITE_FUNCTION &&
963 - actionCode !== constants.SQLITE_RECURSIVE &&
964 - actionCode !== constants.SQLITE_PRAGMA
965 - ) {
966 - classification.backpressure = true;
967 - classification.check = true;
968 - }
969 - return constants.SQLITE_OK;
970 - });
971 - }
686 const hasRows = (statement: NodeSqlite.StatementSync): boolean => {
687 const cached = statementReaderCache.get(statement);
688 if (cached !== undefined) {
@@ -981,12 +695,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
695
696 const prepare = (sql: string) =>
697 Effect.try({
984 - try: () => {
985 - const classification = { backpressure: false, check: false };
986 - const statement = withClassification(classification, () => db.prepare(sql));
987 - if (checkpoint !== undefined) checkpointClassifications.set(statement, classification);
988 - return statement;
989 - },
698 + try: () => db.prepare(sql),
699 catch: (cause) =>
700 new SqlError({
701 reason: classifyError(cause, "Failed to prepare statement", "prepare"),
@@ -1008,52 +717,43 @@ const make = Effect.fn("makeWithDatabase")(function* (
717 mode: "rows" | "raw" | "values" = "rows",
718 ) =>
719 Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
1011 - const classification = checkpointClassifications.get(statement);
1012 - const execute = () => {
1013 - let arrays = false;
1014 - let result: Exit.Exit<ReadonlyArray<unknown>, SqlError>;
720 + let arrays = false;
721 + let result: Exit.Exit<ReadonlyArray<unknown>, SqlError>;
722 + try {
723 + statement.setReadBigInts(Boolean(Context.get(fiber.context, Client.SafeIntegers)));
724 + if (hasRows(statement)) {
725 + arrays = mode === "values";
726 + if (arrays) statement.setReturnArrays(true);
727 + result = Exit.succeed(statement.all(...(params as any)));
728 + } else {
729 + const value = statement.run(...(params as any));
730 + result = Exit.succeed(
731 + mode === "raw" ? (value as unknown as ReadonlyArray<unknown>) : [],
732 + );
733 + }
734 + } catch (cause) {
735 + result = Exit.fail(
736 + new SqlError({
737 + reason: classifyError(cause, "Failed to execute statement", "execute"),
738 + }),
739 + );
740 + }
741 + if (arrays) {
742 try {
1016 - const value = withClassification(classification, () => {
1017 - statement.setReadBigInts(Boolean(Context.get(fiber.context, Client.SafeIntegers)));
1018 - if (hasRows(statement)) {
1019 - arrays = mode === "values";
1020 - if (arrays) statement.setReturnArrays(true);
1021 - return statement.all(...(params as any));
1022 - }
1023 - const result = statement.run(...(params as any));
1024 - return mode === "raw" ? (result as unknown as ReadonlyArray<unknown>) : [];
1025 - });
1026 - result = Exit.succeed(value);
743 + statement.setReturnArrays(false);
744 } catch (cause) {
1028 - result = Exit.fail(
745 + return Exit.die(
746 new SqlError({
1030 - reason: classifyError(cause, "Failed to execute statement", "execute"),
747 + reason: classifyError(
748 + cause,
749 + "Failed to reset statement result mode",
750 + "resetResultMode",
751 + ),
752 }),
753 );
1033 - } finally {
1034 - // OR FAIL may have committed earlier rows even when execution throws.
1035 - if (classification?.check) checkpoint?.check();
1036 - }
1037 - if (arrays) {
1038 - try {
1039 - statement.setReturnArrays(false);
1040 - } catch (cause) {
1041 - return Exit.die(
1042 - new SqlError({
1043 - reason: classifyError(
1044 - cause,
1045 - "Failed to reset statement result mode",
1046 - "resetResultMode",
1047 - ),
1048 - }),
1049 - );
1050 - }
754 }
1052 - return result;
1053 - };
1054 - return checkpoint !== undefined && classification?.backpressure
1055 - ? checkpoint.runWithAdmission(execute)
1056 - : execute();
755 + }
756 + return result;
757 });
758
759 const run = (sql: string, params: ReadonlyArray<unknown>, raw = false) =>
@@ -1093,14 +793,35 @@ const make = Effect.fn("makeWithDatabase")(function* (
793 });
794
795 const semaphore = yield* Semaphore.make(1);
1096 - const connection = yield* makeConnection;
1097 - // A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
796 + const writerWorker = options.writerWorker === true && isFileDatabase;
797 + let checkpointAfterWrite: (() => void) | undefined;
798 + const writerConnection: Effect.Effect<Connection, SqlError> = writerWorker
799 + ? yield* makeWorkerConnection({
800 + filename: options.filename,
801 + role: "writer",
802 + readOnly: options.readonly ?? false,
803 + onReply: () => checkpointAfterWrite?.(),
804 + })
805 + : Effect.succeed(yield* makeConnection);
806 + // Opening now fails the layer for a file that is not a database, not its first query.
807 + const opened = yield* writerConnection;
808 + // Started after the writer, so it stops first and the writer's close can checkpoint.
809 + if (writerWorker && options.readonly !== true) {
810 + checkpointAfterWrite = yield* startCheckpointWorker(options.filename);
811 + if (checkpointAfterWrite !== undefined) {
812 + yield* opened.executeUnprepared(
813 + `PRAGMA wal_autocheckpoint = ${WRITER_BACKSTOP_FRAMES}`,
814 + [],
815 + undefined,
816 + );
817 + }
818 + }
819 const readerPool = { unavailable: false };
820 const readers =
821 options.readerWorker && isFileDatabase
822 ? yield* Effect.forEach(Array.from({ length: READER_POOL_SIZE }), () =>
823 Effect.map(
1103 - Effect.all([makeReader(options, readerPool), Semaphore.make(1)]),
824 + Effect.all([makeReader(options.filename, readerPool), Semaphore.make(1)]),
825 ([acquire, lock]) => ({
826 acquire,
827 lock,
@@ -1110,13 +831,22 @@ const make = Effect.fn("makeWithDatabase")(function* (
831 )
832 : [];
833
1113 - const writer = { name: "writer" as const, lock: semaphore, connection };
834 + interface Target {
835 + readonly name: Acquired["connection"];
836 + readonly lock: Semaphore.Semaphore;
837 + readonly connection: Connection;
838 + }
839 + const writer = Effect.map(writerConnection, (connection): Target => ({
840 + name: "writer",
841 + lock: semaphore,
842 + connection,
843 + }));
844 let activeWriter: ConnectionOwner | undefined;
845 // Callers must run route uninterruptibly and pass `claimed` a finalizer that runs `release` once.
846 const route = (claimed: (release: Effect.Effect<void>) => Effect.Effect<void>) =>
1117 - Effect.withFiber((fiber) => {
847 + Effect.withFiber((fiber): Effect.Effect<Target, SqlError> => {
848 if (readers.length === 0 || readerPool.unavailable || !Context.get(fiber.context, ReadOnly)) {
1119 - return Effect.succeed(writer);
849 + return writer;
850 }
851 // Least claimed wins and ties go to the lowest slot, so idle slots never start a worker.
852 let slot = readers[0]!;
@@ -1129,10 +859,14 @@ const make = Effect.fn("makeWithDatabase")(function* (
859 reader.claims--;
860 }),
861 ),
1132 - Effect.map(reader.acquire, (readerConnection) =>
862 + Effect.flatMap(reader.acquire, (readerConnection) =>
863 readerConnection === undefined
864 ? writer
1135 - : { name: "reader" as const, lock: reader.lock, connection: readerConnection },
865 + : Effect.succeed<Target>({
866 + name: "reader",
867 + lock: reader.lock,
868 + connection: readerConnection,
869 + }),
870 ),
871 );
872 });
@@ -1146,8 +880,6 @@ const make = Effect.fn("makeWithDatabase")(function* (
880 lockWaitMs: Number((clock.monotonicTimeNanosUnsafe() - start) / 1000n) / 1000,
881 });
882
1149 - const writerBorrowBatchSize = 16;
1150 - let writerBorrowsSinceYield = 0;
883 const borrower: Borrower = (use) =>
884 Effect.uninterruptibleMask((restore) => {
885 let release: Effect.Effect<void> = Effect.void;
@@ -1160,11 +892,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
892 );
893 });
894 const borrowTarget = <A, E, R>(
1163 - target: {
1164 - readonly name: Acquired["connection"];
1165 - readonly lock: Semaphore.Semaphore;
1166 - readonly connection: Connection;
1167 - },
895 + target: Target,
896 use: (connection: Connection) => Effect.Effect<A, E, R>,
897 ): Effect.Effect<A, E, R> =>
898 Effect.withFiber((fiber) => {
@@ -1190,34 +918,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
918 }),
919 )
920 : useConnection;
1193 - if (!options.checkpointWorker || target.name !== "writer") {
1194 - return target.lock.withPermits(1)(useOwnedConnection);
1195 - }
1196 - const writerUse = Effect.onExit(useOwnedConnection, (exit) =>
1197 - Effect.sync(() => {
1198 - writerBorrowsSinceYield = Exit.hasInterrupts(exit)
1199 - ? 0
1200 - : Math.min(writerBorrowsSinceYield + 1, writerBorrowBatchSize);
1201 - }),
1202 - );
1203 - return Effect.flatMap(target.lock.withPermitsIfAvailable(1)(writerUse), (result) =>
1204 - Option.isSome(result)
1205 - ? Effect.succeed(result.value)
1206 - : target.lock.withPermits(1)(
1207 - Effect.andThen(
1208 - // Failed SQL also consumes the batch; cancellation hands off immediately.
1209 - Effect.suspend(() => {
1210 - if (writerBorrowsSinceYield < writerBorrowBatchSize) return Effect.void;
1211 - writerBorrowsSinceYield = 0;
1212 - return Effect.callback<void>((resume) => {
1213 - const immediate = setImmediate(() => resume(Effect.void));
1214 - return Effect.sync(() => clearImmediate(immediate));
1215 - });
1216 - }),
1217 - writerUse,
1218 - ),
1219 - ),
1220 - );
921 + return target.lock.withPermits(1)(useOwnedConnection);
922 });
923 const claimedUntilScopeCloses = (release: Effect.Effect<void>) =>
924 Effect.withFiber((fiber) =>
@@ -1293,39 +994,5 @@ const make = Effect.fn("makeWithDatabase")(function* (
994 });
995 });
996
1296 -const makeWorker = Effect.fn("makeWorkerClient")(function* (filename: string) {
1297 - yield* checkNodeSqliteCompat();
1298 - const acquire = yield* makeReader({ filename }, { unavailable: false }, false);
1299 - const connection = yield* acquire;
1300 - if (connection === undefined) {
1301 - return yield* failed(
1302 - new Error("database worker is unavailable"),
1303 - readerFailureMessages.open,
1304 - "open",
1305 - );
1306 - }
1307 - const { acquirer, borrower, onCommitFailure, transactionAcquirer } = Client.makeSqliteAcquirers({
1308 - connection: Effect.succeed(connection),
1309 - semaphore: yield* Semaphore.make(1),
1310 - });
1311 - return yield* Client.make({
1312 - driver: "node:sqlite",
1313 - acquirer,
1314 - borrower,
1315 - compiler: Statement.makeCompilerSqlite(),
1316 - transactionAcquirer,
1317 - onCommitFailure,
1318 - beginTransaction: "BEGIN IMMEDIATE",
1319 - spanAttributes: [[ATTR_DB_SYSTEM_NAME, "sqlite"]],
1320 - });
1321 -});
1322 -
1323 -/**
1324 - * A read-write client whose one connection runs in a worker thread, so long statements such as
1325 - * index builds leave the event loop free. Requires a file database already in WAL mode.
1326 - */
1327 -export const layerWorker = (filename: string): Layer.Layer<Client.SqlClient, SqlError> =>
1328 - Layer.effect(Client.SqlClient, makeWorker(filename)).pipe(Layer.provide(Reactivity.layer));
1329 -
997 export const layer = (config: SqliteClientConfig): Layer.Layer<Client.SqlClient, SqlError> =>
998 Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer));