The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a slow database write no longer freezes the server (#35728)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
sharath committed
Nov 13, 2026 at 05:39 UTC
ce268642cc4472e0671c4b6b8e29d2a21e383602
7 files changed
+418
-841
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+31
-31
@@ -1,4 +1,4 @@
1
-import * as NodeSqlite from "node:sqlite";
1
+import type * as NodeWorkerThreads from "node:worker_threads";
2
import * as NodeServices from "@effect/platform-node/NodeServices";
3
import { assert, it } from "@effect/vitest";
4
import {
@@ -73,6 +73,29 @@ import * as ProjectStore from "./ProjectStore.ts";
73
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
74
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
75
76
+const workerStatements = vi.hoisted(
77
+ (): Array<{ readonly readOnly: boolean; readonly sql: string }> => [],
78
+);
79
+
80
+vi.mock("node:worker_threads", async (importOriginal) => {
81
+ const actual = await importOriginal<typeof NodeWorkerThreads>();
82
+ class RecordedWorker extends actual.Worker {
83
+ readonly #readOnly: boolean;
84
+ constructor(...args: ConstructorParameters<typeof actual.Worker>) {
85
+ super(...args);
86
+ this.#readOnly = args[1]?.workerData?.readOnly === true;
87
+ }
88
+ override postMessage(...args: Parameters<NodeWorkerThreads.Worker["postMessage"]>) {
89
+ const [message] = args;
90
+ if (typeof message === "object" && message !== null && typeof message.sql === "string") {
91
+ workerStatements.push({ readOnly: this.#readOnly, sql: message.sql });
92
+ }
93
+ return super.postMessage(...args);
94
+ }
95
+ }
96
+ return { ...actual, Worker: RecordedWorker };
97
+});
98
+
99
const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
100
const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
101
@@ -193,45 +216,22 @@ it.effect("discovers on the reader worker and deletes on the writer", () =>
216
yield* Effect.gen(function* () {
217
const sql = yield* SqlClient.SqlClient;
218
yield* seedCompaction();
196
- // Establish the reader before recording main-thread statements.
219
+ // Establish the reader before recording statements.
220
yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
198
- const reads: string[] = [];
199
- const writes: string[] = [];
200
- const all = NodeSqlite.StatementSync.prototype.all;
201
- const run = NodeSqlite.StatementSync.prototype.run;
202
- yield* Effect.acquireRelease(
203
- Effect.sync(() => {
204
- const readSpy = vi
205
- .spyOn(NodeSqlite.StatementSync.prototype, "all")
206
- .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
207
- reads.push(this.sourceSQL);
208
- return all.apply(this, params);
209
- });
210
- const writeSpy = vi
211
- .spyOn(NodeSqlite.StatementSync.prototype, "run")
212
- .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
213
- writes.push(this.sourceSQL);
214
- return run.apply(this, params);
215
- });
216
- return { readSpy, writeSpy };
217
- }),
218
- ({ readSpy, writeSpy }) =>
219
- Effect.sync(() => {
220
- readSpy.mockRestore();
221
- writeSpy.mockRestore();
222
- }),
223
- );
221
+ workerStatements.length = 0;
222
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
223
const result = yield* maintenance.pruneEventStore();
224
assert.equal(result.deletedEventCount, 150);
225
assert.equal(result.deletedReceiptCount, 100);
226
+ const writer = workerStatements.filter((statement) => !statement.readOnly);
227
// Only the in-transaction check for rows appended during the pass reads on the writer.
228
assert.deepEqual(
230
- reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")),
229
+ writer.filter(({ sql }) => /\bSELECT\b/.test(sql) && !sql.includes("AS prompt_sequence")),
230
[],
231
);
233
- assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
234
- assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
232
+ assert(workerStatements.some(({ readOnly, sql }) => readOnly && /\bSELECT\b/.test(sql)));
233
+ assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_events")));
234
+ assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_command_receipts")));
235
}).pipe(
236
Effect.provide(
237
ProjectionMaintenance.layer.pipe(
apps/server/src/persistence/Sqlite.test.ts
+1
-74
@@ -2,7 +2,6 @@
2
import * as NodeFS from "node:fs";
3
import * as NodeOS from "node:os";
4
import * as NodePath from "node:path";
5
-import * as NodeSqlite from "node:sqlite";
5
import type * as NodeWorkerThreads from "node:worker_threads";
6
7
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
@@ -23,7 +22,6 @@ import * as ServerOwnership from "../serverOwnership.ts";
22
import { migrationManifest, runMigrations } from "./Migrations.ts";
23
import * as SqlitePersistence from "./Sqlite.ts";
24
26
-const workers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
25
const workerData = vi.hoisted((): Array<unknown> => []);
26
27
vi.mock("node:worker_threads", async (importOriginal) => {
@@ -31,84 +29,12 @@ vi.mock("node:worker_threads", async (importOriginal) => {
29
class RecordedWorker extends actual.Worker {
30
constructor(...args: ConstructorParameters<typeof actual.Worker>) {
31
super(...args);
34
- workers.push(this);
32
workerData.push(args[1]?.workerData);
33
}
34
}
35
return { ...actual, Worker: RecordedWorker };
36
});
37
41
-it.effect("keeps writes moving past a 48 MiB WAL while a reader retains its snapshot", () => {
42
- const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-sqlite-burst-"));
43
- const dbPath = NodePath.join(tempDir, "state.sqlite");
44
- const persistence = SqlitePersistence.layerFromPath(dbPath).pipe(
45
- Layer.provide(NodeServices.layer),
46
- );
47
-
48
- return Effect.gen(function* () {
49
- const sql = yield* SqlClient.SqlClient;
50
- const checkpointWorker = workers.at(-1)!;
51
- yield* sql`CREATE TABLE burst_probe(id INTEGER PRIMARY KEY, payload BLOB)`;
52
- yield* sql`INSERT INTO burst_probe VALUES (1, NULL)`;
53
- yield* sql`PRAGMA wal_checkpoint(TRUNCATE)`;
54
- const reader = yield* Effect.acquireRelease(
55
- Effect.sync(() => new NodeSqlite.DatabaseSync(dbPath, { readOnly: true })),
56
- (database) => Effect.sync(() => database.close()),
57
- );
58
- reader.exec("BEGIN");
59
- assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);
60
-
61
- yield* sql.withTransaction(
62
- sql`INSERT INTO burst_probe VALUES (2, zeroblob(${48 * 1024 * 1024}))`,
63
- );
64
- const state = yield* Effect.callback<{
65
- readonly log: number;
66
- readonly checkpointed: number;
67
- readonly blocked: boolean;
68
- }>((resume) => {
69
- const onMessage = (message: {
70
- readonly type?: string;
71
- readonly log?: number;
72
- readonly checkpointed?: number;
73
- readonly blocked?: boolean;
74
- }) => {
75
- if (
76
- message.type === "checkpoint" &&
77
- typeof message.log === "number" &&
78
- message.log > 10_000 &&
79
- typeof message.checkpointed === "number" &&
80
- typeof message.blocked === "boolean"
81
- ) {
82
- checkpointWorker.off("message", onMessage);
83
- resume(
84
- Effect.succeed({
85
- log: message.log,
86
- checkpointed: message.checkpointed,
87
- blocked: message.blocked,
88
- }),
89
- );
90
- }
91
- };
92
- checkpointWorker.on("message", onMessage);
93
- checkpointWorker.ref();
94
- // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
95
- checkpointWorker.postMessage("barrier");
96
- return Effect.sync(() => checkpointWorker.off("message", onMessage));
97
- });
98
- assert.isBelow(state.checkpointed, state.log);
99
- assert.isFalse(state.blocked);
100
-
101
- yield* sql.withTransaction(sql`INSERT INTO burst_probe VALUES (3, NULL)`);
102
- assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 1);
103
- reader.exec("COMMIT");
104
- assert.equal(reader.prepare("SELECT COUNT(*) AS count FROM burst_probe").get()?.count, 3);
105
- }).pipe(
106
- Effect.scoped,
107
- Effect.provide(persistence),
108
- Effect.ensuring(Effect.sync(() => NodeFS.rmSync(tempDir, { recursive: true, force: true }))),
109
- );
110
-});
111
-
38
// Linux names an open, already-deleted file "<path> (deleted)" in /proc/self/fd.
39
// SQLite names its temp files "etilqs_*" and, without TMPDIR, puts them in
40
// /var/tmp while os.tmpdir() is /tmp, so match the name, not the directory.
@@ -223,6 +149,7 @@ it.effect("upgrades an existing database on a read-write worker connection", ()
149
assert.deepInclude(workerData.slice(startedBefore), {
150
filename: dbPath,
151
readOnly: false,
152
+ requireWal: false,
153
allowExtension: false,
154
prepareCacheSize: 200,
155
});
apps/server/src/persistence/Sqlite.ts
+5
-27
@@ -64,26 +64,6 @@ const runMigrationsAsOwner = Effect.fn("SqlitePersistence.runMigrationsAsOwner")
64
yield* runMigrations();
65
}, Effect.scoped);
66
67
-// Index builds over large tables take minutes; on the main connection they block the event loop,
68
-// so requests hang instead of getting the listener's 503. A fresh database stays on the main
69
-// connection, which must create the first table for its auto_vacuum setting to apply.
70
-const runMigrationsOffMainThread = Effect.fn("SqlitePersistence.runMigrationsOffMainThread")(
71
- function* (dbPath: string) {
72
- const sql = yield* SqlClient.SqlClient;
73
- const tables = yield* sql<{
74
- readonly count: number;
75
- }>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
76
- if (!tables[0]?.count || !(yield* hasPendingMigrations())) return yield* runMigrations();
77
- return yield* Effect.gen(function* () {
78
- const worker = yield* SqlClient.SqlClient;
79
- yield* worker`PRAGMA foreign_keys = ON;`;
80
- yield* worker`PRAGMA synchronous = NORMAL;`;
81
- yield* worker`PRAGMA temp_store = MEMORY;`;
82
- return yield* runMigrations();
83
- }).pipe(Effect.provide(NodeSqliteClient.layerWorker(dbPath)));
84
- },
85
-);
86
-
67
const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
68
Layer.effectDiscard(
69
Effect.gen(function* () {
@@ -97,12 +77,12 @@ const makeLayerSetup = <E, R>(migrate: Effect.Effect<unknown, E, R>) =>
77
}>`SELECT COUNT(*) AS count FROM sqlite_schema WHERE type = 'table'`;
78
if (tables[0]?.count === 0) yield* sql`PRAGMA auto_vacuum = INCREMENTAL;`;
79
yield* sql`PRAGMA journal_mode = WAL;`;
100
- // FULL fsyncs every event commit on the main thread; NORMAL syncs only at WAL checkpoints.
80
+ // FULL fsyncs every event commit; NORMAL syncs only at WAL checkpoints.
81
// App crashes lose nothing; OS crashes or power loss can roll back post-checkpoint commits without corruption.
82
yield* sql`PRAGMA synchronous = NORMAL;`;
83
// macOS fsync misses drive caches; F_FULLFSYNC flushes them at checkpoints and is ignored elsewhere.
84
yield* sql`PRAGMA checkpoint_fullfsync = ON;`;
105
- // Keep statement journals and sort spills off the main thread's disk path.
85
+ // Keep statement journals and sort spills off the disk.
86
// Large statements and open savepoints can increase peak memory use.
87
yield* sql`PRAGMA temp_store = MEMORY;`;
88
// PASSIVE checkpoints retain peak WAL size until the last connection closes.
@@ -131,8 +111,8 @@ const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<neve
111
NodeSqliteClient.layer({
112
filename: dbPath,
113
readerWorker: true,
134
- // Run checkpoint fsyncs off the event loop so slow disk flushes cannot stall clients.
135
- checkpointWorker: true,
114
+ // A slow disk then delays writes instead of freezing every client.
115
+ writerWorker: true,
116
spanAttributes: {
117
"db.name": path.basename(dbPath),
118
"service.name": "t3code-server",
@@ -152,9 +132,7 @@ const makeLayerFromPath = <E, R>(makeSetup: (dbPath: string) => Layer.Layer<neve
132
);
133
}, Layer.unwrap);
134
155
-export const layerFromPath = makeLayerFromPath((dbPath) =>
156
- makeLayerSetup(runMigrationsOffMainThread(dbPath)),
157
-);
135
+export const layerFromPath = makeLayerFromPath(() => layerSetup);
136
137
export const layerMemory = Layer.provideMerge(
138
layerSetup,
apps/server/src/project/ProjectService.test.ts
+6
@@ -552,6 +552,12 @@ it.layer(layerTest)("ProjectService", (it) => {
552
title: "Delete retry",
553
workspaceRoot: "/work/delete-retry",
554
});
555
+ // Enrichment landing between the two deletes would change the project the retry reads.
556
+ yield* waitForProject(
557
+ service,
558
+ projectId,
559
+ (project) => project.faviconPath !== null && project.repositoryIdentity !== null,
560
+ );
561
const input = { commandId: CommandId.make("command:delete-retry:delete"), projectId };
562
const deleted = yield* service.delete(input);
563
assert.deepEqual(yield* service.delete(input), deleted);
docs/operations/observability.md
+3
-2
@@ -57,8 +57,9 @@ The `TraceRecord`, `EffectTraceRecord`, and `OtlpTraceRecord` schemas live in
57
`packages/shared/src/observability.ts`.
58
59
SQL statement spans (named `sqlite`) and `sql.transaction` spans carry `db.connection` and
60
-`db.lock_wait_ms`. `db.connection` is `writer` for the main connection, which runs on the event loop,
61
-or `reader` for the read-only connection in a worker thread. `db.lock_wait_ms` is the part of the
60
+`db.lock_wait_ms`. `db.connection` is `writer` for the read-write connection or `reader` for a
61
+read-only one. For a database file both run in worker threads; an in-memory database has only a
62
+writer, on the event loop. `db.lock_wait_ms` is the part of the
63
span spent waiting for that connection, behind other SQL. That wait does not block the event loop,
64
so subtract it before you blame a slow span on the query. Statements inside a transaction use its
65
connection and do not carry these attributes.
packages/shared/src/nodeSqliteClient.test.ts
+87
-89
@@ -23,6 +23,7 @@ import * as SqliteClient from "./nodeSqliteClient.ts";
23
const startedWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
24
const silentWorkers = vi.hoisted(() => ({ on: false, readyOnTerminate: false }));
25
const failingWorkers = vi.hoisted(() => ({ message: undefined as string | undefined }));
26
+const checkpointWorkers = vi.hoisted((): Array<NodeWorkerThreads.Worker> => []);
27
28
vi.mock("node:worker_threads", async (importOriginal) => {
29
const actual = await importOriginal<typeof NodeWorkerThreads>();
@@ -36,6 +37,7 @@ vi.mock("node:worker_threads", async (importOriginal) => {
37
: args) as ConstructorParameters<typeof actual.Worker>),
38
);
39
startedWorkers.push(this);
40
+ if (args[1]?.workerData?.frames !== undefined) checkpointWorkers.push(this);
41
}
42
override terminate() {
43
// A "ready" already queued when termination starts reaches the parent before "exit".
@@ -222,7 +224,7 @@ it.effect("waits for another process's lock while the database opens", () =>
224
yield* Effect.gen(function* () {
225
const sql = yield* SqlClient.SqlClient;
226
assert.deepEqual(yield* sql`SELECT COUNT(*) FROM entries`.values, [[0]]);
225
- }).pipe(Effect.provide(SqliteClient.layer({ filename, checkpointWorker: true })));
227
+ }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
228
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
229
);
230
@@ -244,6 +246,52 @@ it.effect("reports a transaction blocked by another writer as a lock timeout", (
246
}).pipe(Effect.provide(NodeServices.layer)),
247
);
248
249
+it.effect("lets the event loop run while a write waits for another connection's lock", () =>
250
+ Effect.gen(function* () {
251
+ const { filename, other } = yield* makeTempDatabase;
252
+ yield* Effect.gen(function* () {
253
+ const sql = yield* SqlClient.SqlClient;
254
+ yield* sql`PRAGMA busy_timeout = 5000`;
255
+ yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
256
+ const write = yield* Effect.forkChild(
257
+ sql`UPDATE counters SET value = value + 1 WHERE id = 1`,
258
+ { startImmediately: true },
259
+ );
260
+ // A writer on the event loop sleeps in SQLite's busy handler here, so this commit
261
+ // cannot run until that write has already timed out.
262
+ yield* eventLoopTurn;
263
+ yield* Effect.sync(() => other.exec("UPDATE counters SET value = 10 WHERE id = 1; COMMIT"));
264
+ yield* Fiber.join(write);
265
+ assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[11]]);
266
+ }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
267
+ }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
268
+);
269
+
270
+it.effect("finishes a write already sent to the writer before its caller is interrupted", () =>
271
+ Effect.gen(function* () {
272
+ const { filename, other } = yield* makeTempDatabase;
273
+ yield* Effect.gen(function* () {
274
+ const sql = yield* SqlClient.SqlClient;
275
+ yield* sql`PRAGMA busy_timeout = 5000`;
276
+ yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
277
+ const write = yield* Effect.forkChild(
278
+ sql`UPDATE counters SET value = value + 1 WHERE id = 1`,
279
+ { startImmediately: true },
280
+ );
281
+ yield* eventLoopTurn;
282
+ const interrupted = yield* Effect.forkChild(Fiber.interrupt(write), {
283
+ startImmediately: true,
284
+ });
285
+ yield* eventLoopTurn;
286
+ // The write is waiting in the worker for the lock, so the interrupt has to wait too.
287
+ assert.isUndefined(interrupted.pollUnsafe());
288
+ yield* Effect.sync(() => other.exec("COMMIT"));
289
+ yield* Fiber.join(interrupted);
290
+ assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[1]]);
291
+ }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
292
+ }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
293
+);
294
+
295
it.effect("lets the writer commit while a read-only transaction is open", () =>
296
withReaderClient((sql) =>
297
Effect.gen(function* () {
@@ -420,7 +468,7 @@ it.effect("fails with a typed SqlError when the database file is not SQLite", ()
468
const filename = path.join(directory, "state.sqlite");
469
yield* fs.writeFileString(filename, "not a db\n");
470
423
- const exit = yield* Layer.build(SqliteClient.layer({ filename, checkpointWorker: true })).pipe(
471
+ const exit = yield* Layer.build(SqliteClient.layer({ filename, writerWorker: true })).pipe(
472
Effect.exit,
473
);
474
@@ -514,45 +562,13 @@ it.effect.each([
562
}).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))),
563
);
564
517
-const PAYLOAD = new Uint8Array(1024 * 1024).fill(7);
518
-const WAL_SIZE_LIMIT = 32 * 1024 * 1024;
519
-
520
-// Fails instead of hanging when admission holds the write: the worker retries a blocked
521
-// checkpoint on its own, so two blocked reports for one WAL generation mean a held write.
522
-const insertUnlessHeld = Effect.fnUntraced(function* (
523
- sql: SqlClient.SqlClient,
524
- worker: NodeWorkerThreads.Worker,
525
- label: string,
526
-) {
527
- const stalled = yield* Deferred.make<void>();
528
- let seen: unknown;
529
- const onMessage = (message: { type?: unknown; blocked?: unknown; generation?: unknown }) => {
530
- if (message?.type !== "checkpoint" || message.blocked !== true) return;
531
- if (message.generation === seen) Deferred.doneUnsafe(stalled, Effect.void);
532
- seen = message.generation;
533
- };
534
- worker.on("message", onMessage);
535
- const held = yield* Effect.raceFirst(
536
- sql`INSERT INTO blobs VALUES (${PAYLOAD})`.pipe(Effect.as(false)),
537
- Deferred.await(stalled).pipe(Effect.as(true)),
538
- ).pipe(Effect.ensuring(Effect.sync(() => worker.off("message", onMessage))));
539
- if (held) assert.fail(`${label} is held by admission while an outside reader pins the WAL`);
540
-});
541
-
542
-// `outside` stands in for `sqlite3 statev2.sqlite` or a backup script reading the database.
543
-const withOutsideConnection = <A, E>(
544
- use: (options: {
545
- readonly sql: SqlClient.SqlClient;
546
- readonly worker: NodeWorkerThreads.Worker;
547
- readonly outside: NodeSqlite.DatabaseSync;
548
- readonly walSize: Effect.Effect<number>;
549
- }) => Effect.Effect<A, E, SqlClient.SqlClient>,
550
-) =>
565
+it.effect("keeps writes moving while an outside reader pins the WAL, then trims the WAL", () =>
566
Effect.gen(function* () {
567
const fs = yield* FileSystem.FileSystem;
568
const path = yield* Path.Path;
569
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-sqlite-pinned-wal-" });
570
const filename = path.join(directory, "state.sqlite");
571
+ // Stands in for `sqlite3 statev2.sqlite` or a backup script reading the database.
572
const outside = yield* Effect.acquireRelease(
573
Effect.sync(() => new NodeSqlite.DatabaseSync(filename)),
574
(db) =>
@@ -561,64 +577,46 @@ const withOutsideConnection = <A, E>(
577
db.close();
578
}),
579
);
564
- outside.exec(`
565
- PRAGMA journal_mode = WAL;
566
- PRAGMA wal_autocheckpoint = 0;
567
- CREATE TABLE blobs(data BLOB NOT NULL);
568
- `);
580
+ outside.exec("PRAGMA journal_mode = WAL; CREATE TABLE blobs(data BLOB NOT NULL)");
581
const walSize = fs.stat(`${filename}-wal`).pipe(
582
Effect.map((info) => Number(info.size)),
583
Effect.orDie,
584
);
585
+ const limit = 8 * 1024 * 1024;
586
+ const payload = new Uint8Array(1024 * 1024).fill(7);
587
yield* Effect.gen(function* () {
588
const sql = yield* SqlClient.SqlClient;
575
- yield* sql.unsafe(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT}`);
576
- return yield* use({ sql, worker: startedWorkers.at(-1)!, outside, walSize });
577
- }).pipe(Effect.provide(SqliteClient.layer({ filename, checkpointWorker: true })));
578
- }).pipe(Effect.scoped, Effect.provide(NodeServices.layer));
579
-
580
-const openReadTransaction = (db: NodeSqlite.DatabaseSync) => {
581
- db.exec("BEGIN");
582
- db.prepare("SELECT COUNT(*) FROM blobs").get();
583
-};
584
-
585
-it.effect("keeps admitting writes while an outside read transaction pins the WAL", () =>
586
- withOutsideConnection(({ sql, worker, outside }) =>
587
- Effect.gen(function* () {
588
- yield* sql`INSERT INTO blobs VALUES (${PAYLOAD})`;
589
- openReadTransaction(outside);
590
- // 140 MiB passes both the 32 MiB size limit and the 32,768-frame ceiling.
591
- for (let write = 1; write <= 140; write++) {
592
- yield* insertUnlessHeld(sql, worker, `write ${write}`);
593
- }
594
- const rows = yield* sql<{ count: number }>`SELECT COUNT(*) AS count FROM blobs`;
595
- assert.equal(rows[0]?.count, 141);
596
- }),
597
- ),
598
-);
599
-
600
-it.effect("keeps admitting writes while a reader stops TRUNCATE, then resets the WAL", () =>
601
- withOutsideConnection(({ sql, worker, outside, walSize }) =>
602
- Effect.gen(function* () {
603
- const insert = outside.prepare("INSERT INTO blobs VALUES (?)");
604
- for (let write = 1; write <= 40; write++) insert.run(PAYLOAD);
605
- // A snapshot of an unbackfilled WAL lets PASSIVE finish but stops TRUNCATE.
606
- openReadTransaction(outside);
607
- const stopped = yield* Deferred.make<void>();
608
- const reset = yield* Deferred.make<void>();
609
- worker.on("message", (message: { type?: unknown; truncateId?: unknown; log?: unknown }) => {
610
- if (message?.type !== "checkpoint" || message.truncateId === undefined) return;
611
- Deferred.doneUnsafe(message.log === 0 ? reset : stopped, Effect.void);
612
- });
613
- // Starts a checkpoint without appending a frame past the reader's snapshot.
614
- yield* sql`DELETE FROM blobs WHERE 0`;
615
- yield* Deferred.await(stopped);
616
- yield* insertUnlessHeld(sql, worker, "a write after the failed TRUNCATE");
617
- assert.isAbove(yield* walSize, WAL_SIZE_LIMIT);
589
+ yield* sql.unsafe(`PRAGMA journal_size_limit = ${limit}`);
590
+ yield* sql`INSERT INTO blobs VALUES (${payload})`;
591
+ outside.exec("BEGIN");
592
+ outside.prepare("SELECT COUNT(*) FROM blobs").get();
593
+ for (let write = 0; write < 24; write++) yield* sql`INSERT INTO blobs VALUES (${payload})`;
594
+ assert.isAbove(yield* walSize, limit);
595
outside.exec("COMMIT");
619
- // No further write arrives; the worker has to retry on its own.
620
- yield* Deferred.await(reset);
621
- assert.equal(yield* walSize, 0);
622
- }),
623
- ),
596
+ const checkpointer = checkpointWorkers.at(-1)!;
597
+ const caughtUp = Effect.callback<void>((resume) => {
598
+ const onMessage = (message: { type?: unknown; log?: unknown; checkpointed?: unknown }) => {
599
+ if (
600
+ message?.type === "checked" &&
601
+ typeof message.log === "number" &&
602
+ message.log > 0 &&
603
+ message.log === message.checkpointed
604
+ ) {
605
+ resume(Effect.void);
606
+ }
607
+ };
608
+ checkpointer.on("message", onMessage);
609
+ checkpointer.ref();
610
+ return Effect.sync(() => {
611
+ checkpointer.off("message", onMessage);
612
+ checkpointer.unref();
613
+ });
614
+ });
615
+ // The worker checkpoints the released frames; the next write restarts the WAL at the limit.
616
+ yield* Effect.all([caughtUp, sql`DELETE FROM blobs WHERE rowid = 1`], { concurrency: 2 });
617
+ yield* sql`DELETE FROM blobs WHERE rowid = 2`;
618
+ assert.isAtMost(yield* walSize, limit);
619
+ assert.deepEqual(yield* sql`SELECT COUNT(*) FROM blobs`.values, [[23]]);
620
+ }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
621
+ }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
622
);
packages/shared/src/nodeSqliteClient.ts
+285
-618
@@ -35,8 +35,8 @@ export interface SqliteClientConfig {
35
readonly spanAttributes?: Record<string, unknown> | undefined;
36
/** Ignored for in-memory databases, which cannot share a second connection. */
37
readonly readerWorker?: boolean | undefined;
38
- /** Ignored for in-memory databases. */
39
- readonly checkpointWorker?: boolean | undefined;
38
+ /** Runs the writer and WAL checkpoints off the event loop. Ignored for in-memory databases. */
39
+ readonly writerWorker?: boolean | undefined;
40
}
41
42
// SqlClient masks transaction acquisition, so readOnly carries the caller's interruption policy.
@@ -194,9 +194,15 @@ const hasLargeBinding = (params: ReadonlyArray<unknown>): boolean => {
194
};
195
196
// Inline worker source also runs in bundles and single-executable builds without module files.
197
-const readerWorkerSource = `
197
+const workerSource = `
198
const { parentPort, workerData } = require("node:worker_threads");
199
const { DatabaseSync } = require("node:sqlite");
200
+const describe = (error) => ({
201
+ message: String(error && error.message ? error.message : error),
202
+ code: error && error.code,
203
+ errcode: error && error.errcode,
204
+ errstr: error && error.errstr,
205
+});
206
let db;
207
try {
208
// A read-only open cannot be undone by SQL, unlike PRAGMA query_only.
@@ -205,13 +211,14 @@ try {
211
allowExtension: workerData.allowExtension,
212
});
213
next.exec("PRAGMA busy_timeout = 5000;");
208
- // Without WAL a reader's shared lock blocks the writer's commits, so refuse.
214
+ // Reads the header, so a file that is not a database fails here rather than on first use.
215
const mode = String(next.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
210
- if (mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
216
+ // Without WAL a reader's shared lock blocks the writer's commits, so refuse.
217
+ if (workerData.requireWal && mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
218
db = next;
219
parentPort.postMessage({ type: "ready" });
220
} catch (error) {
214
- parentPort.postMessage({ type: "unavailable", message: String(error && error.message ? error.message : error) });
221
+ parentPort.postMessage({ type: "unavailable", ...describe(error) });
222
}
223
const statements = new Map();
224
const prepare = (sql, cached) => {
@@ -226,10 +233,19 @@ const prepare = (sql, cached) => {
233
statements.set(sql, statement);
234
return statement;
235
};
229
-parentPort.on("message", ({ id, sql, params, cached, values, raw, safeIntegers }) => {
236
+parentPort.on("message", (request) => {
237
+ if (request.type === "close") {
238
+ // Closing lets SQLite checkpoint and remove the WAL, which terminating would skip.
239
+ try {
240
+ if (db !== undefined) db.close();
241
+ } catch {}
242
+ parentPort.close();
243
+ return;
244
+ }
245
+ const { id, sql, params, cached, values, raw, safeIntegers } = request;
246
let operation = "open";
247
try {
232
- if (db === undefined) throw new Error("database reader is unavailable");
248
+ if (db === undefined) throw new Error("database connection is unavailable");
249
operation = "prepare";
250
const statement = prepare(sql, cached);
251
operation = "execute";
@@ -242,21 +258,12 @@ parentPort.on("message", ({ id, sql, params, cached, values, raw, safeIntegers }
258
parentPort.postMessage({ id, rows: raw ? result : [] });
259
}
260
} catch (error) {
245
- parentPort.postMessage({
246
- id,
247
- error: {
248
- operation,
249
- message: String(error && error.message ? error.message : error),
250
- code: error && error.code,
251
- errcode: error && error.errcode,
252
- errstr: error && error.errstr,
253
- },
254
- });
261
+ parentPort.postMessage({ id, error: { operation, ...describe(error) } });
262
}
263
});
264
`;
265
259
-interface ReaderRequest {
266
+interface WorkerRequest {
267
readonly sql: string;
268
readonly params: ReadonlyArray<unknown>;
269
readonly cached: boolean;
@@ -264,24 +271,25 @@ interface ReaderRequest {
271
readonly raw: boolean;
272
}
273
267
-type ReaderStartup =
274
+interface WorkerFailure {
275
+ readonly message: string;
276
+ readonly code?: unknown;
277
+ readonly errcode?: unknown;
278
+ readonly errstr?: unknown;
279
+}
280
+
281
+type WorkerStartup =
282
| { readonly type: "ready" }
269
- | { readonly type: "unavailable"; readonly message: string };
283
+ | ({ readonly type: "unavailable" } & WorkerFailure);
284
271
-type ReaderReply =
285
+type WorkerReply =
286
| { readonly id: number; readonly rows: ReadonlyArray<any> }
287
| {
288
readonly id: number;
275
- readonly error: {
276
- readonly operation: "open" | "prepare" | "execute";
277
- readonly message: string;
278
- readonly code?: unknown;
279
- readonly errcode?: unknown;
280
- readonly errstr?: unknown;
281
- };
289
+ readonly error: WorkerFailure & { readonly operation: "open" | "prepare" | "execute" };
290
};
291
284
-const readerFailureMessages = {
292
+const workerFailureMessages = {
293
open: "Failed to open database",
294
prepare: "Failed to prepare statement",
295
execute: "Failed to execute statement",
@@ -290,48 +298,66 @@ const readerFailureMessages = {
298
const failed = (cause: unknown, message: string, operation: string) =>
299
new SqlError({ reason: classifyError(cause, message, operation) });
300
301
+const failedWith = (
302
+ { message, code, errcode, errstr }: WorkerFailure,
303
+ operation: keyof typeof workerFailureMessages,
304
+) =>
305
+ failed(
306
+ Object.assign(new Error(message), { code, errcode, errstr }),
307
+ workerFailureMessages[operation],
308
+ operation,
309
+ );
310
+
311
// Read-transaction cleanup succeeds after worker exit because it can no longer change anything.
312
const endsReadTransaction = (sql: string) => /^\s*ROLLBACK\b|^\s*COMMIT\b/i.test(sql);
313
314
// WAL readers run concurrently; one worker serialized every read behind the slowest transaction.
315
const READER_POOL_SIZE = 4;
316
+const READER_STARTUP_TIMEOUT = Duration.seconds(10);
317
318
// Bind transactions to one worker; fail after its exit instead of resuming outside the transaction on a replacement.
300
-const makeReader = Effect.fnUntraced(function* (
301
- options: SqliteClientConfig,
302
- pool: { unavailable: boolean },
303
- readOnly = true,
304
-) {
319
+const makeWorkerConnection = Effect.fnUntraced(function* (options: {
320
+ readonly filename: string;
321
+ readonly role: "reader" | "writer";
322
+ readonly readOnly: boolean;
323
+ readonly onReply?: () => void;
324
+}) {
325
+ const label = options.role === "reader" ? "Database reader" : "Database writer";
326
+ const readOnly = options.readOnly;
327
let current:
328
| {
329
readonly worker: NodeWorkerThreads.Worker;
330
readonly connection: Connection;
309
- readonly started: Promise<ReaderStartup>;
331
+ readonly started: Promise<WorkerStartup>;
332
}
333
| undefined;
334
let nextRequestId = 0;
335
+ // A replacement writer would lose the pragmas callers set on it, such as foreign_keys.
336
+ let writerStopped: string | undefined;
337
338
const start = () => {
315
- const worker = new NodeWorkerThreads.Worker(readerWorkerSource, {
339
+ const worker = new NodeWorkerThreads.Worker(workerSource, {
340
eval: true,
341
workerData: {
342
filename: options.filename,
343
readOnly,
344
+ requireWal: options.role === "reader",
345
allowExtension: false,
346
prepareCacheSize: 200,
347
},
348
});
324
- let announce: (startup: ReaderStartup) => void = () => {};
325
- const started = new Promise<ReaderStartup>((resolve) => {
349
+ let announce: (startup: WorkerStartup) => void = () => {};
350
+ const started = new Promise<WorkerStartup>((resolve) => {
351
announce = resolve;
352
});
353
let stoppedMessage: string | undefined;
354
const pending = new Map<
355
number,
331
- { readonly endsTransaction: boolean; readonly resume: (reply: ReaderReply) => void }
356
+ { readonly endsTransaction: boolean; readonly resume: (reply: WorkerReply) => void }
357
>();
358
const stop = (message: string) => {
359
stoppedMessage = message;
360
+ if (options.role === "writer") writerStopped = message;
361
announce({ type: "unavailable", message });
362
if (current?.worker === worker) current = undefined;
363
for (const [id, request] of pending) {
@@ -343,7 +369,7 @@ const makeReader = Effect.fnUntraced(function* (
369
}
370
pending.clear();
371
};
346
- worker.on("message", (reply: ReaderReply | ReaderStartup) => {
372
+ worker.on("message", (reply: WorkerReply | WorkerStartup) => {
373
if ("type" in reply) {
374
announce(reply);
375
if (pending.size === 0) worker.unref();
@@ -353,12 +379,13 @@ const makeReader = Effect.fnUntraced(function* (
379
if (request === undefined) return;
380
pending.delete(reply.id);
381
if (pending.size === 0) worker.unref();
382
+ options.onReply?.();
383
request.resume(reply);
384
});
358
- worker.on("error", (cause) => stop(`Database reader failed: ${cause.message}`));
359
- worker.on("exit", (code) => stop(`Database reader exited with code ${code}`));
385
+ worker.on("error", (cause) => stop(`${label} failed: ${cause.message}`));
386
+ worker.on("exit", (code) => stop(`${label} exited with code ${code}`));
387
361
- const request = (input: ReaderRequest) =>
388
+ const request = (input: WorkerRequest) =>
389
Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
390
// A COMMIT that never reached a writer must fail; only its ROLLBACK is moot.
391
const endsTransaction = readOnly
@@ -367,37 +394,28 @@ const makeReader = Effect.fnUntraced(function* (
394
if (stoppedMessage !== undefined) {
395
return endsTransaction
396
? Effect.succeed([])
370
- : Effect.fail(
371
- failed(new Error(stoppedMessage), readerFailureMessages.execute, "execute"),
372
- );
397
+ : Effect.fail(failedWith({ message: stoppedMessage }, "execute"));
398
}
399
const safeIntegers = Boolean(Context.get(fiber.context, Client.SafeIntegers));
375
- return Effect.callback<ReadonlyArray<any>, SqlError>((resume) => {
400
+ const reply = Effect.callback<ReadonlyArray<any>, SqlError>((resume) => {
401
const id = nextRequestId++;
402
pending.set(id, {
403
endsTransaction,
379
- resume: (reply) => {
380
- if ("rows" in reply) {
381
- resume(Effect.succeed(reply.rows));
382
- return;
383
- }
384
- const { operation, message, ...codes } = reply.error;
404
+ resume: (reply) =>
405
resume(
386
- Effect.fail(
387
- failed(
388
- Object.assign(new Error(message), codes),
389
- readerFailureMessages[operation],
390
- operation,
391
- ),
392
- ),
393
- );
394
- },
406
+ "rows" in reply
407
+ ? Effect.succeed(reply.rows)
408
+ : Effect.fail(failedWith(reply.error, reply.error.operation)),
409
+ ),
410
});
411
if (pending.size === 1) worker.ref();
412
const cached = input.cached && !hasLargeBinding(input.params);
413
// oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
414
worker.postMessage({ id, safeIntegers, ...input, cached });
415
});
416
+ // A posted write runs to the end, as it did on the event loop; the caller and its lock
417
+ // must not move on while the worker is still applying it. Reads stay cancellable.
418
+ return options.role === "writer" ? Effect.uninterruptible(reply) : reply;
419
});
420
421
const connection = identity<Connection>({
@@ -436,220 +454,154 @@ const makeReader = Effect.fnUntraced(function* (
454
455
yield* Effect.addFinalizer(() =>
456
Effect.promise(async () => {
439
- await current?.worker.terminate();
457
+ const open = current;
458
+ if (open === undefined) return;
459
+ if (options.role === "writer") {
460
+ await new Promise<void>((resolve) => {
461
+ open.worker.once("exit", () => resolve());
462
+ open.worker.ref();
463
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
464
+ open.worker.postMessage({ type: "close" });
465
+ });
466
+ }
467
+ await open.worker.terminate();
468
}),
469
);
470
443
- const startupTimeout = Duration.seconds(10);
444
-
445
- const turnOff = (reader: typeof current, reason: string) =>
446
- Effect.suspend(() => {
447
- if (reader !== undefined) {
448
- if (current === reader) current = undefined;
449
- void reader.worker.terminate();
450
- }
451
- if (pool.unavailable) return Effect.undefined;
452
- pool.unavailable = true;
453
- return Effect.logWarning("SQLite reader worker unavailable; reads use the main connection", {
454
- reason,
455
- }).pipe(Effect.as(undefined));
456
- });
471
+ const retire = (worker: NodeWorkerThreads.Worker) => {
472
+ if (current?.worker === worker) current = undefined;
473
+ void worker.terminate();
474
+ };
475
458
- return Effect.suspend((): Effect.Effect<Connection | undefined> => {
459
- if (pool.unavailable) return Effect.undefined;
460
- let reader: NonNullable<typeof current>;
476
+ return Effect.suspend((): Effect.Effect<Connection, SqlError> => {
477
+ if (current === undefined && writerStopped !== undefined) {
478
+ return Effect.fail(failedWith({ message: writerStopped }, "execute"));
479
+ }
480
+ let entry: NonNullable<typeof current>;
481
try {
462
- reader = current ?? start();
482
+ entry = current ?? start();
483
} catch (cause) {
464
- return turnOff(undefined, String(cause));
484
+ return Effect.fail(failed(cause, workerFailureMessages.open, "open"));
485
}
486
+ const started = Effect.promise(() => entry.started);
487
return Effect.flatMap(
488
Effect.withFiber((fiber) => {
489
const restore = Context.get(fiber.context, ReadOnly) ?? identity;
490
+ // Readers fall back to the writer; the writer has no fallback, so a slow disk only delays it.
491
return restore(
470
- Effect.timeoutOption(
471
- Effect.promise(() => reader.started),
472
- startupTimeout,
473
- ),
492
+ options.role === "reader"
493
+ ? Effect.timeoutOption(started, READER_STARTUP_TIMEOUT)
494
+ : Effect.asSome(started),
495
);
496
}),
476
- (startup): Effect.Effect<Connection | undefined> =>
477
- // A late ready event must not hand out a worker retired by another acquire timeout.
478
- pool.unavailable || current !== reader
479
- ? Effect.undefined
480
- : Option.isSome(startup) && startup.value.type === "ready"
481
- ? Effect.succeed(reader.connection)
482
- : turnOff(
483
- reader,
484
- Option.isSome(startup) && startup.value.type === "unavailable"
485
- ? startup.value.message
486
- : `worker did not open the database within ${Duration.toMillis(startupTimeout)} ms`,
487
- ),
497
+ (startup): Effect.Effect<Connection, SqlError> => {
498
+ const outcome = Option.getOrUndefined(startup);
499
+ if (outcome?.type === "ready") {
500
+ // A late ready event must not hand out a worker retired by another acquire timeout.
501
+ return current === entry
502
+ ? Effect.succeed(entry.connection)
503
+ : Effect.fail(failedWith({ message: `${label} stopped while starting` }, "open"));
504
+ }
505
+ retire(entry.worker);
506
+ return Effect.fail(
507
+ failedWith(
508
+ outcome ?? {
509
+ message: `${label} did not open the database within ${Duration.toMillis(READER_STARTUP_TIMEOUT)} ms`,
510
+ },
511
+ "open",
512
+ ),
513
+ );
514
+ },
515
);
516
});
517
});
518
519
+// A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
520
+const makeReader = Effect.fnUntraced(function* (filename: string, pool: { unavailable: boolean }) {
521
+ const acquire = yield* makeWorkerConnection({ filename, role: "reader", readOnly: true });
522
+ return Effect.suspend((): Effect.Effect<Connection | undefined> =>
523
+ pool.unavailable
524
+ ? Effect.undefined
525
+ : acquire.pipe(
526
+ Effect.map((connection) => (pool.unavailable ? undefined : connection)),
527
+ Effect.catch((error) => {
528
+ if (pool.unavailable) return Effect.undefined;
529
+ pool.unavailable = true;
530
+ const cause = error.reason.cause;
531
+ return Effect.logWarning("SQLite reader worker unavailable; reads use the writer", {
532
+ reason: cause instanceof Error ? cause.message : error.message,
533
+ }).pipe(Effect.as(undefined));
534
+ }),
535
+ ),
536
+ );
537
+});
538
+
539
+// A checkpoint copies WAL pages into the database and syncs it. On the writer's thread every
540
+// write would wait for that sync, so a second connection checkpoints beside the writes.
541
const CHECKPOINT_FRAMES = 1000;
542
const CHECKPOINT_IDLE_MS = 100;
494
-const CHECKPOINT_BACKPRESSURE_FRAMES = 32_768;
495
-
496
-const readSafeSqliteInteger = (value: unknown) => {
497
- const number = typeof value === "bigint" ? Number(value) : value;
498
- return typeof number === "number" && Number.isSafeInteger(number) ? number : undefined;
499
-};
543
+// If checkpoints fall behind constant writes, the writer checkpoints itself, delaying writes only.
544
+const WRITER_BACKSTOP_FRAMES = 10_000;
545
501
-// Evaluated source also works in bundles and single-executable builds.
546
const checkpointWorkerSource = `
547
const { parentPort, workerData } = require("node:worker_threads");
548
const { DatabaseSync } = require("node:sqlite");
505
-const { statSync } = require("node:fs");
549
let db;
550
let frames;
508
-let checkpoint;
509
-let truncate;
510
-let journalSizeLimit = -1;
511
-let generation = 0;
512
-let nextTruncateId = 0;
513
-let pendingTruncate;
514
-let pendingBarriers = 0;
551
+let passive;
552
let idle;
516
-const initialize = () => {
517
- if (db !== undefined) return;
518
- // The writer switches to WAL after constructing the client.
519
- db = new DatabaseSync(workerData.filename);
520
- db.exec("PRAGMA busy_timeout = 5000;");
521
- const mode = String(db.prepare("PRAGMA journal_mode").get().journal_mode).toLowerCase();
522
- if (mode !== "wal") throw new Error("database uses journal_mode " + mode + ", not wal");
523
- // macOS requires F_FULLFSYNC to flush the drive cache.
524
- db.exec("PRAGMA checkpoint_fullfsync = ON;");
525
- frames = db.prepare("PRAGMA wal_checkpoint(NOOP)");
526
- checkpoint = db.prepare("PRAGMA wal_checkpoint(PASSIVE)");
527
- truncate = db.prepare("PRAGMA wal_checkpoint(TRUNCATE)");
528
-};
529
-const atCeiling = (state) => state.log >= workerData.backpressureFrames;
530
-const exceedsSizeLimit = () =>
531
- journalSizeLimit >= 0 && statSync(workerData.filename + "-wal").size > journalSizeLimit;
532
-// Retry a WAL past its limits even with admission open, so it resets once a pinning reader closes.
533
-const schedule = (state, retry) => {
534
- if (retry || state.log - state.checkpointed >= workerData.frames) {
535
- idle = setTimeout(() => check(undefined, undefined, true), workerData.idleMs);
553
+const check = (force) => {
554
+ if (db === undefined) {
555
+ db = new DatabaseSync(workerData.filename);
556
+ db.exec("PRAGMA busy_timeout = 5000;");
557
+ // macOS requires F_FULLFSYNC to flush the drive cache.
558
+ db.exec("PRAGMA checkpoint_fullfsync = ON;");
559
+ frames = db.prepare("PRAGMA wal_checkpoint(NOOP)");
560
+ passive = db.prepare("PRAGMA wal_checkpoint(PASSIVE)");
561
}
537
-};
538
-const check = (id, nextJournalSizeLimit, force, nextGeneration = generation) => {
539
- initialize();
540
- if (Number.isSafeInteger(nextGeneration)) generation = Math.max(generation, nextGeneration);
541
- if (Number.isSafeInteger(nextJournalSizeLimit)) journalSizeLimit = nextJournalSizeLimit;
542
- let state = frames.get();
543
- const oversized = exceedsSizeLimit();
544
- let pinned = false;
545
- if (force || state.log - state.checkpointed >= workerData.frames || oversized) {
546
- const result = checkpoint.get();
547
- // A reader's snapshot stopped the backfill; holding writes cannot let it catch up.
548
- pinned = result.checkpointed < result.log;
562
+ const before = frames.get();
563
+ let state = before;
564
+ if (state.log - state.checkpointed >= workerData.frames || (force && state.log > state.checkpointed)) {
565
+ passive.get();
566
state = frames.get();
567
}
551
- if (state.checkpointed === state.log && (atCeiling(state) || oversized)) {
552
- pendingTruncate = { truncateId: ++nextTruncateId, checkId: id, generation, state };
553
- parentPort.postMessage({
554
- type: "truncate-request",
555
- truncateId: pendingTruncate.truncateId,
556
- checkId: id,
557
- generation,
558
- ...state,
559
- });
560
- return;
568
+ // The WAL restarts only once every frame is checkpointed, so catch up when writes pause.
569
+ // A reader pinning the WAL stops progress; the next write retries instead of a timer loop.
570
+ const progressed = !force || state.checkpointed > before.checkpointed;
571
+ if (state.log > state.checkpointed && progressed) {
572
+ idle = setTimeout(() => run(true), workerData.idleMs);
573
}
562
- const blocked = !pinned && (atCeiling(state) || oversized) && state.checkpointed < state.log;
563
- parentPort.postMessage({ type: "checkpoint", id, generation, ...state, blocked });
564
- schedule(state, atCeiling(state) || oversized);
574
+ parentPort.postMessage({ type: "checked", log: state.log, checkpointed: state.checkpointed });
575
};
566
-const finishTruncate = (truncateId) => {
567
- if (pendingTruncate === undefined || pendingTruncate.truncateId !== truncateId) return;
568
- const pending = pendingTruncate;
569
- parentPort.postMessage({ type: "truncate-start", truncateId });
570
- // Admission is paused here; a pinned reader must not make truncation wait.
571
- db.exec("PRAGMA busy_timeout = 0;");
576
+const run = (force) => {
577
+ clearTimeout(idle);
578
try {
573
- truncate.get();
574
- } finally {
575
- db.exec("PRAGMA busy_timeout = 5000;");
576
- }
577
- const state = frames.get();
578
- pendingTruncate = undefined;
579
- parentPort.postMessage({
580
- type: "checkpoint",
581
- id: pending.checkId,
582
- generation: pending.generation,
583
- truncateId,
584
- ...state,
585
- // With writes held, only a reader can stop the reset, and holding them longer would not
586
- // release it.
587
- blocked: false,
588
- });
589
- while (pendingBarriers > 0) {
590
- pendingBarriers--;
591
- parentPort.postMessage("barrier");
579
+ check(force);
580
+ } catch (error) {
581
+ parentPort.postMessage({ type: "failed", message: String(error && error.message ? error.message : error) });
582
}
593
- schedule(state, atCeiling(state) || exceedsSizeLimit());
583
};
595
-parentPort.on("message", (request) => {
596
- clearTimeout(idle);
597
- if (request && request.type === "truncate") {
598
- finishTruncate(request.truncateId);
599
- return;
600
- }
601
- const isBarrier = request === "barrier";
602
- if (pendingTruncate !== undefined) {
603
- if (isBarrier) pendingBarriers++;
604
- else if (request && request.type === "check") {
605
- // Retire the request without labeling the pending snapshot with a newer generation.
606
- parentPort.postMessage({
607
- type: "checkpoint",
608
- id: request.id,
609
- generation: pendingTruncate.generation,
610
- ...pendingTruncate.state,
611
- blocked: true,
612
- });
613
- }
614
- return;
615
- }
616
- const input = request && typeof request === "object" ? request : {};
617
- check(input.id, input.journalSizeLimit, input.force, input.generation);
618
- if (isBarrier) {
619
- if (pendingTruncate !== undefined) pendingBarriers++;
620
- else parentPort.postMessage("barrier");
621
- }
622
-});
584
+parentPort.on("message", () => run(false));
585
`;
586
625
-const startCheckpointWorker = Effect.fnUntraced(function* (
626
- filename: string,
627
- writer: NodeSqlite.DatabaseSync,
628
-) {
629
- const warn = (reason: string) =>
630
- Effect.logWarning(
631
- "SQLite checkpoint worker unavailable; checkpoints run on the main connection",
632
- { reason },
633
- );
634
- if (typeof writer.isTransaction !== "boolean" || typeof writer.setAuthorizer !== "function") {
635
- yield* warn("Node SQLite lacks transaction-state or statement-authorizer support");
636
- return undefined;
587
+const sqliteVersion = () => {
588
+ const db = new NodeSqlite.DatabaseSync(":memory:");
589
+ try {
590
+ return String(db.prepare("SELECT sqlite_version() AS version").get()?.version);
591
+ } finally {
592
+ db.close();
593
}
638
- // These are the writer's first statements, before callers set their own wait. A file that is
639
- // not a database first fails here.
640
- const { version, frames, sizeLimit } = yield* Effect.try({
641
- try: () => {
642
- writer.exec("PRAGMA busy_timeout = 5000;");
643
- return {
644
- version: String(writer.prepare("SELECT sqlite_version() AS version").get()?.version),
645
- frames: writer.prepare("PRAGMA wal_checkpoint(NOOP)"),
646
- sizeLimit: writer.prepare("PRAGMA journal_size_limit"),
647
- };
648
- },
649
- catch: (cause) => failed(cause, "Failed to open database", "open"),
650
- });
594
+};
595
+
596
+// Returns the writer's per-statement notification, or undefined when the writer checkpoints alone.
597
+const startCheckpointWorker = Effect.fnUntraced(function* (filename: string) {
598
+ const warn = (reason: string) =>
599
+ Effect.logWarning("SQLite checkpoint worker unavailable; the writer checkpoints its own WAL", {
600
+ reason,
601
+ });
602
+ const version = sqliteVersion();
603
const [major = 0, minor = 0] = version.split(".").map(Number);
652
- // Older SQLite treats NOOP as PASSIVE, which would checkpoint on the writer.
604
+ // Older SQLite treats NOOP as PASSIVE, so counting frames would checkpoint on every write.
605
if (major < 3 || (major === 3 && minor < 51)) {
606
yield* warn(`SQLite ${version} has no NOOP checkpoint`);
607
return undefined;
@@ -658,12 +610,7 @@ const startCheckpointWorker = Effect.fnUntraced(function* (
610
() =>
611
new NodeWorkerThreads.Worker(checkpointWorkerSource, {
612
eval: true,
661
- workerData: {
662
- filename,
663
- frames: CHECKPOINT_FRAMES,
664
- backpressureFrames: CHECKPOINT_BACKPRESSURE_FRAMES,
665
- idleMs: CHECKPOINT_IDLE_MS,
666
- },
613
+ workerData: { filename, frames: CHECKPOINT_FRAMES, idleMs: CHECKPOINT_IDLE_MS },
614
}),
615
);
616
if (Result.isFailure(started)) {
@@ -671,199 +618,37 @@ const startCheckpointWorker = Effect.fnUntraced(function* (
618
return undefined;
619
}
620
const worker = started.success;
621
+ worker.unref();
622
const runFork = Effect.runForkWith(yield* Effect.context<never>());
623
let stopped = false;
676
- let generation = 0;
677
- let nextId = 0;
678
- let inFlight: number | undefined;
679
- let checkAgain = false;
680
- let blocked = false;
681
- let log = 0;
682
- let checkpointed = 0;
683
- let unknown = false;
684
- let truncate: { id: number; acknowledged: boolean } | undefined;
685
- type Waiter = { resume: () => void; cancelled: boolean };
686
- const waiters: Array<Waiter> = [];
687
- let elected: Waiter | undefined;
688
- const gated = () =>
689
- !stopped && !writer.isTransaction && (unknown || blocked || truncate !== undefined);
690
- const wake = () => {
691
- if (!gated() && elected === undefined) {
692
- elected = waiters.shift();
693
- elected?.resume();
694
- }
695
- if (waiters.length === 0 && elected === undefined) worker.unref();
696
- };
697
- const release = (waiter: Waiter) => {
698
- waiter.cancelled = true;
699
- const index = waiters.indexOf(waiter);
700
- if (index !== -1) waiters.splice(index, 1);
701
- if (elected === waiter) elected = undefined;
702
- wake();
703
- };
704
- const handBack = (reason: string) => {
624
+ const stop = (reason: string) => {
625
if (stopped) return;
626
stopped = true;
707
- truncate = undefined;
708
- try {
709
- writer.exec(`PRAGMA wal_autocheckpoint = ${CHECKPOINT_FRAMES};`);
710
- } catch {}
711
- wake();
627
+ void worker.terminate();
628
runFork(warn(reason));
629
};
714
- const send = (message: unknown) => {
715
- try {
716
- // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
717
- worker.postMessage(message);
718
- } catch (cause) {
719
- handBack(String(cause));
720
- }
721
- };
722
- const acknowledge = () => {
723
- if (truncate === undefined || truncate.acknowledged || stopped || writer.isTransaction) return;
724
- truncate.acknowledged = true;
725
- send({ type: "truncate", truncateId: truncate.id });
726
- };
727
- const observe = () => {
728
- try {
729
- const state = frames.get();
730
- const nextLog = readSafeSqliteInteger(state?.log);
731
- const nextCheckpointed = readSafeSqliteInteger(state?.checkpointed);
732
- if (nextLog === undefined || nextCheckpointed === undefined)
733
- throw new Error("Invalid WAL frame counts");
734
- log = nextLog;
735
- checkpointed = nextCheckpointed;
736
- unknown = false;
737
- // Past the ceiling, hold writes until the worker judges this WAL; a reader can pin it there.
738
- if (log >= CHECKPOINT_BACKPRESSURE_FRAMES) blocked = true;
739
- } catch {
740
- unknown = true;
741
- }
742
- };
743
- const request = (force = false) => {
744
- if (stopped || writer.isTransaction) return;
745
- if (truncate !== undefined) {
746
- acknowledge();
747
- return;
748
- }
749
- if (inFlight !== undefined) {
750
- checkAgain = true;
751
- return;
752
- }
753
- if (!force && !gated() && log - checkpointed < CHECKPOINT_FRAMES) return;
754
- inFlight = ++nextId;
755
- try {
756
- send({
757
- type: "check",
758
- id: inFlight,
759
- generation,
760
- journalSizeLimit: readSafeSqliteInteger(sizeLimit.get()?.journal_size_limit),
761
- force: force || gated(),
762
- });
763
- } catch (cause) {
764
- handBack(String(cause));
765
- }
766
- };
767
- const check = () => {
768
- if (stopped || writer.isTransaction) return;
769
- generation++;
770
- observe();
771
- request();
772
- wake();
773
- };
774
- worker.on("error", (cause) => handBack(cause.message));
775
- worker.on("exit", (code) => handBack(`exited with code ${code}`));
776
- worker.on("message", (message: unknown) => {
777
- if (stopped || typeof message !== "object" || message === null || !("type" in message)) return;
778
- if (
779
- message.type === "truncate-request" &&
780
- "truncateId" in message &&
781
- typeof message.truncateId === "number"
782
- ) {
783
- truncate = { id: message.truncateId, acknowledged: false };
784
- blocked = true;
785
- acknowledge();
786
- return;
787
- }
788
- if (message.type !== "checkpoint") return;
789
- const completedTruncate =
790
- truncate !== undefined && "truncateId" in message && message.truncateId === truncate.id;
791
- if (completedTruncate) truncate = undefined;
792
- // An acknowledged truncate excludes new writer mutations, so its result is current.
793
- if (
794
- (completedTruncate || ("generation" in message && message.generation === generation)) &&
795
- "log" in message &&
796
- typeof message.log === "number" &&
797
- "checkpointed" in message &&
798
- typeof message.checkpointed === "number" &&
799
- "blocked" in message &&
800
- typeof message.blocked === "boolean"
801
- ) {
802
- log = message.log;
803
- checkpointed = message.checkpointed;
804
- blocked = message.blocked;
805
- unknown = false;
806
- }
807
- // Even a stale snapshot must retire its request so a newer check can run.
808
- if ("id" in message && message.id === inFlight) inFlight = undefined;
809
- if (checkAgain && inFlight === undefined) {
810
- checkAgain = false;
811
- request(true);
812
- }
813
- wake();
630
+ worker.on("message", (message: { readonly type?: unknown; readonly message?: unknown }) => {
631
+ if (message?.type === "failed") stop(String(message.message));
632
});
633
+ worker.on("error", (cause) => stop(cause.message));
634
+ worker.on("exit", (code) => stop(`exited with code ${code}`));
635
yield* Effect.addFinalizer(() =>
636
Effect.promise(() => {
637
stopped = true;
818
- truncate = undefined;
819
- wake();
638
return worker.terminate();
639
}),
640
);
823
- writer.exec("PRAGMA wal_autocheckpoint = 0;");
824
- observe();
825
- worker.unref();
826
-
827
- const runWithAdmission = <A, E>(
828
- execute: () => Exit.Exit<A, E>,
829
- admitted?: Waiter,
830
- ): Effect.Effect<A, E> =>
831
- Effect.suspend(() => {
832
- if (!gated()) {
833
- // Release the election after executing; its successor must see the new WAL state.
834
- try {
835
- return execute();
836
- } finally {
837
- if (admitted !== undefined) release(admitted);
838
- }
839
- }
840
- if (admitted !== undefined) release(admitted);
841
- const waiter: Waiter = { resume: () => {}, cancelled: false };
842
- const wait = Effect.uninterruptibleMask((restore) =>
843
- Effect.withFiber((fiber) =>
844
- (fiber.getRef(TransactionAcquire)?.restore ?? restore)(
845
- Effect.callback<void>(function (this: Scheduler.Scheduler, resume) {
846
- const dispatcher = this.makeDispatcher();
847
- waiter.resume = () => {
848
- dispatcher.scheduleTask(() => {
849
- if (!waiter.cancelled) resume(Effect.void);
850
- }, 0);
851
- };
852
- waiters.push(waiter);
853
- worker.ref();
854
- request(true);
855
- wake();
856
- return Effect.sync(() => release(waiter));
857
- }),
858
- ),
859
- ),
860
- );
861
- return wait.pipe(
862
- Effect.flatMap(() => runWithAdmission(execute, waiter)),
863
- Effect.onInterrupt(() => Effect.sync(() => release(waiter))),
864
- );
641
+ let scheduled = false;
642
+ return () => {
643
+ if (stopped || scheduled) return;
644
+ scheduled = true;
645
+ // One check per event-loop turn is enough, however many statements finished in it.
646
+ setImmediate(() => {
647
+ scheduled = false;
648
+ // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin.
649
+ if (!stopped) worker.postMessage(0);
650
});
866
- return { check, runWithAdmission };
651
+ };
652
});
653
654
const make = Effect.fn("makeWithDatabase")(function* (
@@ -897,78 +682,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
682
}),
683
}).pipe(Effect.orDie),
684
);
900
- // Started after the close finalizer, so the worker stops before the writer closes.
901
- const checkpoint =
902
- options.checkpointWorker && isFileDatabase
903
- ? yield* startCheckpointWorker(options.filename, db)
904
- : undefined;
905
-
685
const statementReaderCache = new WeakMap<NodeSqlite.StatementSync, boolean>();
907
- interface CheckpointClassification {
908
- backpressure: boolean;
909
- check: boolean;
910
- }
911
- const checkpointClassifications = new WeakMap<
912
- NodeSqlite.StatementSync,
913
- CheckpointClassification
914
- >();
915
- const mutatingPragmasWithValues = new Set([
916
- "application_id",
917
- "auto_vacuum",
918
- "journal_mode",
919
- "schema_version",
920
- "user_version",
921
- ]);
922
- const mutatesPragma = (name: string | null, value: string | null) => {
923
- const pragma = name?.toLowerCase();
924
- if (pragma === undefined) return false;
925
- if (pragma === "incremental_vacuum" || pragma === "optimize") return true;
926
- if (pragma === "wal_checkpoint") return value?.toLowerCase() !== "noop";
927
- return value !== null && mutatingPragmasWithValues.has(pragma);
928
- };
929
- let activeClassification: CheckpointClassification | undefined;
930
- const withClassification = <A>(
931
- classification: CheckpointClassification | undefined,
932
- f: () => A,
933
- ): A => {
934
- if (classification === undefined) return f();
935
- const previous = activeClassification;
936
- activeClassification = classification;
937
- try {
938
- return f();
939
- } finally {
940
- activeClassification = previous;
941
- }
942
- };
943
- if (checkpoint !== undefined) {
944
- // SQLite may reprepare an expired statement while it executes. Keep this
945
- // callback installed and collect only around that statement's prepare/run.
946
- db.setAuthorizer((actionCode, action, argument) => {
947
- const classification = activeClassification;
948
- if (classification === undefined) return NodeSqlite.constants.SQLITE_OK;
949
- const constants = NodeSqlite.constants;
950
- if (actionCode === constants.SQLITE_TRANSACTION) {
951
- classification.backpressure ||= action === "BEGIN";
952
- classification.check ||= action === "COMMIT" || action === "ROLLBACK";
953
- } else if (actionCode === constants.SQLITE_SAVEPOINT) {
954
- classification.backpressure ||= action === "BEGIN";
955
- classification.check = true;
956
- } else if (actionCode === constants.SQLITE_PRAGMA && mutatesPragma(action, argument)) {
957
- classification.backpressure = true;
958
- classification.check = true;
959
- } else if (
960
- actionCode !== constants.SQLITE_READ &&
961
- actionCode !== constants.SQLITE_SELECT &&
962
- actionCode !== constants.SQLITE_FUNCTION &&
963
- actionCode !== constants.SQLITE_RECURSIVE &&
964
- actionCode !== constants.SQLITE_PRAGMA
965
- ) {
966
- classification.backpressure = true;
967
- classification.check = true;
968
- }
969
- return constants.SQLITE_OK;
970
- });
971
- }
686
const hasRows = (statement: NodeSqlite.StatementSync): boolean => {
687
const cached = statementReaderCache.get(statement);
688
if (cached !== undefined) {
@@ -981,12 +695,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
695
696
const prepare = (sql: string) =>
697
Effect.try({
984
- try: () => {
985
- const classification = { backpressure: false, check: false };
986
- const statement = withClassification(classification, () => db.prepare(sql));
987
- if (checkpoint !== undefined) checkpointClassifications.set(statement, classification);
988
- return statement;
989
- },
698
+ try: () => db.prepare(sql),
699
catch: (cause) =>
700
new SqlError({
701
reason: classifyError(cause, "Failed to prepare statement", "prepare"),
@@ -1008,52 +717,43 @@ const make = Effect.fn("makeWithDatabase")(function* (
717
mode: "rows" | "raw" | "values" = "rows",
718
) =>
719
Effect.withFiber<ReadonlyArray<any>, SqlError>((fiber) => {
1011
- const classification = checkpointClassifications.get(statement);
1012
- const execute = () => {
1013
- let arrays = false;
1014
- let result: Exit.Exit<ReadonlyArray<unknown>, SqlError>;
720
+ let arrays = false;
721
+ let result: Exit.Exit<ReadonlyArray<unknown>, SqlError>;
722
+ try {
723
+ statement.setReadBigInts(Boolean(Context.get(fiber.context, Client.SafeIntegers)));
724
+ if (hasRows(statement)) {
725
+ arrays = mode === "values";
726
+ if (arrays) statement.setReturnArrays(true);
727
+ result = Exit.succeed(statement.all(...(params as any)));
728
+ } else {
729
+ const value = statement.run(...(params as any));
730
+ result = Exit.succeed(
731
+ mode === "raw" ? (value as unknown as ReadonlyArray<unknown>) : [],
732
+ );
733
+ }
734
+ } catch (cause) {
735
+ result = Exit.fail(
736
+ new SqlError({
737
+ reason: classifyError(cause, "Failed to execute statement", "execute"),
738
+ }),
739
+ );
740
+ }
741
+ if (arrays) {
742
try {
1016
- const value = withClassification(classification, () => {
1017
- statement.setReadBigInts(Boolean(Context.get(fiber.context, Client.SafeIntegers)));
1018
- if (hasRows(statement)) {
1019
- arrays = mode === "values";
1020
- if (arrays) statement.setReturnArrays(true);
1021
- return statement.all(...(params as any));
1022
- }
1023
- const result = statement.run(...(params as any));
1024
- return mode === "raw" ? (result as unknown as ReadonlyArray<unknown>) : [];
1025
- });
1026
- result = Exit.succeed(value);
743
+ statement.setReturnArrays(false);
744
} catch (cause) {
1028
- result = Exit.fail(
745
+ return Exit.die(
746
new SqlError({
1030
- reason: classifyError(cause, "Failed to execute statement", "execute"),
747
+ reason: classifyError(
748
+ cause,
749
+ "Failed to reset statement result mode",
750
+ "resetResultMode",
751
+ ),
752
}),
753
);
1033
- } finally {
1034
- // OR FAIL may have committed earlier rows even when execution throws.
1035
- if (classification?.check) checkpoint?.check();
1036
- }
1037
- if (arrays) {
1038
- try {
1039
- statement.setReturnArrays(false);
1040
- } catch (cause) {
1041
- return Exit.die(
1042
- new SqlError({
1043
- reason: classifyError(
1044
- cause,
1045
- "Failed to reset statement result mode",
1046
- "resetResultMode",
1047
- ),
1048
- }),
1049
- );
1050
- }
754
}
1052
- return result;
1053
- };
1054
- return checkpoint !== undefined && classification?.backpressure
1055
- ? checkpoint.runWithAdmission(execute)
1056
- : execute();
755
+ }
756
+ return result;
757
});
758
759
const run = (sql: string, params: ReadonlyArray<unknown>, raw = false) =>
@@ -1093,14 +793,35 @@ const make = Effect.fn("makeWithDatabase")(function* (
793
});
794
795
const semaphore = yield* Semaphore.make(1);
1096
- const connection = yield* makeConnection;
1097
- // A reader that cannot start turns off the whole pool, so routing never picks a dead slot.
796
+ const writerWorker = options.writerWorker === true && isFileDatabase;
797
+ let checkpointAfterWrite: (() => void) | undefined;
798
+ const writerConnection: Effect.Effect<Connection, SqlError> = writerWorker
799
+ ? yield* makeWorkerConnection({
800
+ filename: options.filename,
801
+ role: "writer",
802
+ readOnly: options.readonly ?? false,
803
+ onReply: () => checkpointAfterWrite?.(),
804
+ })
805
+ : Effect.succeed(yield* makeConnection);
806
+ // Opening now fails the layer for a file that is not a database, not its first query.
807
+ const opened = yield* writerConnection;
808
+ // Started after the writer, so it stops first and the writer's close can checkpoint.
809
+ if (writerWorker && options.readonly !== true) {
810
+ checkpointAfterWrite = yield* startCheckpointWorker(options.filename);
811
+ if (checkpointAfterWrite !== undefined) {
812
+ yield* opened.executeUnprepared(
813
+ `PRAGMA wal_autocheckpoint = ${WRITER_BACKSTOP_FRAMES}`,
814
+ [],
815
+ undefined,
816
+ );
817
+ }
818
+ }
819
const readerPool = { unavailable: false };
820
const readers =
821
options.readerWorker && isFileDatabase
822
? yield* Effect.forEach(Array.from({ length: READER_POOL_SIZE }), () =>
823
Effect.map(
1103
- Effect.all([makeReader(options, readerPool), Semaphore.make(1)]),
824
+ Effect.all([makeReader(options.filename, readerPool), Semaphore.make(1)]),
825
([acquire, lock]) => ({
826
acquire,
827
lock,
@@ -1110,13 +831,22 @@ const make = Effect.fn("makeWithDatabase")(function* (
831
)
832
: [];
833
1113
- const writer = { name: "writer" as const, lock: semaphore, connection };
834
+ interface Target {
835
+ readonly name: Acquired["connection"];
836
+ readonly lock: Semaphore.Semaphore;
837
+ readonly connection: Connection;
838
+ }
839
+ const writer = Effect.map(writerConnection, (connection): Target => ({
840
+ name: "writer",
841
+ lock: semaphore,
842
+ connection,
843
+ }));
844
let activeWriter: ConnectionOwner | undefined;
845
// Callers must run route uninterruptibly and pass `claimed` a finalizer that runs `release` once.
846
const route = (claimed: (release: Effect.Effect<void>) => Effect.Effect<void>) =>
1117
- Effect.withFiber((fiber) => {
847
+ Effect.withFiber((fiber): Effect.Effect<Target, SqlError> => {
848
if (readers.length === 0 || readerPool.unavailable || !Context.get(fiber.context, ReadOnly)) {
1119
- return Effect.succeed(writer);
849
+ return writer;
850
}
851
// Least claimed wins and ties go to the lowest slot, so idle slots never start a worker.
852
let slot = readers[0]!;
@@ -1129,10 +859,14 @@ const make = Effect.fn("makeWithDatabase")(function* (
859
reader.claims--;
860
}),
861
),
1132
- Effect.map(reader.acquire, (readerConnection) =>
862
+ Effect.flatMap(reader.acquire, (readerConnection) =>
863
readerConnection === undefined
864
? writer
1135
- : { name: "reader" as const, lock: reader.lock, connection: readerConnection },
865
+ : Effect.succeed<Target>({
866
+ name: "reader",
867
+ lock: reader.lock,
868
+ connection: readerConnection,
869
+ }),
870
),
871
);
872
});
@@ -1146,8 +880,6 @@ const make = Effect.fn("makeWithDatabase")(function* (
880
lockWaitMs: Number((clock.monotonicTimeNanosUnsafe() - start) / 1000n) / 1000,
881
});
882
1149
- const writerBorrowBatchSize = 16;
1150
- let writerBorrowsSinceYield = 0;
883
const borrower: Borrower = (use) =>
884
Effect.uninterruptibleMask((restore) => {
885
let release: Effect.Effect<void> = Effect.void;
@@ -1160,11 +892,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
892
);
893
});
894
const borrowTarget = <A, E, R>(
1163
- target: {
1164
- readonly name: Acquired["connection"];
1165
- readonly lock: Semaphore.Semaphore;
1166
- readonly connection: Connection;
1167
- },
895
+ target: Target,
896
use: (connection: Connection) => Effect.Effect<A, E, R>,
897
): Effect.Effect<A, E, R> =>
898
Effect.withFiber((fiber) => {
@@ -1190,34 +918,7 @@ const make = Effect.fn("makeWithDatabase")(function* (
918
}),
919
)
920
: useConnection;
1193
- if (!options.checkpointWorker || target.name !== "writer") {
1194
- return target.lock.withPermits(1)(useOwnedConnection);
1195
- }
1196
- const writerUse = Effect.onExit(useOwnedConnection, (exit) =>
1197
- Effect.sync(() => {
1198
- writerBorrowsSinceYield = Exit.hasInterrupts(exit)
1199
- ? 0
1200
- : Math.min(writerBorrowsSinceYield + 1, writerBorrowBatchSize);
1201
- }),
1202
- );
1203
- return Effect.flatMap(target.lock.withPermitsIfAvailable(1)(writerUse), (result) =>
1204
- Option.isSome(result)
1205
- ? Effect.succeed(result.value)
1206
- : target.lock.withPermits(1)(
1207
- Effect.andThen(
1208
- // Failed SQL also consumes the batch; cancellation hands off immediately.
1209
- Effect.suspend(() => {
1210
- if (writerBorrowsSinceYield < writerBorrowBatchSize) return Effect.void;
1211
- writerBorrowsSinceYield = 0;
1212
- return Effect.callback<void>((resume) => {
1213
- const immediate = setImmediate(() => resume(Effect.void));
1214
- return Effect.sync(() => clearImmediate(immediate));
1215
- });
1216
- }),
1217
- writerUse,
1218
- ),
1219
- ),
1220
- );
921
+ return target.lock.withPermits(1)(useOwnedConnection);
922
});
923
const claimedUntilScopeCloses = (release: Effect.Effect<void>) =>
924
Effect.withFiber((fiber) =>
@@ -1293,39 +994,5 @@ const make = Effect.fn("makeWithDatabase")(function* (
994
});
995
});
996
1296
-const makeWorker = Effect.fn("makeWorkerClient")(function* (filename: string) {
1297
- yield* checkNodeSqliteCompat();
1298
- const acquire = yield* makeReader({ filename }, { unavailable: false }, false);
1299
- const connection = yield* acquire;
1300
- if (connection === undefined) {
1301
- return yield* failed(
1302
- new Error("database worker is unavailable"),
1303
- readerFailureMessages.open,
1304
- "open",
1305
- );
1306
- }
1307
- const { acquirer, borrower, onCommitFailure, transactionAcquirer } = Client.makeSqliteAcquirers({
1308
- connection: Effect.succeed(connection),
1309
- semaphore: yield* Semaphore.make(1),
1310
- });
1311
- return yield* Client.make({
1312
- driver: "node:sqlite",
1313
- acquirer,
1314
- borrower,
1315
- compiler: Statement.makeCompilerSqlite(),
1316
- transactionAcquirer,
1317
- onCommitFailure,
1318
- beginTransaction: "BEGIN IMMEDIATE",
1319
- spanAttributes: [[ATTR_DB_SYSTEM_NAME, "sqlite"]],
1320
- });
1321
-});
1322
-
1323
-/**
1324
- * A read-write client whose one connection runs in a worker thread, so long statements such as
1325
- * index builds leave the event loop free. Requires a file database already in WAL mode.
1326
- */
1327
-export const layerWorker = (filename: string): Layer.Layer<Client.SqlClient, SqlError> =>
1328
- Layer.effect(Client.SqlClient, makeWorker(filename)).pipe(Layer.provide(Reactivity.layer));
1329
-
997
export const layer = (config: SqliteClientConfig): Layer.Layer<Client.SqlClient, SqlError> =>
998
Layer.effect(Client.SqlClient, make(config)).pipe(Layer.provide(Reactivity.layer));