The best way to control Claude Code, Codex and more from any computer or phone

main
ts 950 lines 42.4 KB
Raw
1 import { NodeFileSystem } from "@effect/platform-node"
2 import { SqliteClient } from "@effect/sql-sqlite-node"
3 import { assert, describe, it, vi } from "@effect/vitest"
4 import { Array as Arr, Effect, Exit, FileSystem, Layer, Option, PubSub, Queue, Schema } from "effect"
5 import * as ConflictsTest from "effect-test/eventlog/EventJournalConflictsTest"
6 import * as EventGroup from "effect/eventlog/EventGroup"
7 import * as EventJournal from "effect/eventlog/EventJournal"
8 import * as EventLog from "effect/eventlog/EventLog"
9 import * as EventLogEncryption from "effect/eventlog/EventLogEncryption"
10 import * as EventLogRemote from "effect/eventlog/EventLogRemote"
11 import * as SqlEventJournal from "effect/eventlog/SqlEventJournal"
12 import { Reactivity } from "effect/reactivity"
13 import * as SqlClient from "effect/sql/SqlClient"
14 import * as SqlError from "effect/sql/SqlError"
15 import * as Statement from "effect/sql/Statement"
16 import { TestClock } from "effect/testing"
17
18 const makeJournal = Effect.gen(function*() {
19 const sql = yield* SqliteClient.make({ filename: ":memory:" })
20 return yield* SqlEventJournal.make().pipe(
21 Effect.provideService(SqlClient.SqlClient, sql)
22 )
23 }).pipe(Effect.provide(Reactivity.layer))
24
25 // `tail` fixes the random id bytes, so entries sharing a millisecond sort in a
26 // known order by id.
27 const entry = (msecs: number, event = "Repro", primaryKey = "key", tail?: number) => {
28 const id = EventJournal.makeEntryIdUnsafe({ msecs })
29 if (tail !== undefined) id.fill(tail, 6)
30 return new EventJournal.Entry({
31 id,
32 event,
33 primaryKey,
34 payload: new Uint8Array()
35 }, { disableChecks: true })
36 }
37
38 const localEntry = (primaryKey: string) => ({ event: "Repro", primaryKey, payload: new Uint8Array() })
39
40 const remoteEntries = (entries: ReadonlyArray<EventJournal.Entry>) =>
41 entries.map((entry, remoteSequence) => new EventJournal.RemoteEntry({ remoteSequence, entry }))
42
43 // Durable Object SQLite binds at most 100 parameters per statement. Rejecting
44 // larger statements makes oversized batches fail whatever the local SQLite build.
45 const limitParameters = Effect.provideService(
46 Statement.CurrentTransformer,
47 (statement) => statement.compile()[1].length > 100 ? Effect.die("too many parameters") : Effect.succeed(statement)
48 )
49
50 // Fresh module instances keep their own entry id clock. They load before any
51 // test runs, so loading them cannot stall running tests.
52 const isolate = () => {
53 vi.resetModules()
54 return Promise.all([
55 import("effect/Effect"),
56 import("effect/eventlog/EventJournal"),
57 import("effect/eventlog/SqlEventJournal"),
58 import("effect/sql/SqlClient"),
59 import("@effect/sql-sqlite-node/SqliteClient"),
60 import("effect/reactivity/Reactivity")
61 ]).then(([Effect, EventJournal, SqlEventJournal, SqlClient, SqliteClient, Reactivity]) => ({
62 Effect,
63 EventJournal,
64 SqlEventJournal,
65 SqlClient,
66 SqliteClient,
67 Reactivity
68 }))
69 }
70 // Stands in for another process.
71 const OtherProcess = await isolate()
72 // Importing the maximum timestamp can exhaust the entry id clock of its module.
73 const IsolatedRolledBackImport = await isolate()
74
75 // Saturated `rand_a` bits sort the imported entry above every local id counter
76 // in the same millisecond.
77 const importFromOtherProcess = (filename: string, msecs: number) => {
78 const { Effect, EventJournal, Reactivity, SqlClient, SqlEventJournal, SqliteClient } = OtherProcess
79 const id = EventJournal.makeEntryIdUnsafe({ msecs })
80 id[6] = 0x7f
81 id[7] = 0xff
82 const entry = new EventJournal.Entry({ id, event: "Repro", primaryKey: "key", payload: new Uint8Array() }, {
83 disableChecks: true
84 })
85 return Effect.runPromise(
86 Effect.gen(function*() {
87 const sql = yield* SqliteClient.make({ filename })
88 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
89 yield* journal.writeFromRemote({
90 remoteId: EventJournal.makeRemoteIdUnsafe(),
91 entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry })],
92 effect: () => Effect.void
93 })
94 return id
95 }).pipe(Effect.scoped, Effect.provide(Reactivity.layer))
96 )
97 }
98
99 // node:sqlite blocks the Vitest worker; concurrent tests would share a running
100 // timeout while waiting for each other's queries.
101 describe("SqlEventJournal", { concurrent: false }, () => {
102 ConflictsTest.suite(makeJournal)
103
104 it.effect("deduplicates overlapping remote entries within one incoming array", () =>
105 Effect.gen(function*() {
106 const journal = yield* makeJournal
107 const remoteId = EventJournal.makeRemoteIdUnsafe()
108 const a = entry(1_000)
109 const b = entry(2_000)
110 const handled: Array<string> = []
111 // Flattened overlapping batches [A] and [A, B], not separate journal writes.
112 const repeatedA = new EventJournal.Entry({ ...a, id: a.id.slice() as EventJournal.EntryId })
113 yield* journal.writeFromRemote({
114 remoteId,
115 entries: [a, repeatedA, b].map((entry, remoteSequence) =>
116 new EventJournal.RemoteEntry({ remoteSequence, entry })
117 ),
118 effect: (batch) =>
119 Effect.sync(() => {
120 handled.push(...batch.map(({ entry }) => entry.idString))
121 })
122 })
123
124 assert.deepStrictEqual({
125 handled,
126 stored: (yield* journal.entries).map((entry) => entry.idString),
127 nextSequence: yield* journal.nextRemoteSequence(remoteId)
128 }, {
129 handled: [a.idString, b.idString],
130 stored: [a.idString, b.idString],
131 nextSequence: 3
132 })
133 }))
134
135 it.effect("advances remote progress past a trailing known entry in a partly known batch", () =>
136 Effect.gen(function*() {
137 const journal = yield* makeJournal
138 const remoteId = EventJournal.makeRemoteIdUnsafe()
139 const a = entry(1_000)
140 const b = entry(2_000)
141 const handled: Array<string> = []
142 const effect = (batch: ReadonlyArray<{ readonly entry: EventJournal.Entry }>) =>
143 Effect.sync(() => {
144 handled.push(...batch.map(({ entry }) => entry.idString))
145 })
146 yield* journal.writeFromRemote({
147 remoteId,
148 entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry: a })],
149 effect
150 })
151 yield* journal.writeFromRemote({
152 remoteId,
153 entries: [
154 new EventJournal.RemoteEntry({ remoteSequence: 1, entry: b }),
155 new EventJournal.RemoteEntry({ remoteSequence: 2, entry: a })
156 ],
157 effect
158 })
159
160 assert.deepStrictEqual({
161 handled,
162 stored: (yield* journal.entries).map((entry) => entry.idString),
163 nextSequence: yield* journal.nextRemoteSequence(remoteId)
164 }, {
165 handled: [a.idString, b.idString],
166 stored: [a.idString, b.idString],
167 nextSequence: 3
168 })
169 }))
170
171 it.effect("entriesById preserves order, repeated ids and missing entries", () =>
172 Effect.gen(function*() {
173 const journal = yield* makeJournal
174 const entries: Array<EventJournal.Entry> = []
175 for (const primaryKey of ["first", "second"]) {
176 entries.push(
177 yield* journal.write({
178 event: "Repro",
179 primaryKey,
180 payload: new Uint8Array([entries.length + 1]),
181 effect: Effect.succeed
182 })
183 )
184 }
185 const [first, second] = entries
186 const missing = EventJournal.makeEntryIdUnsafe()
187 const found = yield* journal.entriesById([
188 second.id.slice() as EventJournal.EntryId,
189 missing,
190 first.id.slice() as EventJournal.EntryId,
191 second.id.slice() as EventJournal.EntryId
192 ])
193 assert.deepStrictEqual(found, [second, undefined, first, second])
194 assert.deepStrictEqual(yield* journal.entriesById([]), [])
195 }))
196
197 it.effect("replay invalidation observes committed entries from another connection", () =>
198 Effect.gen(function*() {
199 const fs = yield* FileSystem.FileSystem
200 const directory = yield* fs.makeTempDirectoryScoped()
201 const filename = directory + "/journal.db"
202 const writer = yield* SqliteClient.make({ filename })
203 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, writer))
204 const reader = yield* SqliteClient.make({ filename, readonly: true })
205 // A separate connection cannot see the replay transaction's uncommitted rows.
206 const readJournal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, reader))
207 const group = EventGroup.empty.add({
208 tag: "UserCreated",
209 primaryKey: (payload) => payload.id,
210 payload: Schema.Struct({ id: Schema.String })
211 })
212 const handlers = EventLog.group(group, (handlers) => handlers.handle("UserCreated", () => Effect.void)).pipe(
213 Layer.merge(EventLog.groupReactivity(group, ["users"])),
214 Layer.provide(EventLog.layerRegistry)
215 )
216 const layer = EventLog.layer(EventLog.schema(group), handlers).pipe(
217 Layer.provide(Layer.succeed(EventJournal.EventJournal, journal)),
218 Layer.provide(
219 Layer.effect(EventLog.Identity, EventLog.makeIdentity).pipe(
220 Layer.provide(EventLogEncryption.layerSubtle)
221 )
222 )
223 )
224 const reactivity = yield* Reactivity.Reactivity
225 const results = yield* reactivity.query(["users"], readJournal.entries)
226 assert.deepStrictEqual(yield* Queue.take(results), [])
227 const entry = new EventJournal.Entry({
228 id: EventJournal.makeEntryIdUnsafe(),
229 event: "UserCreated",
230 primaryKey: "user-1",
231 payload: yield* Schema.encodeUnknownEffect(group.events.UserCreated.payloadSchemaBinary)({ id: "user-1" })
232 }, { disableChecks: true })
233 const changes = yield* Queue.unbounded<EventJournal.RemoteEntry, EventLogRemote.EventLogRemoteError>()
234 yield* Queue.offer(changes, new EventJournal.RemoteEntry({ remoteSequence: 0, entry }))
235 yield* Effect.gen(function*() {
236 const registry = yield* EventLog.Registry
237 yield* registry.registerRemote(EventLogRemote.EventLogRemote.of({
238 id: EventJournal.makeRemoteIdUnsafe(),
239 changes: () => Effect.succeed(changes),
240 write: () => Effect.void,
241 whenAuthenticated: (effect) => effect
242 }))
243 assert.deepStrictEqual((yield* Queue.take(results)).map((entry) => entry.idString), [entry.idString])
244 }).pipe(Effect.provide(layer))
245 }).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer])))
246
247 it.effect("writeMany generates ids after entries another process imported into the shared database", () =>
248 Effect.gen(function*() {
249 const fs = yield* FileSystem.FileSystem
250 const filename = (yield* fs.makeTempDirectoryScoped()) + "/journal.db"
251 const sql = yield* SqliteClient.make({ filename })
252 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
253 // This process's clock lags the entry the other process imports.
254 const millis = EventJournal.entryIdMillis(EventJournal.makeEntryIdUnsafe()) + 60_000
255 const imported = yield* Effect.promise(() => importFromOtherProcess(filename, millis))
256 const [local] = yield* journal.writeMany({ entries: [localEntry("key")], effect: Effect.succeed })
257 assert.strictEqual(EventJournal.EntryIdOrder(imported, local.id), -1)
258 }).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer])))
259
260 it("keeps local writes working after an enclosing transaction rolls back an import at the maximum timestamp and a write after it", async () => {
261 const { Effect, EventJournal, Reactivity, SqlClient, SqlEventJournal, SqliteClient } = IsolatedRolledBackImport
262 const id = EventJournal.makeEntryIdUnsafe({ msecs: 2 ** 48 - 1 })
263 id[6] = 0x7f
264 id[7] = 0xff
265 const remote = new EventJournal.Entry({ id, event: "Repro", primaryKey: "key", payload: new Uint8Array() }, {
266 disableChecks: true
267 })
268 const { entries, rolledBack, written } = await Effect.runPromise(
269 Effect.gen(function*() {
270 const sql = yield* SqliteClient.make({ filename: ":memory:" })
271 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
272 // The import itself succeeds, and a local write in the same transaction
273 // reads it before it commits. The transaction around both then fails.
274 const rolledBack = yield* journal.writeFromRemote({
275 remoteId: EventJournal.makeRemoteIdUnsafe(),
276 entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry: remote })],
277 effect: () => Effect.void
278 }).pipe(
279 Effect.andThen(Effect.exit(journal.write({ ...localEntry("key"), effect: Effect.succeed }))),
280 Effect.andThen(Effect.fail("rolled back")),
281 sql.withTransaction,
282 Effect.exit
283 )
284 const written = yield* Effect.exit(journal.write({ ...localEntry("key"), effect: Effect.succeed }))
285 return { rolledBack, written, entries: (yield* journal.entries).map((entry) => entry.idString) }
286 }).pipe(Effect.scoped, Effect.provide(Reactivity.layer))
287 )
288 assert.strictEqual(rolledBack._tag, "Failure")
289 if (written._tag === "Failure") return assert.fail("the local write failed after the import was rolled back")
290 assert.deepStrictEqual(entries, [written.value.idString])
291 })
292
293 it.effect("orders a local write after an import earlier in the same transaction", () =>
294 Effect.gen(function*() {
295 const sql = yield* SqliteClient.make({ filename: ":memory:" })
296 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
297 // Ahead of this process's clock, with saturated `rand_a` bits.
298 const id = EventJournal.makeEntryIdUnsafe({
299 msecs: EventJournal.entryIdMillis(EventJournal.makeEntryIdUnsafe()) + 60_000
300 })
301 id[6] = 0x7f
302 id[7] = 0xff
303 const remote = new EventJournal.Entry({ id, event: "Repro", primaryKey: "key", payload: new Uint8Array() }, {
304 disableChecks: true
305 })
306 const local = yield* journal.writeFromRemote({
307 remoteId: EventJournal.makeRemoteIdUnsafe(),
308 entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry: remote })],
309 effect: () => Effect.void
310 }).pipe(
311 Effect.andThen(journal.write({ ...localEntry("key"), effect: Effect.succeed })),
312 sql.withTransaction
313 )
314 assert.strictEqual(EventJournal.EntryIdOrder(remote.id, local.id), -1)
315 }).pipe(Effect.provide(Reactivity.layer)))
316
317 it.effect("rolls back a failed same-client handler transaction with the journal and redelivers the batch", () =>
318 Effect.gen(function*() {
319 const sql = yield* SqliteClient.make({ filename: ":memory:" })
320 yield* sql`CREATE TABLE projection (id TEXT PRIMARY KEY)`
321 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
322 const group = EventGroup.empty.add({
323 tag: "UserCreated",
324 primaryKey: (payload) => payload.id,
325 payload: Schema.Struct({ id: Schema.String })
326 })
327 const handlers = EventLog.group(group, (handlers) =>
328 handlers.handle("UserCreated", ({ payload }) =>
329 // The primary key turns a missing rollback into a failed redelivery.
330 sql`INSERT INTO projection (id) VALUES (${payload.id})`.pipe(Effect.orDie))).pipe(
331 Layer.merge(EventLog.groupReactivity(group, ["users"])),
332 Layer.provide(EventLog.layerRegistry)
333 )
334 let attempts = 0
335 const layer = EventLog.layer(EventLog.schema(group), handlers).pipe(
336 Layer.provide([
337 Layer.succeed(EventJournal.EventJournal, journal),
338 Layer.effect(EventLog.Identity, EventLog.makeIdentity).pipe(Layer.provide(EventLogEncryption.layerSubtle)),
339 Layer.succeed(EventLog.HandlerTransaction, (effect) =>
340 sql.withTransaction(effect).pipe(
341 Effect.catchIf(SqlError.isSqlError, Effect.die),
342 // Abort the first attempt after the handler transaction completes,
343 // so only the enclosing journal transaction can roll it back.
344 Effect.tap(() => attempts++ === 0 ? Effect.die("aborted") : Effect.void)
345 ))
346 ])
347 )
348 const reactivity = yield* Reactivity.Reactivity
349 // Queries re-run after the replay releases the journal lock and transaction.
350 const results = yield* reactivity.query(
351 ["users"],
352 Effect.all({
353 projection: sql<{ id: string }>`SELECT id FROM projection ORDER BY id`.pipe(
354 Effect.map((rows) => rows.map((row) => row.id))
355 ),
356 journal: Effect.map(journal.entries, (entries) => entries.map((entry) => entry.primaryKey))
357 })
358 )
359 assert.deepStrictEqual(yield* Queue.take(results), { projection: [], journal: [] })
360 const encode = Schema.encodeUnknownEffect(group.events.UserCreated.payloadSchemaBinary)
361 const entries: Array<EventJournal.RemoteEntry> = []
362 for (const id of ["user-1", "user-2"]) {
363 entries.push(
364 new EventJournal.RemoteEntry({
365 remoteSequence: entries.length,
366 entry: new EventJournal.Entry({
367 id: EventJournal.makeEntryIdUnsafe(),
368 event: "UserCreated",
369 primaryKey: id,
370 payload: yield* encode({ id })
371 }, { disableChecks: true })
372 })
373 )
374 }
375 yield* Effect.gen(function*() {
376 const registry = yield* EventLog.Registry
377 yield* registry.registerRemote(EventLogRemote.EventLogRemote.of({
378 id: EventJournal.makeRemoteIdUnsafe(),
379 // Every subscription redelivers from the requested sequence.
380 changes: ({ startSequence }) =>
381 Effect.tap(
382 Queue.unbounded<EventJournal.RemoteEntry, EventLogRemote.EventLogRemoteError>(),
383 (queue) => Queue.offerAll(queue, entries.filter((entry) => entry.remoteSequence >= startSequence))
384 ),
385 write: () => Effect.void,
386 whenAuthenticated: (effect) => effect
387 }))
388 const afterFailure = yield* Queue.take(results)
389 yield* TestClock.adjust("200 millis")
390 const afterRedelivery = yield* Queue.take(results)
391 assert.deepStrictEqual({ afterFailure, afterRedelivery, attempts }, {
392 afterFailure: { projection: [], journal: [] },
393 afterRedelivery: { projection: ["user-1", "user-2"], journal: ["user-1", "user-2"] },
394 attempts: 2
395 })
396 }).pipe(Effect.provide(layer))
397 }).pipe(Effect.provide(Reactivity.layer)))
398
399 it.effect("replays each remote batch in one callback with same-event-and-key conflicts that have greater ids", () =>
400 Effect.gen(function*() {
401 const journal = yield* makeJournal
402 const newer = entry(2000, "Repro", "key", 0x80)
403 const otherEvent = entry(3000, "Other", "key", 0x10)
404 const otherKey = entry(3000, "Repro", "other", 0x20)
405 const older = entry(1000)
406 const sameBatch = entry(1500)
407 const sameMillis = entry(2000, "Repro", "key", 0x10)
408 const batches: Array<
409 ReadonlyArray<{ readonly entry: EventJournal.Entry; readonly conflicts: ReadonlyArray<EventJournal.Entry> }>
410 > = []
411 const writeBatch = (entries: ReadonlyArray<EventJournal.Entry>) =>
412 journal.writeFromRemote({
413 remoteId: EventJournal.makeRemoteIdUnsafe(),
414 entries: remoteEntries(entries),
415 effect: (batch) =>
416 Effect.sync(() => {
417 batches.push(batch)
418 })
419 })
420 yield* writeBatch([newer, otherEvent, otherKey])
421 yield* writeBatch([older, sameBatch, sameMillis])
422
423 assert.strictEqual(batches.length, 2)
424 // Conflicts exclude the incoming batch and other event/key pairs. A
425 // stored entry in the same millisecond with a greater id conflicts.
426 assert.deepStrictEqual(
427 batches.map((batch) => batch.map(({ conflicts, entry }) => [entry.idString, conflicts.map((c) => c.idString)])),
428 [
429 [[newer.idString, []], [otherEvent.idString, []], [otherKey.idString, []]],
430 [
431 [older.idString, [newer.idString]],
432 [sameBatch.idString, [newer.idString]],
433 [sameMillis.idString, [newer.idString]]
434 ]
435 ]
436 )
437 }))
438
439 it.effect("journal queries use indexes", () =>
440 Effect.gen(function*() {
441 const sql = yield* SqliteClient.make({ filename: ":memory:" })
442 yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
443 const plan = (query: string) =>
444 sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`).pipe(
445 Effect.map((rows) => rows.map((row) => row.detail.replace(/INDEX \S+/, "INDEX <index>")))
446 )
447 assert.deepStrictEqual({
448 conflicts: yield* plan(
449 "SELECT * FROM effect_event_journal WHERE event IN ('a', 'b') AND primary_key IN ('a', 'b') AND timestamp > 0"
450 ),
451 entries: yield* plan("SELECT * FROM effect_event_journal ORDER BY timestamp ASC"),
452 source: yield* plan("SELECT remote_id FROM effect_event_journal_sources WHERE entry_id = x'00'")
453 }, {
454 conflicts: ["SEARCH effect_event_journal USING INDEX <index> (event=? AND primary_key=? AND timestamp>?)"],
455 entries: ["SCAN effect_event_journal USING INDEX <index>"],
456 source: ["SEARCH effect_event_journal_sources USING INDEX <index> (entry_id=?)"]
457 })
458 }).pipe(Effect.provide(Reactivity.layer)))
459
460 it.effect("opens and keeps working on a database created before upload checkpoints", () =>
461 Effect.gen(function*() {
462 const sql = yield* SqliteClient.make({ filename: ":memory:" })
463 // The schema as created by the first release of SqlEventJournal.
464 yield* sql`
465 CREATE TABLE effect_event_journal (
466 id BLOB PRIMARY KEY,
467 event TEXT NOT NULL,
468 primary_key TEXT NOT NULL,
469 payload BLOB NOT NULL,
470 timestamp INTEGER NOT NULL
471 )`
472 yield* sql`
473 CREATE TABLE effect_event_remotes (
474 remote_id BLOB NOT NULL,
475 entry_id BLOB NOT NULL,
476 sequence INT NOT NULL,
477 PRIMARY KEY (remote_id, entry_id)
478 )`
479 const remoteId = EventJournal.makeRemoteIdUnsafe()
480 const local = entry(1000, "Repro", "local")
481 const imported = entry(2000, "Repro", "imported")
482 for (const { event, id, primaryKey, payload, createdAtMillis } of [local, imported]) {
483 yield* sql`INSERT INTO effect_event_journal ${
484 sql.insert({ id, event, primary_key: primaryKey, payload, timestamp: createdAtMillis })
485 }`
486 }
487 yield* sql`INSERT INTO effect_event_remotes ${
488 sql.insert({ remote_id: remoteId, entry_id: imported.id, sequence: 0 })
489 }`
490
491 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
492 const upload = (remoteId: EventJournal.RemoteId) =>
493 journal.withRemoteUncommited(remoteId, (entries) => Effect.succeed(entries.map((entry) => entry.idString)))
494 assert.deepStrictEqual({
495 remote: yield* upload(remoteId),
496 otherRemote: yield* upload(EventJournal.makeRemoteIdUnsafe())
497 }, {
498 remote: Option.some([local.idString]),
499 otherRemote: Option.some([local.idString, imported.idString])
500 })
501 }).pipe(Effect.provide(Reactivity.layer)))
502
503 it.effect("replaces per-entry remote rows with entry sources in one restartable migration", () =>
504 Effect.gen(function*() {
505 const fs = yield* FileSystem.FileSystem
506 const filename = (yield* fs.makeTempDirectoryScoped()) + "/journal.db"
507 const sql = yield* SqliteClient.make({ filename })
508 // A journal written by an unreleased build with upload checkpoints and cursors.
509 yield* sql`
510 CREATE TABLE effect_event_journal (
511 id BLOB PRIMARY KEY,
512 event TEXT NOT NULL,
513 primary_key TEXT NOT NULL,
514 payload BLOB NOT NULL,
515 timestamp INTEGER NOT NULL
516 )`
517 yield* sql`
518 CREATE TABLE effect_event_remotes (
519 remote_id BLOB NOT NULL,
520 entry_id BLOB NOT NULL,
521 sequence INT NOT NULL,
522 PRIMARY KEY (remote_id, entry_id)
523 )`
524 yield* sql`CREATE TABLE effect_event_journal_sequence (sequence INTEGER PRIMARY KEY, entry_id BLOB NOT NULL UNIQUE)`
525 yield* sql`CREATE TABLE effect_event_remotes_checkpoints (remote_id BLOB PRIMARY KEY, sequence INTEGER NOT NULL)`
526 yield* sql`CREATE TABLE effect_event_remotes_cursors (remote_id BLOB PRIMARY KEY, next_sequence INT NOT NULL)`
527 yield* sql`CREATE INDEX effect_event_journal_event_idx ON effect_event_journal (event, primary_key, timestamp)`
528 yield* sql`CREATE INDEX effect_event_journal_timestamp_idx ON effect_event_journal (timestamp)`
529 yield* sql`CREATE INDEX effect_event_remotes_sequence_idx ON effect_event_remotes (remote_id, sequence)`
530
531 const remoteA = EventJournal.makeRemoteIdUnsafe()
532 const remoteB = EventJournal.makeRemoteIdUnsafe()
533 const uploaded = entry(1000, "Repro", "uploaded")
534 const fromA = entry(2000, "Repro", "fromA")
535 // Acknowledged by A, but the upload checkpoint was not saved before a restart.
536 const acked = entry(3000, "Repro", "acked")
537 const fromB = entry(4000, "Repro", "fromB")
538 // Written offline; no remote has it.
539 const pending = entry(5000, "Repro", "pending")
540 for (const { createdAtMillis, event, id, payload, primaryKey } of [uploaded, fromA, acked, fromB, pending]) {
541 yield* sql`INSERT INTO effect_event_journal ${
542 sql.insert({ id, event, primary_key: primaryKey, payload, timestamp: createdAtMillis })
543 }`
544 yield* sql`INSERT INTO effect_event_journal_sequence ${sql.insert({ entry_id: id })}`
545 }
546 yield* sql`INSERT INTO effect_event_remotes ${
547 sql.insert([
548 { remote_id: remoteA, entry_id: fromA.id, sequence: 0 },
549 { remote_id: remoteA, entry_id: acked.id, sequence: 1 },
550 { remote_id: remoteB, entry_id: fromB.id, sequence: 3 }
551 ])
552 }`
553 yield* sql`INSERT INTO effect_event_remotes_checkpoints ${sql.insert({ remote_id: remoteA, sequence: 1 })}`
554 // B has no cursor row, so its download progress is only in the remotes table.
555 yield* sql`INSERT INTO effect_event_remotes_cursors ${sql.insert({ remote_id: remoteA, next_sequence: 2 })}`
556
557 const open = (client: SqlClient.SqlClient) =>
558 SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, client))
559 const failDrop: Statement.Transformer = (statement) =>
560 statement.compile()[0].includes("DROP TABLE")
561 ? Effect.die("interrupted migration")
562 : Effect.succeed(statement)
563 const interrupted = yield* Effect.exit(
564 open(sql).pipe(Effect.provideService(Statement.CurrentTransformer, failDrop))
565 )
566
567 const journal = yield* open(sql)
568 const readonly = yield* open(yield* SqliteClient.make({ filename, readonly: true }))
569 const state = (journal: EventJournal.EventJournal["Service"]) =>
570 Effect.all({
571 entries: Effect.map(journal.entries, (entries) => entries.map((entry) => entry.idString)),
572 nextA: journal.nextRemoteSequence(remoteA),
573 nextB: journal.nextRemoteSequence(remoteB)
574 })
575 const migrated = yield* state(journal)
576 const reopened = yield* state(readonly)
577 const tables = yield* sql<{ name: string }>`SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name`
578
579 const fresh = entry(6000, "Repro", "fresh")
580 yield* journal.writeFromRemote({
581 remoteId: remoteB,
582 entries: [new EventJournal.RemoteEntry({ remoteSequence: 4, entry: fresh })],
583 effect: () => Effect.void
584 })
585 const upload = (remoteId: EventJournal.RemoteId) =>
586 journal.withRemoteUncommited(remoteId, (entries) => Effect.succeed(entries.map((entry) => entry.idString)))
587 const ids = (...entries: ReadonlyArray<EventJournal.Entry>) => entries.map((entry) => entry.idString)
588
589 const expected = {
590 entries: ids(uploaded, fromA, acked, fromB, pending),
591 nextA: 2,
592 nextB: 4
593 }
594 assert.deepStrictEqual({
595 interrupted: Exit.isFailure(interrupted),
596 migrated,
597 reopened,
598 tables: tables.map((table) => table.name),
599 nextB: yield* journal.nextRemoteSequence(remoteB),
600 uploadA: yield* upload(remoteA),
601 uploadB: yield* upload(remoteB),
602 uploadLate: yield* upload(EventJournal.makeRemoteIdUnsafe())
603 }, {
604 interrupted: true,
605 migrated: expected,
606 reopened: expected,
607 tables: [
608 "effect_event_journal",
609 "effect_event_journal_sequence",
610 "effect_event_journal_sources",
611 "effect_event_remotes_checkpoints",
612 "effect_event_remotes_cursors"
613 ],
614 nextB: 5,
615 // Never back to the source, and nothing at or below a checkpoint.
616 uploadA: Option.some(ids(fromB, pending, fresh)),
617 uploadB: Option.some(ids(uploaded, fromA, acked, pending)),
618 uploadLate: Option.some(ids(uploaded, fromA, acked, fromB, pending, fresh))
619 })
620 }).pipe(Effect.provide([NodeFileSystem.layer, Reactivity.layer])))
621
622 for (const [name, make] of [["SQL", makeJournal], ["memory", EventJournal.makeMemory]] as const) {
623 describe(name, () => {
624 it.effect("withRemoteUncommited uploads each entry once, retries failures and forwards older imports", () =>
625 Effect.gen(function*() {
626 const journal = yield* make
627 const remoteId = EventJournal.makeRemoteIdUnsafe()
628 const upload = journal.withRemoteUncommited(
629 remoteId,
630 (entries) => Effect.succeed(entries.map((entry) => entry.idString))
631 )
632 const a = yield* journal.write({ ...localEntry("a"), effect: Effect.succeed })
633 const failed = yield* Effect.exit(journal.withRemoteUncommited(remoteId, () => Effect.fail("upload failed")))
634 const retried = yield* upload
635 // No echo from the remote: a successful upload alone must advance it.
636 const b = yield* journal.write({ ...localEntry("b"), effect: Effect.succeed })
637 const next = yield* upload
638 const older = entry(a.createdAtMillis - 1)
639 yield* journal.writeFromRemote({
640 remoteId: EventJournal.makeRemoteIdUnsafe(),
641 entries: remoteEntries([older]),
642 effect: () => Effect.void
643 })
644 const forwarded = yield* upload
645 const idle = yield* upload
646 assert.deepStrictEqual({ failed: Exit.isFailure(failed), retried, next, forwarded, idle }, {
647 failed: true,
648 retried: Option.some([a.idString]),
649 next: Option.some([b.idString]),
650 forwarded: Option.some([older.idString]),
651 idle: Option.none()
652 })
653 }))
654
655 it.effect("writeMany commits and publishes a batch in order only when its callback succeeds", () =>
656 Effect.gen(function*() {
657 const journal = yield* make
658 const changes = yield* journal.changes
659 const entries = [localEntry("a"), localEntry("b"), localEntry("c")] as const
660 const failed = yield* Effect.exit(
661 journal.writeMany({ entries, effect: () => Effect.fail("callback failed") })
662 )
663 const afterFailure = {
664 failed: Exit.isFailure(failed),
665 entries: (yield* journal.entries).length,
666 published: yield* PubSub.remaining(changes)
667 }
668 const written = yield* journal.writeMany({ entries, effect: Effect.succeed })
669 const ids = written.map((entry) => entry.idString)
670 assert.deepStrictEqual({
671 afterFailure,
672 keys: written.map((entry) => entry.primaryKey),
673 ascending: written.every((entry, i) =>
674 i === 0 || EventJournal.EntryIdOrder(written[i - 1].id, entry.id) < 0
675 ),
676 committed: (yield* journal.entries).map((entry) => entry.idString),
677 uncommitted: yield* journal.withRemoteUncommited(
678 EventJournal.makeRemoteIdUnsafe(),
679 (entries) => Effect.succeed(entries.map((entry) => entry.idString))
680 ),
681 published: (yield* PubSub.takeAll(changes)).map((entry) => entry.idString)
682 }, {
683 afterFailure: { failed: true, entries: 0, published: 0 },
684 keys: ["a", "b", "c"],
685 ascending: true,
686 committed: ids,
687 uncommitted: Option.some(ids),
688 published: ids
689 })
690 }))
691
692 it.effect("failed remote callbacks persist nothing and allow redelivery", () =>
693 Effect.gen(function*() {
694 const journal = yield* make
695 const remoteId = EventJournal.makeRemoteIdUnsafe()
696 const stored = entry(500)
697 yield* journal.writeFromRemote({
698 remoteId: EventJournal.makeRemoteIdUnsafe(),
699 entries: remoteEntries([stored]),
700 effect: () => Effect.void
701 })
702 const added = entry(1000)
703 // A new entry followed by an echo of an entry the journal already has.
704 const entries = remoteEntries([added, stored])
705 const exit = yield* Effect.exit(journal.writeFromRemote({
706 remoteId,
707 entries,
708 effect: () =>
709 Effect.fail(
710 new EventJournal.EventJournalError({
711 method: "writeFromRemote",
712 cause: "callback failed"
713 })
714 )
715 }))
716 assert.isTrue(Exit.isFailure(exit))
717 const afterFailure = {
718 entries: (yield* journal.entries).map((entry) => entry.idString),
719 sequence: yield* journal.nextRemoteSequence(remoteId)
720 }
721 const retried: Array<string> = []
722 yield* journal.writeFromRemote({
723 remoteId,
724 entries,
725 effect: (batch) =>
726 Effect.sync(() => {
727 for (const { entry } of batch) retried.push(entry.idString)
728 })
729 })
730 assert.deepStrictEqual({ afterFailure, retried }, {
731 afterFailure: { entries: [stored.idString], sequence: 0 },
732 retried: [added.idString]
733 })
734 assert.deepStrictEqual((yield* journal.entries).map((entry) => entry.idString), [
735 stored.idString,
736 added.idString
737 ])
738 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 2)
739 }))
740 })
741 }
742
743 it.effect("preserves write callback error identity", () =>
744 Effect.gen(function*() {
745 const journal = yield* makeJournal
746 const error = new Error("callback failed")
747 const actual = yield* Effect.flip(journal.write({
748 event: "Repro",
749 primaryKey: "key",
750 payload: new Uint8Array([1]),
751 effect: () => Effect.fail(error)
752 }))
753 assert.strictEqual(actual, error)
754 }))
755
756 it.effect("preserves remote callback error identity", () =>
757 Effect.gen(function*() {
758 const journal = yield* makeJournal
759 yield* journal.write({
760 event: "Repro",
761 primaryKey: "key",
762 payload: new Uint8Array([1]),
763 effect: () => Effect.void
764 })
765 const error = new Error("callback failed")
766 const actual = yield* Effect.flip(
767 journal.withRemoteUncommited(EventJournal.makeRemoteIdUnsafe(), () => Effect.fail(error))
768 )
769 assert.strictEqual(actual, error)
770 }))
771
772 it.effect("commits only after the write callback succeeds", () =>
773 Effect.gen(function*() {
774 const sql = yield* SqliteClient.make({ filename: ":memory:" })
775 const journal = yield* SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
776 yield* Effect.exit(journal.write({
777 event: "Repro",
778 primaryKey: "key",
779 payload: new Uint8Array([1]),
780 effect: () => Effect.fail("callback failed")
781 }))
782 assert.deepStrictEqual(yield* journal.entries, [])
783 }).pipe(Effect.provide(Reactivity.layer)))
784
785 it.effect("writeMany commits a batch over the statement parameter limit", () =>
786 Effect.gen(function*() {
787 const journal = yield* makeJournal
788 const written = yield* journal.writeMany({
789 entries: Arr.makeBy(101, (i) => localEntry(String(i))),
790 effect: Effect.succeed
791 })
792 assert.deepStrictEqual(
793 (yield* journal.entries).map((entry) => entry.idString),
794 written.map((entry) => entry.idString)
795 )
796 }).pipe(limitParameters))
797
798 it.effect("writeFromRemote reports each conflict once for a batch over the statement parameter limit", () =>
799 Effect.gen(function*() {
800 const journal = yield* makeJournal
801 const remoteId = EventJournal.makeRemoteIdUnsafe()
802 // Two events sharing 101 keys: as 101 is prime, conflict lookups split
803 // into chunks that mix both events, so a stored row matches several.
804 const incoming = ["A", "B"].flatMap((event, e) =>
805 Arr.makeBy(101, (i) => entry(1_000 + e * 101 + i, event, String(i)))
806 )
807 const newer = incoming.map((stored) => entry(stored.createdAtMillis + 1_000, stored.event, stored.primaryKey))
808 yield* journal.writeFromRemote({
809 remoteId: EventJournal.makeRemoteIdUnsafe(),
810 entries: remoteEntries(newer),
811 effect: () => Effect.void
812 })
813 const batches: Array<ReadonlyArray<readonly [string, ReadonlyArray<string>]>> = []
814 yield* journal.writeFromRemote({
815 remoteId,
816 entries: remoteEntries(incoming),
817 effect: (batch) =>
818 Effect.sync(() => {
819 batches.push(
820 batch.map(({ conflicts, entry }) => [entry.idString, conflicts.map((c) => c.idString)] as const)
821 )
822 })
823 })
824 assert.deepStrictEqual({ batches, nextSequence: yield* journal.nextRemoteSequence(remoteId) }, {
825 batches: [incoming.map((entry, i) => [entry.idString, [newer[i].idString]] as const)],
826 nextSequence: 202
827 })
828 }).pipe(limitParameters))
829
830 it.effect("writes and reads entries", () =>
831 Effect.gen(function*() {
832 const journal = yield* makeJournal
833 let createdAt = 0
834 yield* journal.write({
835 event: "UserCreated",
836 primaryKey: "user-1",
837 payload: new Uint8Array([1]),
838 effect: (entry) =>
839 Effect.sync(() => {
840 createdAt = entry.createdAtMillis
841 })
842 })
843 const entries = yield* journal.entries
844 assert.strictEqual(entries.length, 1)
845 assert.strictEqual(entries[0].event, "UserCreated")
846 assert.strictEqual(entries[0].createdAtMillis, createdAt)
847 }))
848
849 it.effect("writes remote entries and sequences", () =>
850 Effect.gen(function*() {
851 const journal = yield* makeJournal
852 const remoteId = EventJournal.makeRemoteIdUnsafe()
853 const initial = yield* journal.nextRemoteSequence(remoteId)
854 assert.strictEqual(initial, 0)
855
856 const entryA = new EventJournal.Entry({
857 id: EventJournal.makeEntryIdUnsafe(),
858 event: "UserCreated",
859 primaryKey: "user-2",
860 payload: new Uint8Array([2])
861 }, { disableChecks: true })
862 const entryB = new EventJournal.Entry({
863 id: EventJournal.makeEntryIdUnsafe(),
864 event: "UserCreated",
865 primaryKey: "user-3",
866 payload: new Uint8Array([3])
867 }, { disableChecks: true })
868 const remoteEntries = [
869 new EventJournal.RemoteEntry({ remoteSequence: 0, entry: entryA }),
870 new EventJournal.RemoteEntry({ remoteSequence: 1, entry: entryB })
871 ]
872 const seenConflicts: Array<ReadonlyArray<EventJournal.Entry>> = []
873 yield* journal.writeFromRemote({
874 remoteId,
875 entries: remoteEntries,
876 effect: (batch) =>
877 Effect.sync(() => {
878 for (const { conflicts } of batch) seenConflicts.push(conflicts)
879 })
880 })
881 const next = yield* journal.nextRemoteSequence(remoteId)
882 assert.strictEqual(next, 2)
883
884 let emptyCalls = 0
885 const emptyResult = yield* journal.withRemoteUncommited(remoteId, () =>
886 Effect.sync(() => {
887 emptyCalls++
888 return "called"
889 }))
890 assert.isTrue(Option.isNone(emptyResult))
891 assert.strictEqual(emptyCalls, 0)
892
893 yield* journal.write({
894 event: "LocalCreated",
895 primaryKey: "local-1",
896 payload: new Uint8Array([4]),
897 effect: () => Effect.void
898 })
899 let uncommittedCalls = 0
900 const uncommitted = yield* journal.withRemoteUncommited(remoteId, (entries) =>
901 Effect.sync(() => {
902 uncommittedCalls++
903 return entries
904 }))
905 if (Option.isNone(uncommitted)) assert.fail("Expected the callback result")
906 assert.strictEqual(uncommitted.value.length, 1)
907 assert.strictEqual(uncommitted.value[0].event, "LocalCreated")
908 assert.strictEqual(uncommittedCalls, 1)
909 assert.deepStrictEqual(seenConflicts, [[], []])
910 }))
911
912 it.effect("advances the remote sequence only across acknowledgements that follow it", () =>
913 Effect.gen(function*() {
914 const sql = yield* SqliteClient.make({ filename: ":memory:" })
915 const open = SqlEventJournal.make().pipe(Effect.provideService(SqlClient.SqlClient, sql))
916 const remoteId = EventJournal.makeRemoteIdUnsafe()
917 const ack = (sequence: number, entryId = EventJournal.makeEntryIdUnsafe()) => ({ entryId, sequence })
918 // A database from before the cursors table: every remote row was downloaded.
919 yield* sql`CREATE TABLE effect_event_remotes (
920 remote_id BLOB NOT NULL,
921 entry_id BLOB NOT NULL,
922 sequence INT NOT NULL,
923 PRIMARY KEY (remote_id, entry_id)
924 )`
925 yield* sql`INSERT INTO effect_event_remotes ${
926 sql.insert({ remote_id: remoteId, entry_id: EventJournal.makeEntryIdUnsafe(), sequence: 4 })
927 }`
928 const journal = yield* open
929 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 5)
930
931 // Sequence 5, written by another replica, has not been downloaded.
932 const local = yield* journal.write({ ...localEntry("local-1"), effect: Effect.succeed })
933 yield* journal.writeRemoteAcks({ remoteId, acks: [ack(6, local.id)] })
934 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 5)
935 yield* journal.writeFromRemote({
936 remoteId,
937 entries: [new EventJournal.RemoteEntry({ remoteSequence: 5, entry: entry(1_000) })],
938 effect: () => Effect.void
939 })
940 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 6)
941 yield* journal.writeRemoteAcks({ remoteId, acks: [ack(7), ack(6, local.id), ack(3)] })
942 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 8)
943 assert.strictEqual(yield* (yield* open).nextRemoteSequence(remoteId), 8)
944
945 // A remote that numbers from 1 can be acknowledged before any download.
946 const otherRemoteId = EventJournal.makeRemoteIdUnsafe()
947 yield* journal.writeRemoteAcks({ remoteId: otherRemoteId, acks: [ack(2), ack(1)], firstSequence: 1 })
948 assert.strictEqual(yield* journal.nextRemoteSequence(otherRemoteId), 3)
949 }).pipe(Effect.provide(Reactivity.layer)))
950 })