The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { PgClient } from "@effect/sql-pg" |
| 2 | import { assert, it } from "@effect/vitest" |
| 3 | import { Deferred, Effect, Fiber, Layer, Option, Redacted } from "effect" |
| 4 | import * as SqlEventJournalSharedTest from "effect-test/eventlog/SqlEventJournalSharedTest" |
| 5 | import * as Arr from "effect/Array" |
| 6 | import * as EventJournal from "effect/eventlog/EventJournal" |
| 7 | import * as EventLogMessage from "effect/eventlog/EventLogMessage" |
| 8 | import * as SqlEventJournal from "effect/eventlog/SqlEventJournal" |
| 9 | import { Reactivity } from "effect/reactivity" |
| 10 | import * as SqlClient from "effect/sql/SqlClient" |
| 11 | import * as Crypto from "node:crypto" |
| 12 | import { PgContainer, realSleep } from "./utils.ts" |
| 13 | |
| 14 | const localEntry = (primaryKey: string) => ({ event: "Repro", primaryKey, payload: new Uint8Array() }) |
| 15 | |
| 16 | const remoteEntry = (primaryKey: string, msecs?: number) => |
| 17 | new EventJournal.RemoteEntry({ |
| 18 | remoteSequence: 0, |
| 19 | entry: new EventJournal.Entry({ |
| 20 | id: EventJournal.makeEntryIdUnsafe(msecs === undefined ? {} : { msecs }), |
| 21 | event: "Repro", |
| 22 | primaryKey, |
| 23 | payload: new Uint8Array() |
| 24 | }, { disableChecks: true }) |
| 25 | }) |
| 26 | |
| 27 | const backendPid = Effect.gen(function*() { |
| 28 | const sql = yield* SqlClient.SqlClient |
| 29 | const [{ pid }] = yield* sql<{ pid: number }>`SELECT pg_backend_pid() AS pid` |
| 30 | return pid |
| 31 | }) |
| 32 | |
| 33 | // Fails unless, within ten seconds, `count` backends are blocked by `holder` |
| 34 | // directly or behind one that is, or `fiber` succeeds. |
| 35 | const waitUntilBlockedBy = Effect.fnUntraced(function*<A, E>(holder: number, count: number, fiber?: Fiber.Fiber<A, E>) { |
| 36 | const sql = yield* SqlClient.SqlClient |
| 37 | const deadline = Date.now() + 10_000 |
| 38 | while (Date.now() < deadline) { |
| 39 | const exit = fiber?.pollUnsafe() |
| 40 | if (exit !== undefined) return yield* exit |
| 41 | const [{ blocked }] = yield* sql<{ blocked: number }>` |
| 42 | SELECT count(*)::int4 AS blocked FROM pg_stat_activity waiter |
| 43 | WHERE ${holder} = ANY(pg_blocking_pids(waiter.pid)) OR EXISTS ( |
| 44 | SELECT 1 FROM pg_stat_activity other |
| 45 | WHERE other.pid = ANY(pg_blocking_pids(waiter.pid)) AND ${holder} = ANY(pg_blocking_pids(other.pid)) |
| 46 | ) |
| 47 | ` |
| 48 | if (blocked >= count) return |
| 49 | yield* realSleep |
| 50 | } |
| 51 | return yield* Effect.die(new Error(`Expected ${count} backends blocked by ${holder} within 10 seconds`)) |
| 52 | }) |
| 53 | |
| 54 | // Exposes the container so tests can open a read-only client. |
| 55 | const TestLayer = Layer.merge(PgContainer.layerClient, PgContainer.layer) |
| 56 | |
| 57 | it.layer(TestLayer, { timeout: "30 seconds" })("SqlEventJournal", (it) => { |
| 58 | it.effect("round-trips a local entry", () => |
| 59 | Effect.gen(function*() { |
| 60 | const journal = yield* SqlEventJournal.make({ |
| 61 | entryTable: "local_entries", |
| 62 | remotesTable: "local_remotes" |
| 63 | }) |
| 64 | const written = yield* journal.write({ |
| 65 | event: "UserCreated", |
| 66 | primaryKey: "user-1", |
| 67 | payload: new Uint8Array([1]), |
| 68 | effect: (entry) => Effect.succeed(entry) |
| 69 | }) |
| 70 | const entries = yield* journal.entries |
| 71 | assert.strictEqual(entries.length, 1) |
| 72 | assert.strictEqual(entries[0].idString, written.idString) |
| 73 | assert.strictEqual(entries[0].event, "UserCreated") |
| 74 | assert.strictEqual(entries[0].primaryKey, "user-1") |
| 75 | assert.deepStrictEqual(entries[0].payload, new Uint8Array([1])) |
| 76 | assert.strictEqual(entries[0].createdAtMillis, written.createdAtMillis) |
| 77 | })) |
| 78 | |
| 79 | it.effect("round-trips a remote entry and advances its sequence", () => |
| 80 | Effect.gen(function*() { |
| 81 | const journal = yield* SqlEventJournal.make({ |
| 82 | entryTable: "remote_entries", |
| 83 | remotesTable: "remote_remotes" |
| 84 | }) |
| 85 | const remoteId = EventJournal.makeRemoteIdUnsafe() |
| 86 | const remoteEntry = new EventJournal.Entry({ |
| 87 | id: EventJournal.makeEntryIdUnsafe({ msecs: 1_700_000_000_000 }), |
| 88 | event: "UserCreated", |
| 89 | primaryKey: "user-2", |
| 90 | payload: new Uint8Array([2]) |
| 91 | }, { disableChecks: true }) |
| 92 | assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 0) |
| 93 | yield* journal.writeFromRemote({ |
| 94 | remoteId, |
| 95 | entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry: remoteEntry })], |
| 96 | effect: () => Effect.void |
| 97 | }) |
| 98 | |
| 99 | const entries = yield* journal.entries |
| 100 | assert.strictEqual(entries.length, 1) |
| 101 | assert.strictEqual(entries[0].idString, remoteEntry.idString) |
| 102 | assert.strictEqual(entries[0].event, "UserCreated") |
| 103 | assert.strictEqual(entries[0].primaryKey, "user-2") |
| 104 | assert.deepStrictEqual(entries[0].payload, new Uint8Array([2])) |
| 105 | assert.strictEqual(entries[0].createdAtMillis, 1_700_000_000_000) |
| 106 | assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 1) |
| 107 | })) |
| 108 | |
| 109 | it.effect("reopens a migrated journal read-only", () => |
| 110 | Effect.gen(function*() { |
| 111 | const container = yield* PgContainer |
| 112 | const readonly = yield* PgClient.make({ |
| 113 | url: Redacted.make(container.getConnectionUri()), |
| 114 | startupParameters: { default_transaction_read_only: "on" } |
| 115 | }) |
| 116 | const options = { entryTable: "readonly_entries", remotesTable: "readonly_remotes" } |
| 117 | const journal = yield* SqlEventJournal.make(options) |
| 118 | const written = yield* journal.write({ ...localEntry("a"), effect: Effect.succeed }) |
| 119 | const reopened = yield* SqlEventJournal.make(options).pipe(Effect.provideService(SqlClient.SqlClient, readonly)) |
| 120 | assert.deepStrictEqual((yield* reopened.entries).map((entry) => entry.idString), [written.idString]) |
| 121 | }).pipe(Effect.provide(Reactivity.layer))) |
| 122 | |
| 123 | it.effect("writes and finds conflicts for primary keys beyond the index tuple limit", () => |
| 124 | Effect.gen(function*() { |
| 125 | const journal = yield* SqlEventJournal.make({ entryTable: "long_key_entries", remotesTable: "long_key_remotes" }) |
| 126 | // Random text barely compresses, so the key stays above the ~2.7 KB B-tree tuple limit. |
| 127 | const primaryKey = Crypto.randomBytes(6000).toString("base64") |
| 128 | const written = yield* journal.write({ ...localEntry(primaryKey), effect: Effect.succeed }) |
| 129 | const conflicts: Array<string> = [] |
| 130 | yield* journal.writeFromRemote({ |
| 131 | remoteId: EventJournal.makeRemoteIdUnsafe(), |
| 132 | entries: [remoteEntry(primaryKey, written.createdAtMillis - 1)], |
| 133 | effect: (batch) => Effect.sync(() => conflicts.push(...batch[0].conflicts.map((entry) => entry.idString))) |
| 134 | }) |
| 135 | assert.deepStrictEqual(conflicts, [written.idString]) |
| 136 | })) |
| 137 | |
| 138 | it.effect("concurrent opens migrate a legacy database once", () => |
| 139 | Effect.gen(function*() { |
| 140 | const sql = yield* SqlClient.SqlClient |
| 141 | const options = { entryTable: "concurrent_open_entries", remotesTable: "concurrent_open_remotes" } |
| 142 | // The entry table as created by the first release of SqlEventJournal. |
| 143 | yield* sql` |
| 144 | CREATE TABLE concurrent_open_entries ( |
| 145 | id BYTEA PRIMARY KEY, |
| 146 | event TEXT NOT NULL, |
| 147 | primary_key TEXT NOT NULL, |
| 148 | payload BYTEA NOT NULL, |
| 149 | timestamp BIGINT NOT NULL |
| 150 | )` |
| 151 | const remoteA = EventJournal.makeRemoteIdUnsafe() |
| 152 | const remoteB = EventJournal.makeRemoteIdUnsafe() |
| 153 | const entries = Arr.make(remoteEntry("pending").entry, remoteEntry("fromA").entry, remoteEntry("fromB").entry) |
| 154 | yield* sql`INSERT INTO concurrent_open_entries ${ |
| 155 | sql.insert(Arr.map(entries, (entry) => ({ |
| 156 | id: entry.id, |
| 157 | event: entry.event, |
| 158 | primary_key: entry.primaryKey, |
| 159 | payload: entry.payload, |
| 160 | timestamp: entry.createdAtMillis |
| 161 | }))) |
| 162 | }` |
| 163 | yield* sql` |
| 164 | CREATE TABLE concurrent_open_remotes ( |
| 165 | remote_id BYTEA NOT NULL, |
| 166 | entry_id BYTEA NOT NULL, |
| 167 | sequence INT NOT NULL, |
| 168 | PRIMARY KEY (remote_id, entry_id) |
| 169 | )` |
| 170 | yield* sql`INSERT INTO concurrent_open_remotes ${ |
| 171 | sql.insert([ |
| 172 | { remote_id: remoteA, entry_id: entries[1].id, sequence: 1 }, |
| 173 | { remote_id: remoteB, entry_id: entries[2].id, sequence: 7 } |
| 174 | ]) |
| 175 | }` |
| 176 | yield* sql` |
| 177 | CREATE TABLE concurrent_open_remotes_cursors ( |
| 178 | remote_id BYTEA PRIMARY KEY, |
| 179 | next_sequence INT NOT NULL |
| 180 | )` |
| 181 | yield* sql`INSERT INTO concurrent_open_remotes_cursors ${sql.insert({ remote_id: remoteA, next_sequence: 9 })}` |
| 182 | const warmed = yield* Deferred.make<void>() |
| 183 | const startWaiting = yield* Deferred.make<void>() |
| 184 | const migrated = yield* Deferred.make<number>() |
| 185 | const release = yield* Deferred.make<void>() |
| 186 | const waiting = yield* Effect.gen(function*() { |
| 187 | // Pin this backend and cache both old names before the other opener changes them. |
| 188 | const [before] = yield* sql<{ legacy: string | null; sources: string | null }>` |
| 189 | SELECT to_regclass('concurrent_open_remotes')::text AS legacy, |
| 190 | to_regclass('concurrent_open_entries_sources')::text AS sources |
| 191 | ` |
| 192 | assert.strictEqual(before.legacy, "concurrent_open_remotes") |
| 193 | assert.isNull(before.sources) |
| 194 | yield* Deferred.succeed(warmed, undefined) |
| 195 | yield* Deferred.await(startWaiting) |
| 196 | return yield* SqlEventJournal.make(options) |
| 197 | }).pipe(sql.withTransaction, Effect.forkChild) |
| 198 | yield* Deferred.await(warmed) |
| 199 | const first = yield* SqlEventJournal.make(options).pipe( |
| 200 | Effect.tap(() => backendPid.pipe(Effect.flatMap((pid) => Deferred.succeed(migrated, pid)))), |
| 201 | Effect.tap(Deferred.await(release)), |
| 202 | sql.withTransaction, |
| 203 | Effect.forkChild |
| 204 | ) |
| 205 | const holder = yield* Deferred.await(migrated) |
| 206 | yield* Deferred.succeed(startWaiting, undefined) |
| 207 | yield* waitUntilBlockedBy(holder, 1, waiting) |
| 208 | yield* Deferred.succeed(release, undefined) |
| 209 | const journals = [yield* Fiber.join(first), yield* Fiber.join(waiting)] |
| 210 | for (const journal of journals) { |
| 211 | assert.strictEqual(yield* journal.nextRemoteSequence(remoteA), 9) |
| 212 | assert.strictEqual(yield* journal.nextRemoteSequence(remoteB), 8) |
| 213 | assert.deepStrictEqual( |
| 214 | (yield* journal.entries).map((entry) => entry.idString).sort(), |
| 215 | entries.map((entry) => entry.idString).sort() |
| 216 | ) |
| 217 | } |
| 218 | const upload = (remoteId: EventJournal.RemoteId) => |
| 219 | journals[0].withRemoteUncommited(remoteId, (entries) => |
| 220 | Effect.succeed(entries.map((entry) => entry.primaryKey).sort())) |
| 221 | assert.deepStrictEqual(yield* upload(remoteA), Option.some(["fromB", "pending"])) |
| 222 | assert.deepStrictEqual(yield* upload(remoteB), Option.some(["fromA", "pending"])) |
| 223 | const [{ legacy }] = yield* sql<{ legacy: string | null }>` |
| 224 | SELECT to_regclass('concurrent_open_remotes')::text AS legacy |
| 225 | ` |
| 226 | assert.isNull(legacy) |
| 227 | })) |
| 228 | |
| 229 | it.effect("withRemoteUncommited uploads entries whose commits finish out of order", () => |
| 230 | Effect.gen(function*() { |
| 231 | const sql = yield* SqlClient.SqlClient |
| 232 | const journal = yield* SqlEventJournal.make({ entryTable: "ordering_entries", remotesTable: "ordering_remotes" }) |
| 233 | const remoteId = EventJournal.makeRemoteIdUnsafe() |
| 234 | const upload = journal.withRemoteUncommited(remoteId, (entries) => Effect.succeed(entries.map((e) => e.idString))) |
| 235 | const written = yield* Deferred.make<number>() |
| 236 | const release = yield* Deferred.make<void>() |
| 237 | // The first write stays uncommitted until released. |
| 238 | const first = yield* journal.write({ ...localEntry("first"), effect: Effect.succeed }).pipe( |
| 239 | Effect.tap(Effect.flatMap(backendPid, (pid) => Deferred.succeed(written, pid))), |
| 240 | Effect.tap(Deferred.await(release)), |
| 241 | sql.withTransaction, |
| 242 | Effect.forkChild |
| 243 | ) |
| 244 | const holder = yield* Deferred.await(written) |
| 245 | const second = yield* Effect.forkChild(journal.write({ ...localEntry("second"), effect: Effect.succeed })) |
| 246 | yield* waitUntilBlockedBy(holder, 1, second) |
| 247 | const beforeCommit = yield* upload |
| 248 | yield* Deferred.succeed(release, undefined) |
| 249 | const ids = [(yield* Fiber.join(first)).idString, (yield* Fiber.join(second)).idString] |
| 250 | const afterCommit = yield* upload |
| 251 | const uploaded = [...Option.getOrElse(beforeCommit, () => []), ...Option.getOrElse(afterCommit, () => [])] |
| 252 | assert.deepStrictEqual(uploaded.sort(), ids.sort()) |
| 253 | })) |
| 254 | |
| 255 | it.effect("an outer transaction can import an entry that another remote is importing", () => |
| 256 | Effect.gen(function*() { |
| 257 | const sql = yield* SqlClient.SqlClient |
| 258 | const journal = yield* SqlEventJournal.make({ |
| 259 | entryTable: "overlapping_entries", |
| 260 | remotesTable: "overlapping_remotes" |
| 261 | }) |
| 262 | const b = remoteEntry("b") |
| 263 | const importEntry = (remoteId: EventJournal.RemoteId, entry: EventJournal.RemoteEntry) => |
| 264 | journal.writeFromRemote({ remoteId, entries: [entry], effect: () => Effect.void }) |
| 265 | const imported = yield* Deferred.make<number>() |
| 266 | const proceed = yield* Deferred.make<void>() |
| 267 | const remoteId = EventJournal.makeRemoteIdUnsafe() |
| 268 | const outer = yield* importEntry(remoteId, remoteEntry("a")).pipe( |
| 269 | Effect.andThen(backendPid), |
| 270 | Effect.flatMap((pid) => Deferred.succeed(imported, pid)), |
| 271 | Effect.andThen(Deferred.await(proceed)), |
| 272 | Effect.andThen(importEntry(remoteId, b)), |
| 273 | sql.withTransaction, |
| 274 | Effect.forkChild |
| 275 | ) |
| 276 | const holder = yield* Deferred.await(imported) |
| 277 | const other = yield* Effect.forkChild(importEntry(EventJournal.makeRemoteIdUnsafe(), b)) |
| 278 | yield* waitUntilBlockedBy(holder, 1, other) |
| 279 | yield* Deferred.succeed(proceed, undefined) |
| 280 | yield* Fiber.join(outer) |
| 281 | yield* Fiber.join(other) |
| 282 | })) |
| 283 | |
| 284 | it.effect("a local write and a remote import updating the same projection row both commit", () => |
| 285 | Effect.gen(function*() { |
| 286 | const sql = yield* SqlClient.SqlClient |
| 287 | const journal = yield* SqlEventJournal.make({ |
| 288 | entryTable: "projection_entries", |
| 289 | remotesTable: "projection_remotes" |
| 290 | }) |
| 291 | // EventLog runs handlers inside the journal lock's transaction. |
| 292 | const withLock = journal.withLock(EventLogMessage.StoreId.make("store")) |
| 293 | yield* sql`CREATE TABLE callback_projection (id TEXT PRIMARY KEY, count INT NOT NULL)` |
| 294 | yield* sql`INSERT INTO callback_projection (id, count) VALUES ('p', 0)` |
| 295 | const project = Effect.orDie(sql`UPDATE callback_projection SET count = count + 1 WHERE id = 'p'`) |
| 296 | const updated = yield* Deferred.make<number>() |
| 297 | const proceed = yield* Deferred.make<void>() |
| 298 | // The local handler keeps the projection row locked until released. |
| 299 | const local = yield* journal.write({ |
| 300 | ...localEntry("local"), |
| 301 | effect: () => |
| 302 | project.pipe( |
| 303 | Effect.andThen(backendPid), |
| 304 | Effect.flatMap((pid) => Deferred.succeed(updated, pid)), |
| 305 | Effect.andThen(Deferred.await(proceed)) |
| 306 | ) |
| 307 | }).pipe(withLock, Effect.forkChild) |
| 308 | const holder = yield* Deferred.await(updated) |
| 309 | const remote = yield* journal.writeFromRemote({ |
| 310 | remoteId: EventJournal.makeRemoteIdUnsafe(), |
| 311 | entries: [remoteEntry("imported")], |
| 312 | effect: () => Effect.asVoid(project) |
| 313 | }).pipe(withLock, Effect.forkChild) |
| 314 | yield* waitUntilBlockedBy(holder, 1, remote) |
| 315 | yield* Deferred.succeed(proceed, undefined) |
| 316 | yield* Fiber.join(local) |
| 317 | yield* Fiber.join(remote) |
| 318 | const [{ count }] = yield* sql<{ count: number }>`SELECT count FROM callback_projection WHERE id = 'p'` |
| 319 | assert.strictEqual(count, 2) |
| 320 | })) |
| 321 | |
| 322 | it.effect("a write without an outer transaction rolls back its callback's SQL when the callback fails", () => |
| 323 | Effect.gen(function*() { |
| 324 | const sql = yield* SqlClient.SqlClient |
| 325 | const journal = yield* SqlEventJournal.make({ entryTable: "rollback_entries", remotesTable: "rollback_remotes" }) |
| 326 | yield* sql`CREATE TABLE rollback_projection (id TEXT PRIMARY KEY)` |
| 327 | yield* Effect.flip(journal.write({ |
| 328 | ...localEntry("failed"), |
| 329 | effect: () => Effect.andThen(sql`INSERT INTO rollback_projection (id) VALUES ('failed')`, Effect.fail("failed")) |
| 330 | })) |
| 331 | assert.deepStrictEqual(yield* sql`SELECT id FROM rollback_projection`, []) |
| 332 | })) |
| 333 | }) |
| 334 | |
| 335 | SqlEventJournalSharedTest.suite("pg", PgContainer.layerClient) |