The best way to control Claude Code, Codex and more from any computer or phone

main
ts 335 lines 15.8 KB
Raw
1 import { PgClient } from "@effect/sql-pg"
2 import { assert, it } from "@effect/vitest"
3 import { Deferred, Effect, Fiber, Layer, Option, Redacted } from "effect"
4 import * as SqlEventJournalSharedTest from "effect-test/eventlog/SqlEventJournalSharedTest"
5 import * as Arr from "effect/Array"
6 import * as EventJournal from "effect/eventlog/EventJournal"
7 import * as EventLogMessage from "effect/eventlog/EventLogMessage"
8 import * as SqlEventJournal from "effect/eventlog/SqlEventJournal"
9 import { Reactivity } from "effect/reactivity"
10 import * as SqlClient from "effect/sql/SqlClient"
11 import * as Crypto from "node:crypto"
12 import { PgContainer, realSleep } from "./utils.ts"
13
14 const localEntry = (primaryKey: string) => ({ event: "Repro", primaryKey, payload: new Uint8Array() })
15
16 const remoteEntry = (primaryKey: string, msecs?: number) =>
17 new EventJournal.RemoteEntry({
18 remoteSequence: 0,
19 entry: new EventJournal.Entry({
20 id: EventJournal.makeEntryIdUnsafe(msecs === undefined ? {} : { msecs }),
21 event: "Repro",
22 primaryKey,
23 payload: new Uint8Array()
24 }, { disableChecks: true })
25 })
26
27 const backendPid = Effect.gen(function*() {
28 const sql = yield* SqlClient.SqlClient
29 const [{ pid }] = yield* sql<{ pid: number }>`SELECT pg_backend_pid() AS pid`
30 return pid
31 })
32
33 // Fails unless, within ten seconds, `count` backends are blocked by `holder`
34 // directly or behind one that is, or `fiber` succeeds.
35 const waitUntilBlockedBy = Effect.fnUntraced(function*<A, E>(holder: number, count: number, fiber?: Fiber.Fiber<A, E>) {
36 const sql = yield* SqlClient.SqlClient
37 const deadline = Date.now() + 10_000
38 while (Date.now() < deadline) {
39 const exit = fiber?.pollUnsafe()
40 if (exit !== undefined) return yield* exit
41 const [{ blocked }] = yield* sql<{ blocked: number }>`
42 SELECT count(*)::int4 AS blocked FROM pg_stat_activity waiter
43 WHERE ${holder} = ANY(pg_blocking_pids(waiter.pid)) OR EXISTS (
44 SELECT 1 FROM pg_stat_activity other
45 WHERE other.pid = ANY(pg_blocking_pids(waiter.pid)) AND ${holder} = ANY(pg_blocking_pids(other.pid))
46 )
47 `
48 if (blocked >= count) return
49 yield* realSleep
50 }
51 return yield* Effect.die(new Error(`Expected ${count} backends blocked by ${holder} within 10 seconds`))
52 })
53
54 // Exposes the container so tests can open a read-only client.
55 const TestLayer = Layer.merge(PgContainer.layerClient, PgContainer.layer)
56
57 it.layer(TestLayer, { timeout: "30 seconds" })("SqlEventJournal", (it) => {
58 it.effect("round-trips a local entry", () =>
59 Effect.gen(function*() {
60 const journal = yield* SqlEventJournal.make({
61 entryTable: "local_entries",
62 remotesTable: "local_remotes"
63 })
64 const written = yield* journal.write({
65 event: "UserCreated",
66 primaryKey: "user-1",
67 payload: new Uint8Array([1]),
68 effect: (entry) => Effect.succeed(entry)
69 })
70 const entries = yield* journal.entries
71 assert.strictEqual(entries.length, 1)
72 assert.strictEqual(entries[0].idString, written.idString)
73 assert.strictEqual(entries[0].event, "UserCreated")
74 assert.strictEqual(entries[0].primaryKey, "user-1")
75 assert.deepStrictEqual(entries[0].payload, new Uint8Array([1]))
76 assert.strictEqual(entries[0].createdAtMillis, written.createdAtMillis)
77 }))
78
79 it.effect("round-trips a remote entry and advances its sequence", () =>
80 Effect.gen(function*() {
81 const journal = yield* SqlEventJournal.make({
82 entryTable: "remote_entries",
83 remotesTable: "remote_remotes"
84 })
85 const remoteId = EventJournal.makeRemoteIdUnsafe()
86 const remoteEntry = new EventJournal.Entry({
87 id: EventJournal.makeEntryIdUnsafe({ msecs: 1_700_000_000_000 }),
88 event: "UserCreated",
89 primaryKey: "user-2",
90 payload: new Uint8Array([2])
91 }, { disableChecks: true })
92 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 0)
93 yield* journal.writeFromRemote({
94 remoteId,
95 entries: [new EventJournal.RemoteEntry({ remoteSequence: 0, entry: remoteEntry })],
96 effect: () => Effect.void
97 })
98
99 const entries = yield* journal.entries
100 assert.strictEqual(entries.length, 1)
101 assert.strictEqual(entries[0].idString, remoteEntry.idString)
102 assert.strictEqual(entries[0].event, "UserCreated")
103 assert.strictEqual(entries[0].primaryKey, "user-2")
104 assert.deepStrictEqual(entries[0].payload, new Uint8Array([2]))
105 assert.strictEqual(entries[0].createdAtMillis, 1_700_000_000_000)
106 assert.strictEqual(yield* journal.nextRemoteSequence(remoteId), 1)
107 }))
108
109 it.effect("reopens a migrated journal read-only", () =>
110 Effect.gen(function*() {
111 const container = yield* PgContainer
112 const readonly = yield* PgClient.make({
113 url: Redacted.make(container.getConnectionUri()),
114 startupParameters: { default_transaction_read_only: "on" }
115 })
116 const options = { entryTable: "readonly_entries", remotesTable: "readonly_remotes" }
117 const journal = yield* SqlEventJournal.make(options)
118 const written = yield* journal.write({ ...localEntry("a"), effect: Effect.succeed })
119 const reopened = yield* SqlEventJournal.make(options).pipe(Effect.provideService(SqlClient.SqlClient, readonly))
120 assert.deepStrictEqual((yield* reopened.entries).map((entry) => entry.idString), [written.idString])
121 }).pipe(Effect.provide(Reactivity.layer)))
122
123 it.effect("writes and finds conflicts for primary keys beyond the index tuple limit", () =>
124 Effect.gen(function*() {
125 const journal = yield* SqlEventJournal.make({ entryTable: "long_key_entries", remotesTable: "long_key_remotes" })
126 // Random text barely compresses, so the key stays above the ~2.7 KB B-tree tuple limit.
127 const primaryKey = Crypto.randomBytes(6000).toString("base64")
128 const written = yield* journal.write({ ...localEntry(primaryKey), effect: Effect.succeed })
129 const conflicts: Array<string> = []
130 yield* journal.writeFromRemote({
131 remoteId: EventJournal.makeRemoteIdUnsafe(),
132 entries: [remoteEntry(primaryKey, written.createdAtMillis - 1)],
133 effect: (batch) => Effect.sync(() => conflicts.push(...batch[0].conflicts.map((entry) => entry.idString)))
134 })
135 assert.deepStrictEqual(conflicts, [written.idString])
136 }))
137
138 it.effect("concurrent opens migrate a legacy database once", () =>
139 Effect.gen(function*() {
140 const sql = yield* SqlClient.SqlClient
141 const options = { entryTable: "concurrent_open_entries", remotesTable: "concurrent_open_remotes" }
142 // The entry table as created by the first release of SqlEventJournal.
143 yield* sql`
144 CREATE TABLE concurrent_open_entries (
145 id BYTEA PRIMARY KEY,
146 event TEXT NOT NULL,
147 primary_key TEXT NOT NULL,
148 payload BYTEA NOT NULL,
149 timestamp BIGINT NOT NULL
150 )`
151 const remoteA = EventJournal.makeRemoteIdUnsafe()
152 const remoteB = EventJournal.makeRemoteIdUnsafe()
153 const entries = Arr.make(remoteEntry("pending").entry, remoteEntry("fromA").entry, remoteEntry("fromB").entry)
154 yield* sql`INSERT INTO concurrent_open_entries ${
155 sql.insert(Arr.map(entries, (entry) => ({
156 id: entry.id,
157 event: entry.event,
158 primary_key: entry.primaryKey,
159 payload: entry.payload,
160 timestamp: entry.createdAtMillis
161 })))
162 }`
163 yield* sql`
164 CREATE TABLE concurrent_open_remotes (
165 remote_id BYTEA NOT NULL,
166 entry_id BYTEA NOT NULL,
167 sequence INT NOT NULL,
168 PRIMARY KEY (remote_id, entry_id)
169 )`
170 yield* sql`INSERT INTO concurrent_open_remotes ${
171 sql.insert([
172 { remote_id: remoteA, entry_id: entries[1].id, sequence: 1 },
173 { remote_id: remoteB, entry_id: entries[2].id, sequence: 7 }
174 ])
175 }`
176 yield* sql`
177 CREATE TABLE concurrent_open_remotes_cursors (
178 remote_id BYTEA PRIMARY KEY,
179 next_sequence INT NOT NULL
180 )`
181 yield* sql`INSERT INTO concurrent_open_remotes_cursors ${sql.insert({ remote_id: remoteA, next_sequence: 9 })}`
182 const warmed = yield* Deferred.make<void>()
183 const startWaiting = yield* Deferred.make<void>()
184 const migrated = yield* Deferred.make<number>()
185 const release = yield* Deferred.make<void>()
186 const waiting = yield* Effect.gen(function*() {
187 // Pin this backend and cache both old names before the other opener changes them.
188 const [before] = yield* sql<{ legacy: string | null; sources: string | null }>`
189 SELECT to_regclass('concurrent_open_remotes')::text AS legacy,
190 to_regclass('concurrent_open_entries_sources')::text AS sources
191 `
192 assert.strictEqual(before.legacy, "concurrent_open_remotes")
193 assert.isNull(before.sources)
194 yield* Deferred.succeed(warmed, undefined)
195 yield* Deferred.await(startWaiting)
196 return yield* SqlEventJournal.make(options)
197 }).pipe(sql.withTransaction, Effect.forkChild)
198 yield* Deferred.await(warmed)
199 const first = yield* SqlEventJournal.make(options).pipe(
200 Effect.tap(() => backendPid.pipe(Effect.flatMap((pid) => Deferred.succeed(migrated, pid)))),
201 Effect.tap(Deferred.await(release)),
202 sql.withTransaction,
203 Effect.forkChild
204 )
205 const holder = yield* Deferred.await(migrated)
206 yield* Deferred.succeed(startWaiting, undefined)
207 yield* waitUntilBlockedBy(holder, 1, waiting)
208 yield* Deferred.succeed(release, undefined)
209 const journals = [yield* Fiber.join(first), yield* Fiber.join(waiting)]
210 for (const journal of journals) {
211 assert.strictEqual(yield* journal.nextRemoteSequence(remoteA), 9)
212 assert.strictEqual(yield* journal.nextRemoteSequence(remoteB), 8)
213 assert.deepStrictEqual(
214 (yield* journal.entries).map((entry) => entry.idString).sort(),
215 entries.map((entry) => entry.idString).sort()
216 )
217 }
218 const upload = (remoteId: EventJournal.RemoteId) =>
219 journals[0].withRemoteUncommited(remoteId, (entries) =>
220 Effect.succeed(entries.map((entry) => entry.primaryKey).sort()))
221 assert.deepStrictEqual(yield* upload(remoteA), Option.some(["fromB", "pending"]))
222 assert.deepStrictEqual(yield* upload(remoteB), Option.some(["fromA", "pending"]))
223 const [{ legacy }] = yield* sql<{ legacy: string | null }>`
224 SELECT to_regclass('concurrent_open_remotes')::text AS legacy
225 `
226 assert.isNull(legacy)
227 }))
228
229 it.effect("withRemoteUncommited uploads entries whose commits finish out of order", () =>
230 Effect.gen(function*() {
231 const sql = yield* SqlClient.SqlClient
232 const journal = yield* SqlEventJournal.make({ entryTable: "ordering_entries", remotesTable: "ordering_remotes" })
233 const remoteId = EventJournal.makeRemoteIdUnsafe()
234 const upload = journal.withRemoteUncommited(remoteId, (entries) => Effect.succeed(entries.map((e) => e.idString)))
235 const written = yield* Deferred.make<number>()
236 const release = yield* Deferred.make<void>()
237 // The first write stays uncommitted until released.
238 const first = yield* journal.write({ ...localEntry("first"), effect: Effect.succeed }).pipe(
239 Effect.tap(Effect.flatMap(backendPid, (pid) => Deferred.succeed(written, pid))),
240 Effect.tap(Deferred.await(release)),
241 sql.withTransaction,
242 Effect.forkChild
243 )
244 const holder = yield* Deferred.await(written)
245 const second = yield* Effect.forkChild(journal.write({ ...localEntry("second"), effect: Effect.succeed }))
246 yield* waitUntilBlockedBy(holder, 1, second)
247 const beforeCommit = yield* upload
248 yield* Deferred.succeed(release, undefined)
249 const ids = [(yield* Fiber.join(first)).idString, (yield* Fiber.join(second)).idString]
250 const afterCommit = yield* upload
251 const uploaded = [...Option.getOrElse(beforeCommit, () => []), ...Option.getOrElse(afterCommit, () => [])]
252 assert.deepStrictEqual(uploaded.sort(), ids.sort())
253 }))
254
255 it.effect("an outer transaction can import an entry that another remote is importing", () =>
256 Effect.gen(function*() {
257 const sql = yield* SqlClient.SqlClient
258 const journal = yield* SqlEventJournal.make({
259 entryTable: "overlapping_entries",
260 remotesTable: "overlapping_remotes"
261 })
262 const b = remoteEntry("b")
263 const importEntry = (remoteId: EventJournal.RemoteId, entry: EventJournal.RemoteEntry) =>
264 journal.writeFromRemote({ remoteId, entries: [entry], effect: () => Effect.void })
265 const imported = yield* Deferred.make<number>()
266 const proceed = yield* Deferred.make<void>()
267 const remoteId = EventJournal.makeRemoteIdUnsafe()
268 const outer = yield* importEntry(remoteId, remoteEntry("a")).pipe(
269 Effect.andThen(backendPid),
270 Effect.flatMap((pid) => Deferred.succeed(imported, pid)),
271 Effect.andThen(Deferred.await(proceed)),
272 Effect.andThen(importEntry(remoteId, b)),
273 sql.withTransaction,
274 Effect.forkChild
275 )
276 const holder = yield* Deferred.await(imported)
277 const other = yield* Effect.forkChild(importEntry(EventJournal.makeRemoteIdUnsafe(), b))
278 yield* waitUntilBlockedBy(holder, 1, other)
279 yield* Deferred.succeed(proceed, undefined)
280 yield* Fiber.join(outer)
281 yield* Fiber.join(other)
282 }))
283
284 it.effect("a local write and a remote import updating the same projection row both commit", () =>
285 Effect.gen(function*() {
286 const sql = yield* SqlClient.SqlClient
287 const journal = yield* SqlEventJournal.make({
288 entryTable: "projection_entries",
289 remotesTable: "projection_remotes"
290 })
291 // EventLog runs handlers inside the journal lock's transaction.
292 const withLock = journal.withLock(EventLogMessage.StoreId.make("store"))
293 yield* sql`CREATE TABLE callback_projection (id TEXT PRIMARY KEY, count INT NOT NULL)`
294 yield* sql`INSERT INTO callback_projection (id, count) VALUES ('p', 0)`
295 const project = Effect.orDie(sql`UPDATE callback_projection SET count = count + 1 WHERE id = 'p'`)
296 const updated = yield* Deferred.make<number>()
297 const proceed = yield* Deferred.make<void>()
298 // The local handler keeps the projection row locked until released.
299 const local = yield* journal.write({
300 ...localEntry("local"),
301 effect: () =>
302 project.pipe(
303 Effect.andThen(backendPid),
304 Effect.flatMap((pid) => Deferred.succeed(updated, pid)),
305 Effect.andThen(Deferred.await(proceed))
306 )
307 }).pipe(withLock, Effect.forkChild)
308 const holder = yield* Deferred.await(updated)
309 const remote = yield* journal.writeFromRemote({
310 remoteId: EventJournal.makeRemoteIdUnsafe(),
311 entries: [remoteEntry("imported")],
312 effect: () => Effect.asVoid(project)
313 }).pipe(withLock, Effect.forkChild)
314 yield* waitUntilBlockedBy(holder, 1, remote)
315 yield* Deferred.succeed(proceed, undefined)
316 yield* Fiber.join(local)
317 yield* Fiber.join(remote)
318 const [{ count }] = yield* sql<{ count: number }>`SELECT count FROM callback_projection WHERE id = 'p'`
319 assert.strictEqual(count, 2)
320 }))
321
322 it.effect("a write without an outer transaction rolls back its callback's SQL when the callback fails", () =>
323 Effect.gen(function*() {
324 const sql = yield* SqlClient.SqlClient
325 const journal = yield* SqlEventJournal.make({ entryTable: "rollback_entries", remotesTable: "rollback_remotes" })
326 yield* sql`CREATE TABLE rollback_projection (id TEXT PRIMARY KEY)`
327 yield* Effect.flip(journal.write({
328 ...localEntry("failed"),
329 effect: () => Effect.andThen(sql`INSERT INTO rollback_projection (id) VALUES ('failed')`, Effect.fail("failed"))
330 }))
331 assert.deepStrictEqual(yield* sql`SELECT id FROM rollback_projection`, [])
332 }))
333 })
334
335 SqlEventJournalSharedTest.suite("pg", PgContainer.layerClient)