@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 283 lines 10.7 KB
Raw
1 import { assert, it } from "@effect/vitest"
2 import { Duration, Effect, Fiber, Latch, Layer, Schema } from "effect"
3 import * as PersistedCacheTest from "effect-test/persistence/PersistedCacheTest"
4 import * as PersistedQueueSqlTest from "effect-test/persistence/PersistedQueueSqlTest"
5 import * as PersistedQueueTest from "effect-test/persistence/PersistedQueueTest"
6 import * as SqlCleanupTest from "effect-test/persistence/SqlCleanupTest"
7 import * as Arr from "effect/Array"
8 import { PersistedQueue, Persistence } from "effect/persistence"
9 import { SqlClient } from "effect/sql"
10 import { TestClock } from "effect/testing"
11 import { PgContainer } from "./utils.ts"
12
13 PersistedCacheTest.suite(
14 "sql-pg-multi",
15 Persistence.layerSqlMultiTable.pipe(Layer.provide(PgContainer.layerClient))
16 )
17
18 PersistedCacheTest.suite(
19 "sql-pg-single",
20 Persistence.layerSql.pipe(Layer.provide(PgContainer.layerClient))
21 )
22
23 PersistedQueueTest.suite(
24 "sql-pg",
25 PersistedQueue.layerStoreSql().pipe(Layer.provide(PgContainer.layerClient))
26 )
27
28 it.layer(PgContainer.layerClient, { timeout: "30 seconds" })((it) => PersistedQueueSqlTest.suiteWith(it))
29
30 it.layer(PgContainer.layerClient, { timeout: "30 seconds" })("PersistedQueue SQL locks", (it) => {
31 it.effect("refreshes locks for acquired elements", () =>
32 Effect.gen(function*() {
33 const options = {
34 tableName: "effect_queue_lock_refresh",
35 pollInterval: "10 millis",
36 lockRefreshInterval: "100 millis",
37 lockExpiration: "1 second"
38 } as const
39 const store1 = yield* PersistedQueue.makeStoreSql(options)
40 const store2 = yield* PersistedQueue.makeStoreSql(options)
41 const element = { message: "hello" }
42
43 yield* store1.offer({
44 name: "lock-refresh",
45 id: crypto.randomUUID(),
46 element,
47 isCustomId: false
48 })
49
50 const takeOptions = {
51 name: "lock-refresh",
52 maxAttempts: 10,
53 retryDelay: () => Effect.succeed(Duration.zero)
54 }
55 const acquired = Latch.makeUnsafe()
56 const first = yield* Effect.scoped(Effect.gen(function*() {
57 yield* store1.take(takeOptions)
58 yield* acquired.open
59 return yield* Effect.never
60 })).pipe(Effect.forkScoped)
61
62 yield* acquired.await
63
64 const second = yield* Effect.scoped(
65 store2.take(takeOptions)
66 ).pipe(Effect.forkScoped)
67
68 yield* Effect.sleep("1500 millis")
69 assert.isUndefined(second.pollUnsafe())
70
71 yield* Fiber.interrupt(first)
72 const received = yield* Fiber.join(second)
73 assert.deepStrictEqual(received.element, element)
74 }).pipe(TestClock.withLive))
75
76 it.effect("dead-letters malformed JSON and continues", () =>
77 Effect.gen(function*() {
78 const tableName = "effect_queue_invalid_json"
79 const store = yield* PersistedQueue.makeStoreSql({
80 tableName,
81 pollInterval: "10 millis"
82 })
83 const factory = yield* PersistedQueue.makeFactory.pipe(
84 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
85 )
86 const queue = yield* factory.make({
87 name: "invalid-json",
88 schema: Schema.String
89 })
90 const sql = (yield* SqlClient.SqlClient).withoutTransforms()
91 const table = sql(tableName)
92 const poisonId = crypto.randomUUID()
93
94 yield* store.offer({
95 name: "invalid-json",
96 id: poisonId,
97 element: "poison",
98 isCustomId: false
99 })
100 yield* sql`UPDATE ${table} SET element = ${"{"} WHERE id = ${poisonId}`
101 yield* queue.offer("valid")
102
103 // the malformed element is skipped and the next one is delivered
104 const value = yield* queue.take(Effect.succeed)
105 assert.strictEqual(value, "valid")
106
107 const rows = yield* sql<{
108 readonly state: string
109 readonly attempts: number
110 readonly last_failure: string | null
111 }>`SELECT state, attempts, last_failure FROM ${table} WHERE id = ${poisonId}`
112 assert.strictEqual(rows[0].state, "failed")
113 assert.strictEqual(Number(rows[0].attempts), 1)
114 assert.isNotNull(rows[0].last_failure)
115 }).pipe(TestClock.withLive))
116
117 it.effect("processes elements exactly once across two workers", () =>
118 Effect.gen(function*() {
119 const options = {
120 tableName: "effect_queue_two_workers",
121 pollInterval: "10 millis"
122 } as const
123 const makeQueue = Effect.fnUntraced(function*(store: PersistedQueue.PersistedQueueStore["Service"]) {
124 const factory = yield* PersistedQueue.makeFactory.pipe(
125 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
126 )
127 return yield* factory.make({ name: "two-workers", schema: Schema.Number })
128 })
129 const queue1 = yield* makeQueue(yield* PersistedQueue.makeStoreSql(options))
130 const queue2 = yield* makeQueue(yield* PersistedQueue.makeStoreSql(options))
131
132 const total = 20
133 yield* Effect.forEach(
134 Array.from({ length: total }, (_, i) => i),
135 (n) => queue1.offer(n),
136 { concurrency: 8, discard: true }
137 )
138
139 const seen = new Map<number, number>()
140 const worker = (queue: typeof queue1) =>
141 queue.take((n) =>
142 Effect.sync(() => {
143 seen.set(n, (seen.get(n) ?? 0) + 1)
144 })
145 ).pipe(Effect.forever, Effect.forkScoped)
146 yield* worker(queue1)
147 yield* worker(queue1)
148 yield* worker(queue2)
149 yield* worker(queue2)
150
151 yield* waitFor(() => Effect.sync(() => seen.size === total))
152
153 assert.strictEqual(seen.size, total)
154 for (const [n, count] of seen) {
155 assert.strictEqual(count, 1, `deliveries for element ${n}`)
156 }
157 }).pipe(TestClock.withLive), { timeout: 30000 })
158
159 it.effect("recovers elements from crashed workers after lock expiration", () =>
160 Effect.gen(function*() {
161 const tableName = "effect_queue_crash_recovery"
162 const store = yield* PersistedQueue.makeStoreSql({
163 tableName,
164 pollInterval: "10 millis",
165 lockExpiration: "1 second"
166 })
167 const factory = yield* PersistedQueue.makeFactory.pipe(
168 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
169 )
170 const queue = yield* factory.make({ name: "crash-recovery", schema: Schema.Number })
171 const sql = (yield* SqlClient.SqlClient).withoutTransforms()
172 const table = sql(tableName)
173
174 const id = yield* queue.offer(1)
175 // simulate a worker that claimed the element and then crashed: the lock
176 // is held by a dead worker and the claim consumed an attempt
177 yield* sql`
178 UPDATE ${table}
179 SET acquired_by = ${crypto.randomUUID()}, acquired_at = NOW(), attempts = 1
180 WHERE id = ${id}
181 `
182
183 const attempts = yield* queue.take((_n, metadata) => Effect.succeed(metadata.attempts))
184 assert.strictEqual(attempts, 2)
185 }).pipe(TestClock.withLive), { timeout: 20000 })
186
187 it.effect("dead-letters elements from workers that crashed on the final attempt", () =>
188 Effect.gen(function*() {
189 const tableName = "effect_queue_crash_exhausted"
190 const store = yield* PersistedQueue.makeStoreSql({
191 tableName,
192 pollInterval: "10 millis",
193 lockExpiration: "500 millis",
194 lockRefreshInterval: "200 millis"
195 })
196 const factory = yield* PersistedQueue.makeFactory.pipe(
197 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
198 )
199 const queue = yield* factory.make({ name: "crash-exhausted", schema: Schema.Number, maxAttempts: 1 })
200 const sql = (yield* SqlClient.SqlClient).withoutTransforms()
201 const table = sql(tableName)
202
203 const id = yield* queue.offer(1)
204 // the final attempt was claimed by a worker that crashed, so no
205 // finalizer will ever settle this element
206 yield* sql`
207 UPDATE ${table}
208 SET acquired_by = ${crypto.randomUUID()}, acquired_at = NOW(), attempts = 1
209 WHERE id = ${id}
210 `
211
212 // an active taker runs the periodic pass that flips such rows to failed
213 const fiber = yield* queue.take(Effect.succeed).pipe(Effect.forkScoped)
214
215 const state = () =>
216 sql<{ readonly state: string; readonly last_failure: string | null }>`
217 SELECT state, last_failure FROM ${table} WHERE id = ${id}
218 `.pipe(Effect.map((rows) => rows[0]))
219 yield* waitFor(() => state().pipe(Effect.map((row) => row.state === "failed")))
220
221 const row = yield* state()
222 assert.strictEqual(row.state, "failed")
223 assert.include(row.last_failure ?? "", "Lock expired after final attempt")
224 assert.isUndefined(fiber.pollUnsafe())
225 }).pipe(TestClock.withLive), { timeout: 20000 })
226 })
227
228 // polls a condition on the live clock until it holds or the rounds run out
229 const waitFor = Effect.fnUntraced(function*<E>(condition: () => Effect.Effect<boolean, E>) {
230 for (let i = 0; i < 100; i++) {
231 if (yield* condition()) return
232 yield* Effect.sleep(100)
233 }
234 })
235
236 it.layer(PgContainer.layerClient, { timeout: "30 seconds" })("Persistence SQL cleanup", (it) => {
237 it.effect("deletes expired entries in batches", () =>
238 Effect.gen(function*() {
239 const sql = (yield* SqlClient.SqlClient).withoutTransforms()
240 const table = sql("effect_persistence")
241 const expiredCount = sql<{ readonly count: number }>`
242 SELECT COUNT(*)::INT AS count FROM ${table} WHERE store_id = 'expired'
243 `.pipe(Effect.map((rows) => rows[0].count))
244 yield* sql`
245 CREATE TABLE ${table} (
246 store_id TEXT NOT NULL,
247 id TEXT NOT NULL,
248 value TEXT NOT NULL,
249 expires BIGINT,
250 PRIMARY KEY (store_id, id)
251 )
252 `
253
254 const entries = Array.from({ length: SqlCleanupTest.expiredEntryCount }, (_, i) => ({
255 store_id: "expired",
256 id: String(i),
257 value: "{}",
258 expires: SqlCleanupTest.expiredAtEpoch
259 }))
260 assert(Arr.isArrayNonEmpty(entries))
261 yield* sql`INSERT INTO ${table} ${sql.insert(entries)}`.unprepared
262 yield* sql`
263 INSERT INTO ${table} (store_id, id, value, expires)
264 VALUES ('live', 'live', '{}', NULL), ('live', 'future', '{}', ${BigInt(SqlCleanupTest.futureExpiresAt)})
265 `
266
267 yield* Layer.build(Persistence.layerBackingSql).pipe(TestClock.withLive)
268
269 const expired = yield* SqlCleanupTest.waitForCount(expiredCount, (count) => count === 0)
270 assert.strictEqual(expired, 0)
271 const live = yield* sql<{ readonly count: number }>`
272 SELECT COUNT(*)::INT AS count FROM ${table} WHERE store_id = 'live'
273 `
274 assert.strictEqual(live[0].count, 2)
275
276 const indexes = yield* sql<{ readonly count: number }>`
277 SELECT COUNT(*)::INT AS count FROM pg_indexes
278 WHERE tablename = 'effect_persistence'
279 AND indexname = 'effect_persistence_expires_idx'
280 `
281 assert.strictEqual(indexes[0].count, 1)
282 }), { timeout: SqlCleanupTest.testTimeout })
283 })