@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 502 lines 21.1 KB
Raw
1 import { NodeRedis } from "@effect/platform-node"
2 import { assert, it } from "@effect/vitest"
3 import { RedisContainer } from "@testcontainers/redis"
4 import { Clock, Duration, Effect, Fiber, Latch, Layer, Queue, Schema } from "effect"
5 import * as BackingPersistenceTest from "effect-test/persistence/BackingPersistenceTest"
6 import * as CircuitBreakerRedisTest from "effect-test/persistence/CircuitBreakerRedisTest"
7 import * as CircuitBreakerTest from "effect-test/persistence/CircuitBreakerTest"
8 import * as PersistedCacheTest from "effect-test/persistence/PersistedCacheTest"
9 import * as PersistedQueueTest from "effect-test/persistence/PersistedQueueTest"
10 import * as RateLimiterTest from "effect-test/persistence/RateLimiterTest"
11 import { CircuitBreaker, PersistedQueue, Persistence, RateLimiter, Redis } from "effect/persistence"
12 import { TestClock } from "effect/testing"
13 import { createServer } from "node:net"
14
15 const RedisLayer = Layer.unwrap(
16 Effect.gen(function*() {
17 const container = yield* Effect.acquireRelease(
18 Effect.promise(() => new RedisContainer("redis:alpine").start()),
19 (container) => Effect.promise(() => container.stop())
20 )
21 return NodeRedis.layer({
22 socket: {
23 host: container.getHost(),
24 port: container.getMappedPort(6379)
25 }
26 })
27 }).pipe(
28 Effect.catchCause(() => Effect.fail(new PersistedCacheTest.TransientError()))
29 )
30 )
31
32 PersistedCacheTest.suite(
33 "NodeRedis",
34 Persistence.layerRedis.pipe(Layer.provide(RedisLayer))
35 )
36
37 BackingPersistenceTest.suite(
38 "NodeRedis",
39 Persistence.layerBackingRedis.pipe(Layer.provide(RedisLayer))
40 )
41
42 PersistedQueueTest.suite(
43 "NodeRedis",
44 // short intervals so the periodic reset runs while the suite's takes are
45 // in flight
46 PersistedQueue.layerStoreRedis({
47 pollInterval: "50 millis",
48 lockRefreshInterval: "100 millis"
49 }).pipe(Layer.provide(RedisLayer))
50 )
51
52 RateLimiterTest.suite(
53 "NodeRedis",
54 RateLimiter.layerStoreRedis().pipe(Layer.provide(RedisLayer))
55 )
56
57 CircuitBreakerTest.suite(
58 "NodeRedis",
59 (options) => CircuitBreaker.layerStoreRedis(options).pipe(Layer.provide(RedisLayer)),
60 // the store reads Redis TIME, so time only moves on the wall clock
61 (duration) => TestClock.withLive(Effect.sleep(duration))
62 )
63
64 CircuitBreakerRedisTest.suite("NodeRedis", RedisLayer)
65
66 it.layer(RateLimiter.layerStoreRedis().pipe(Layer.provideMerge(RedisLayer)), {
67 timeout: "30 seconds",
68 concurrent: false
69 })(
70 "RateLimiter token-bucket storage (NodeRedis)",
71 (it) => {
72 it.effect("returns the persisted balance after accumulating fractional costs", () =>
73 Effect.gen(function*() {
74 const redis = yield* Redis.Redis
75 const key = "fractional-persisted-balance"
76 const results = yield* RateLimiterTest.consumeFractionalCosts(key)
77 const stored = Number(yield* redis.send<string>("GET", `ratelimiter:${key}`))
78
79 assert.strictEqual(stored, 3.9999999999999996)
80 assert.strictEqual(results[results.length - 1].remaining, stored)
81 }))
82
83 it.effect(
84 "does not restore capacity early after a fractional token cost",
85 () =>
86 Effect.gen(function*() {
87 const redis = yield* Redis.Redis
88 const store = yield* RateLimiter.RateLimiterStore
89 const memory = yield* RateLimiter.RateLimiterStore.pipe(Effect.provide(RateLimiter.layerStoreMemory))
90 const opts = {
91 key: "fractional-cost-expiry",
92 limit: 5,
93 refillRate: Duration.seconds(4),
94 allowOverflow: false
95 }
96 const key = `ratelimiter:${opts.key}`
97 const refillKey = `${key}:refill`
98 yield* memory.tokenBucket({ ...opts, tokens: 0.5 })
99 yield* store.tokenBucket({ ...opts, tokens: 0.5 })
100 const refillAt = Number(yield* redis.send<string>("GET", refillKey))
101 assert.isAbove(yield* redis.send<number>("PTTL", key), 0)
102
103 // Use wall time: the reported bug expires these keys after 2s, before the 4s refill.
104 yield* Effect.sleep("2500 millis")
105 const beforeRefill = {
106 keys: yield* redis.send<number>("EXISTS", key, refillKey),
107 tokens: yield* redis.send<string | null>("GET", key),
108 memoryRemaining: (yield* memory.tokenBucket({ ...opts, tokens: 0 }))[0],
109 redisRemaining: (yield* store.tokenBucket({ ...opts, tokens: 0 }))[0]
110 }
111 const elapsed = (yield* Clock.currentTimeMillis) - refillAt
112 assert.isAtLeast(elapsed, 2_500)
113 assert.isBelow(elapsed, 4_000)
114
115 yield* Effect.sleep(4_100 - elapsed)
116 assert.strictEqual((yield* memory.tokenBucket({ ...opts, tokens: 0 }))[0], 5)
117 assert.strictEqual((yield* store.tokenBucket({ ...opts, tokens: 0 }))[0], 5)
118 assert.deepStrictEqual(beforeRefill, { keys: 2, tokens: "4.5", memoryRemaining: 4.5, redisRemaining: 4.5 })
119 }).pipe(TestClock.withLive),
120 10_000
121 )
122
123 for (const onExceeded of ["fail", "delay"] as const) {
124 it.effect(`${onExceeded}: restarts the refill interval after both Redis keys expire`, () => {
125 const key = `timing-expired-${onExceeded}`
126 const redisKey = `ratelimiter:${key}`
127 const refillKey = `${redisKey}:refill`
128 // TestClock does not advance Redis's expiry clock, so expire the keys by hand.
129 const idle = Effect.gen(function*() {
130 const redis = yield* Redis.Redis
131 yield* TestClock.adjust("359 seconds")
132 assert.strictEqual(yield* redis.send<number>("EXISTS", redisKey, refillKey), 2)
133 assert.strictEqual(yield* redis.send<number>("PEXPIRE", redisKey, "0"), 1)
134 assert.strictEqual(yield* redis.send<number>("PEXPIRE", refillKey, "0"), 1)
135 assert.strictEqual(yield* redis.send<number>("EXISTS", redisKey, refillKey), 0)
136 })
137 return RateLimiterTest.restartsInterval(key, onExceeded, 5, idle)
138 })
139 }
140 }
141 )
142
143 const PersistedQueueRedisLayer = Layer.mergeAll(
144 RedisLayer,
145 PersistedQueue.layer.pipe(
146 Layer.provideMerge(
147 PersistedQueue.layerStoreRedis().pipe(Layer.provide(RedisLayer))
148 )
149 )
150 )
151
152 it.effect("fails the initial connection by default", () =>
153 Effect.gen(function*() {
154 const port = yield* closedPort
155 const error = yield* Layer.build(NodeRedis.layer({
156 socket: {
157 host: "127.0.0.1",
158 port
159 }
160 })).pipe(Effect.flip)
161
162 assert.instanceOf(error, Redis.RedisError)
163 }))
164
165 it.layer(PersistedQueueRedisLayer, { timeout: "30 seconds" })(
166 "PersistedQueue (NodeRedis)",
167 (it) => {
168 it.effect("uses the node-redis protocol default", () =>
169 Effect.gen(function*() {
170 const redis = yield* NodeRedis.NodeRedis
171 assert.strictEqual(redis.client.options.RESP, undefined)
172 }))
173
174 it.effect("receives published messages", () =>
175 Effect.gen(function*() {
176 const redis = yield* Redis.Redis
177 const subscription = yield* redis.subscribe("effect-test")
178
179 yield* redis.send("PUBLISH", "effect-test", "hello")
180
181 assert.deepStrictEqual(yield* Queue.take(subscription), {
182 channel: "effect-test",
183 message: "hello"
184 })
185 }))
186
187 it.effect("runs blocking reads on a reserved connection without delaying shared commands", () =>
188 Effect.gen(function*() {
189 const redis = yield* Redis.Redis
190 const read = yield* redis.reserve.pipe(
191 Effect.flatMap((connection) =>
192 connection.send("XREAD", "BLOCK", "5000", "STREAMS", "effect-test-reserve", "$")
193 ),
194 Effect.scoped,
195 Effect.forkChild
196 )
197 // XREAD with "$" only sees entries added after it starts blocking
198 yield* redis.send<string>("CLIENT", "LIST").pipe(
199 Effect.repeat({ until: (clients) => clients.includes("cmd=xread") })
200 )
201 yield* redis.send("XADD", "effect-test-reserve", "*", "key", "value")
202
203 assert.include(JSON.stringify(yield* Fiber.join(read)), "value")
204 }))
205
206 // The shared PersistedQueue suite can only assert that exhausted elements
207 // are no longer delivered, which is also true if they are silently
208 // dropped. There is no public API for reading failed elements, so
209 // verifying they are preserved in the dead-letter list requires
210 // inspecting Redis directly.
211 it.effect("moves exhausted elements to the failed list", () =>
212 Effect.gen(function*() {
213 const redis = yield* NodeRedis.NodeRedis
214 const queueName = "test-redis-failed"
215
216 const queue = yield* PersistedQueue.make({
217 name: queueName,
218 schema: RedisItem,
219 maxAttempts: 1
220 })
221 const id = yield* queue.offer({ n: 42 })
222 const error = yield* queue.take(() => Effect.fail("boom")).pipe(Effect.flip)
223 assert.strictEqual(error, "boom")
224
225 const failed = yield* redis.use((client) => client.lRange(`effectq:${queueName}:failed`, 0, -1))
226 assert.strictEqual(failed.length, 1)
227 const failedItem = JSON.parse(failed[0])
228 assert.strictEqual(failedItem.id, id)
229 assert.deepStrictEqual(failedItem.element, { n: 42 })
230 assert.strictEqual(failedItem.attempts, 1)
231
232 const pending = yield* redis.use((client) => client.hLen(`effectq:${queueName}:pending`))
233 assert.strictEqual(pending, 0)
234 }))
235
236 it.effect("success after lease expiry but before reset is not redelivered", () =>
237 Effect.gen(function*() {
238 // Neither refresh nor reset can run again before this handler succeeds.
239 const queue = yield* makeOwnershipQueue("effectq-late-success:", "1 hour")
240 yield* queue.offer({ n: 42 }, { id: "order-1" })
241 yield* queue.take((_value, { attempts }) =>
242 Effect.gen(function*() {
243 assert.strictEqual(attempts, 1)
244 // Redis owns the lease clock; let it expire without another claimant.
245 yield* Effect.sleep("1100 millis")
246 })
247 )
248
249 // Only now allow recovery to run: a recorded success must leave no work.
250 const recovery = yield* makeOwnershipQueue("effectq-late-success:", "100 millis")
251 const duplicate = yield* recovery.take((_value, metadata) => Effect.succeed(metadata)).pipe(Effect.forkScoped)
252 yield* Effect.sleep("1500 millis")
253 assert.isUndefined(duplicate.pollUnsafe())
254 }).pipe(TestClock.withLive), { timeout: 20000 })
255
256 // it.layer shares its TestClock across concurrent tests. Each paused worker
257 // needs a private clock so another test cannot renew its lease before recovery.
258 it.effect("renews an expired lease before recovery revokes ownership", () =>
259 Effect.gen(function*() {
260 const queue = yield* makeOwnershipQueue("effectq-late-renewal:", "100 millis")
261 // Stagger the timers: renewal is due at 100ms, reset at 150ms.
262 yield* TestClock.adjust("50 millis")
263 yield* queue.offer({ n: 42 }, { id: "order-1" })
264 const taken = Latch.makeUnsafe()
265 const finish = Latch.makeUnsafe()
266 const claim = yield* queue.take(() => taken.open.pipe(Effect.andThen(finish.await))).pipe(Effect.forkScoped)
267 yield* taken.await
268 yield* Effect.sleep("1100 millis").pipe(TestClock.withLive)
269 // Only renewal runs, after Redis has expired the lease. Give its I/O
270 // time to complete before introducing a recovery worker.
271 yield* TestClock.adjust("50 millis")
272 yield* Effect.sleep("100 millis").pipe(TestClock.withLive)
273 const recovery = yield* makeOwnershipQueue("effectq-late-renewal:", "100 millis").pipe(TestClock.withLive)
274 const duplicate = yield* recovery.take((_value, metadata) => Effect.succeed(metadata)).pipe(
275 TestClock.withLive,
276 Effect.forkScoped
277 )
278 yield* Effect.sleep("400 millis").pipe(TestClock.withLive)
279 assert.isUndefined(duplicate.pollUnsafe())
280 yield* finish.open
281 yield* Fiber.join(claim)
282 yield* Effect.sleep("1500 millis").pipe(TestClock.withLive)
283 assert.isUndefined(duplicate.pollUnsafe())
284 }).pipe(Effect.provide(TestClock.layer())), { timeout: 20000 })
285
286 it.effect("revoked ownership cannot settle recovered work before reacquisition", () =>
287 Effect.gen(function*() {
288 const prefix = "effectq-revoked-owner:"
289 // Store construction pauses renewal. Takes use live time so polling
290 // and reset keep running even while this test's clock is stopped.
291 const queue = yield* makeOwnershipQueue(prefix, "100 millis")
292 yield* queue.offer({ n: 42 }, { id: "order-1" })
293 const taken = Latch.makeUnsafe()
294 const finish = Latch.makeUnsafe()
295 const stale = yield* queue.take(() => taken.open.pipe(Effect.andThen(finish.await))).pipe(
296 TestClock.withLive,
297 Effect.forkScoped
298 )
299 yield* taken.await
300 // Reset recovers the expired lease, but no taker can acquire it yet.
301 yield* Effect.sleep("1500 millis").pipe(TestClock.withLive)
302 yield* TestClock.adjust("100 millis")
303 yield* Effect.sleep("100 millis").pipe(TestClock.withLive)
304 yield* finish.open
305 yield* Fiber.join(stale)
306 const recovered = yield* queue.take((value, metadata) => Effect.succeed({ value, ...metadata })).pipe(
307 TestClock.withLive
308 )
309 assert.deepStrictEqual(recovered, { value: { n: 42 }, id: "order-1", attempts: 2 })
310 const duplicate = yield* queue.take(Effect.succeed).pipe(TestClock.withLive, Effect.forkScoped)
311 yield* Effect.sleep("1500 millis").pipe(TestClock.withLive)
312 assert.isUndefined(duplicate.pollUnsafe())
313 }).pipe(Effect.provide(TestClock.layer())), { timeout: 20000 })
314
315 it.effect(
316 "stale interruption does not release a replacement or decrement its attempts",
317 () =>
318 Effect.gen(function*() {
319 const a = yield* makeOwnershipQueue("effectq-stale-interrupt:", "1 hour")
320 const b = yield* makeOwnershipQueue("effectq-stale-interrupt:", "100 millis")
321 yield* a.offer({ n: 42 }, { id: "order-1" })
322 const taken = Latch.makeUnsafe()
323 const stale = yield* a.take(() => taken.open.pipe(Effect.andThen(Effect.never))).pipe(Effect.forkScoped)
324 yield* taken.await
325 const replaced = Latch.makeUnsafe()
326 const replacement = yield* b.take((_value, { attempts }) =>
327 Effect.gen(function*() {
328 assert.strictEqual(attempts, 2)
329 yield* replaced.open
330 return yield* Effect.never
331 })
332 ).pipe(Effect.forkScoped)
333 yield* replaced.await
334 yield* Fiber.interrupt(stale)
335 const duplicate = yield* b.take((_value, metadata) => Effect.succeed(metadata)).pipe(Effect.forkScoped)
336 yield* Effect.sleep("1500 millis")
337 assert.isUndefined(duplicate.pollUnsafe())
338 yield* Fiber.interrupt(replacement)
339 assert.strictEqual((yield* Fiber.join(duplicate)).attempts, 2)
340 }).pipe(TestClock.withLive),
341 { timeout: 20000 }
342 )
343
344 it.effect(
345 "same-store stale success preserves the replacement and its refresh registration",
346 () =>
347 Effect.gen(function*() {
348 const queue = yield* makeOwnershipQueue("effectq-same-store:", "100 millis")
349 yield* queue.offer({ n: 42 }, { id: "order-1" })
350 const aTaken = Latch.makeUnsafe()
351 const releaseA = Latch.makeUnsafe()
352 const stale = yield* queue.take(() => aTaken.open.pipe(Effect.andThen(releaseA.await))).pipe(
353 TestClock.withLive,
354 Effect.forkScoped
355 )
356 yield* aTaken.await
357 // Poll/reset use live time; only renewal stays paused until B takes.
358 const bTaken = Latch.makeUnsafe()
359 const replacement = yield* queue.take((_value, { attempts }) =>
360 Effect.gen(function*() {
361 assert.strictEqual(attempts, 2)
362 yield* bTaken.open
363 return yield* Effect.never
364 })
365 ).pipe(TestClock.withLive, Effect.forkScoped)
366 yield* bTaken.await
367 // Only the replacement can now renew. Keep ticking while Redis I/O
368 // completes, including while testing the old finalizer below.
369 yield* Effect.sleep("100 millis").pipe(
370 TestClock.withLive,
371 Effect.andThen(TestClock.adjust("100 millis")),
372 Effect.forever,
373 Effect.forkScoped
374 )
375 yield* releaseA.open
376 yield* Fiber.join(stale)
377 const duplicate = yield* queue.take((_value, metadata) => Effect.succeed(metadata)).pipe(
378 TestClock.withLive,
379 Effect.forkScoped
380 )
381 // The old finalizer must not unregister the new acquisition's refresh.
382 yield* Effect.sleep("1500 millis").pipe(TestClock.withLive)
383 assert.isUndefined(duplicate.pollUnsafe())
384 yield* Fiber.interrupt(replacement)
385 assert.strictEqual((yield* Fiber.join(duplicate)).attempts, 2)
386 }).pipe(Effect.provide(TestClock.layer())),
387 { timeout: 20000 }
388 )
389
390 it.effect("recovers elements from crashed workers", () =>
391 Effect.gen(function*() {
392 const prefix = "effectq-crash:"
393 const store = yield* PersistedQueue.makeStoreRedis({
394 prefix,
395 pollInterval: "50 millis",
396 lockRefreshInterval: "100 millis",
397 lockExpiration: "1 second"
398 })
399 const factory = yield* PersistedQueue.makeFactory.pipe(
400 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
401 )
402 const queue = yield* factory.make({ name: "crash-recovery", schema: RedisItem })
403 const redis = yield* Redis.Redis
404
405 // simulate a worker that claimed the element and then crashed: the
406 // element sits in the pending hash with a consumed attempt and no lock
407 yield* redis.send(
408 "HSET",
409 `${prefix}crash-recovery:pending`,
410 "crashed",
411 JSON.stringify({ id: "crashed", element: { n: 1 } })
412 )
413 yield* redis.send("HSET", `${prefix}crash-recovery:attempts`, "crashed", "1")
414
415 const result = yield* queue.take((value, metadata) => Effect.succeed([value.n, metadata.attempts]))
416 assert.deepStrictEqual(result, [1, 2])
417 }).pipe(TestClock.withLive), { timeout: 20000 })
418
419 it.effect("dead-letters elements from workers that crashed on the final attempt", () =>
420 Effect.gen(function*() {
421 const prefix = "effectq-crash-exhausted:"
422 const store = yield* PersistedQueue.makeStoreRedis({
423 prefix,
424 pollInterval: "50 millis",
425 lockRefreshInterval: "100 millis",
426 lockExpiration: "1 second"
427 })
428 const factory = yield* PersistedQueue.makeFactory.pipe(
429 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
430 )
431 const queue = yield* factory.make({ name: "crash-exhausted", schema: RedisItem, maxAttempts: 1 })
432 const redis = yield* Redis.Redis
433
434 // the final attempt was claimed by a worker that crashed, so no
435 // finalizer will ever settle this element
436 yield* redis.send(
437 "HSET",
438 `${prefix}crash-exhausted:pending`,
439 "crashed",
440 JSON.stringify({ id: "crashed", element: { n: 1 } })
441 )
442 yield* redis.send("HSET", `${prefix}crash-exhausted:attempts`, "crashed", "1")
443
444 // an active taker runs the periodic reset that dead-letters such
445 // elements instead of redelivering them
446 const fiber = yield* queue.take(Effect.succeed).pipe(Effect.forkScoped)
447 yield* Effect.sleep(1000)
448
449 const failed = yield* redis.send<Array<string>>("LRANGE", `${prefix}crash-exhausted:failed`, "0", "-1")
450 assert.strictEqual(failed.length, 1)
451 const failedItem = JSON.parse(failed[0])
452 assert.strictEqual(failedItem.id, "crashed")
453 assert.deepStrictEqual(failedItem.element, { n: 1 })
454 assert.strictEqual(failedItem.attempts, 1)
455 assert.include(failedItem.lastFailure, "Lock expired after final attempt")
456
457 const pending = yield* redis.send<number>("HLEN", `${prefix}crash-exhausted:pending`)
458 assert.strictEqual(Number(pending), 0)
459 assert.isUndefined(fiber.pollUnsafe())
460 }).pipe(TestClock.withLive), { timeout: 20000 })
461 }
462 )
463
464 const makeOwnershipQueue = Effect.fnUntraced(function*(prefix: string, lockRefreshInterval: Duration.Input) {
465 const store = yield* PersistedQueue.makeStoreRedis({
466 prefix,
467 pollInterval: "50 millis",
468 lockExpiration: "1 second",
469 lockRefreshInterval
470 })
471 const factory = yield* PersistedQueue.makeFactory.pipe(
472 Effect.provideService(PersistedQueue.PersistedQueueStore, store)
473 )
474 return yield* factory.make({ name: "ownership", schema: RedisItem })
475 })
476
477 const RedisItem = Schema.Struct({
478 n: Schema.Number
479 })
480
481 const closedPort = Effect.promise(
482 () =>
483 new Promise<number>((resolve, reject) => {
484 const server = createServer()
485 server.once("error", reject)
486 server.listen(0, "127.0.0.1", () => {
487 const address = server.address()
488 if (address === null || typeof address === "string") {
489 server.close()
490 reject(new Error("Could not allocate a TCP port"))
491 return
492 }
493 server.close((error) => {
494 if (error) {
495 reject(error)
496 } else {
497 resolve(address.port)
498 }
499 })
500 })
501 })
502 )