The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { NodeRedis } from "@effect/platform-node" |
| 2 | import { assert, it } from "@effect/vitest" |
| 3 | import { RedisContainer } from "@testcontainers/redis" |
| 4 | import { Clock, Duration, Effect, Fiber, Latch, Layer, Queue, Schema } from "effect" |
| 5 | import * as BackingPersistenceTest from "effect-test/persistence/BackingPersistenceTest" |
| 6 | import * as CircuitBreakerRedisTest from "effect-test/persistence/CircuitBreakerRedisTest" |
| 7 | import * as CircuitBreakerTest from "effect-test/persistence/CircuitBreakerTest" |
| 8 | import * as PersistedCacheTest from "effect-test/persistence/PersistedCacheTest" |
| 9 | import * as PersistedQueueTest from "effect-test/persistence/PersistedQueueTest" |
| 10 | import * as RateLimiterTest from "effect-test/persistence/RateLimiterTest" |
| 11 | import { CircuitBreaker, PersistedQueue, Persistence, RateLimiter, Redis } from "effect/persistence" |
| 12 | import { TestClock } from "effect/testing" |
| 13 | import { createServer } from "node:net" |
| 14 | |
| 15 | const RedisLayer = Layer.unwrap( |
| 16 | Effect.gen(function*() { |
| 17 | const container = yield* Effect.acquireRelease( |
| 18 | Effect.promise(() => new RedisContainer("redis:alpine").start()), |
| 19 | (container) => Effect.promise(() => container.stop()) |
| 20 | ) |
| 21 | return NodeRedis.layer({ |
| 22 | socket: { |
| 23 | host: container.getHost(), |
| 24 | port: container.getMappedPort(6379) |
| 25 | } |
| 26 | }) |
| 27 | }).pipe( |
| 28 | Effect.catchCause(() => Effect.fail(new PersistedCacheTest.TransientError())) |
| 29 | ) |
| 30 | ) |
| 31 | |
| 32 | PersistedCacheTest.suite( |
| 33 | "NodeRedis", |
| 34 | Persistence.layerRedis.pipe(Layer.provide(RedisLayer)) |
| 35 | ) |
| 36 | |
| 37 | BackingPersistenceTest.suite( |
| 38 | "NodeRedis", |
| 39 | Persistence.layerBackingRedis.pipe(Layer.provide(RedisLayer)) |
| 40 | ) |
| 41 | |
| 42 | PersistedQueueTest.suite( |
| 43 | "NodeRedis", |
| 44 | // short intervals so the periodic reset runs while the suite's takes are |
| 45 | // in flight |
| 46 | PersistedQueue.layerStoreRedis({ |
| 47 | pollInterval: "50 millis", |
| 48 | lockRefreshInterval: "100 millis" |
| 49 | }).pipe(Layer.provide(RedisLayer)) |
| 50 | ) |
| 51 | |
| 52 | RateLimiterTest.suite( |
| 53 | "NodeRedis", |
| 54 | RateLimiter.layerStoreRedis().pipe(Layer.provide(RedisLayer)) |
| 55 | ) |
| 56 | |
| 57 | CircuitBreakerTest.suite( |
| 58 | "NodeRedis", |
| 59 | (options) => CircuitBreaker.layerStoreRedis(options).pipe(Layer.provide(RedisLayer)), |
| 60 | // the store reads Redis TIME, so time only moves on the wall clock |
| 61 | (duration) => TestClock.withLive(Effect.sleep(duration)) |
| 62 | ) |
| 63 | |
| 64 | CircuitBreakerRedisTest.suite("NodeRedis", RedisLayer) |
| 65 | |
| 66 | it.layer(RateLimiter.layerStoreRedis().pipe(Layer.provideMerge(RedisLayer)), { |
| 67 | timeout: "30 seconds", |
| 68 | concurrent: false |
| 69 | })( |
| 70 | "RateLimiter token-bucket storage (NodeRedis)", |
| 71 | (it) => { |
| 72 | it.effect("returns the persisted balance after accumulating fractional costs", () => |
| 73 | Effect.gen(function*() { |
| 74 | const redis = yield* Redis.Redis |
| 75 | const key = "fractional-persisted-balance" |
| 76 | const results = yield* RateLimiterTest.consumeFractionalCosts(key) |
| 77 | const stored = Number(yield* redis.send<string>("GET", `ratelimiter:${key}`)) |
| 78 | |
| 79 | assert.strictEqual(stored, 3.9999999999999996) |
| 80 | assert.strictEqual(results[results.length - 1].remaining, stored) |
| 81 | })) |
| 82 | |
| 83 | it.effect( |
| 84 | "does not restore capacity early after a fractional token cost", |
| 85 | () => |
| 86 | Effect.gen(function*() { |
| 87 | const redis = yield* Redis.Redis |
| 88 | const store = yield* RateLimiter.RateLimiterStore |
| 89 | const memory = yield* RateLimiter.RateLimiterStore.pipe(Effect.provide(RateLimiter.layerStoreMemory)) |
| 90 | const opts = { |
| 91 | key: "fractional-cost-expiry", |
| 92 | limit: 5, |
| 93 | refillRate: Duration.seconds(4), |
| 94 | allowOverflow: false |
| 95 | } |
| 96 | const key = `ratelimiter:${opts.key}` |
| 97 | const refillKey = `${key}:refill` |
| 98 | yield* memory.tokenBucket({ ...opts, tokens: 0.5 }) |
| 99 | yield* store.tokenBucket({ ...opts, tokens: 0.5 }) |
| 100 | const refillAt = Number(yield* redis.send<string>("GET", refillKey)) |
| 101 | assert.isAbove(yield* redis.send<number>("PTTL", key), 0) |
| 102 | |
| 103 | // Use wall time: the reported bug expires these keys after 2s, before the 4s refill. |
| 104 | yield* Effect.sleep("2500 millis") |
| 105 | const beforeRefill = { |
| 106 | keys: yield* redis.send<number>("EXISTS", key, refillKey), |
| 107 | tokens: yield* redis.send<string | null>("GET", key), |
| 108 | memoryRemaining: (yield* memory.tokenBucket({ ...opts, tokens: 0 }))[0], |
| 109 | redisRemaining: (yield* store.tokenBucket({ ...opts, tokens: 0 }))[0] |
| 110 | } |
| 111 | const elapsed = (yield* Clock.currentTimeMillis) - refillAt |
| 112 | assert.isAtLeast(elapsed, 2_500) |
| 113 | assert.isBelow(elapsed, 4_000) |
| 114 | |
| 115 | yield* Effect.sleep(4_100 - elapsed) |
| 116 | assert.strictEqual((yield* memory.tokenBucket({ ...opts, tokens: 0 }))[0], 5) |
| 117 | assert.strictEqual((yield* store.tokenBucket({ ...opts, tokens: 0 }))[0], 5) |
| 118 | assert.deepStrictEqual(beforeRefill, { keys: 2, tokens: "4.5", memoryRemaining: 4.5, redisRemaining: 4.5 }) |
| 119 | }).pipe(TestClock.withLive), |
| 120 | 10_000 |
| 121 | ) |
| 122 | |
| 123 | for (const onExceeded of ["fail", "delay"] as const) { |
| 124 | it.effect(`${onExceeded}: restarts the refill interval after both Redis keys expire`, () => { |
| 125 | const key = `timing-expired-${onExceeded}` |
| 126 | const redisKey = `ratelimiter:${key}` |
| 127 | const refillKey = `${redisKey}:refill` |
| 128 | // TestClock does not advance Redis's expiry clock, so expire the keys by hand. |
| 129 | const idle = Effect.gen(function*() { |
| 130 | const redis = yield* Redis.Redis |
| 131 | yield* TestClock.adjust("359 seconds") |
| 132 | assert.strictEqual(yield* redis.send<number>("EXISTS", redisKey, refillKey), 2) |
| 133 | assert.strictEqual(yield* redis.send<number>("PEXPIRE", redisKey, "0"), 1) |
| 134 | assert.strictEqual(yield* redis.send<number>("PEXPIRE", refillKey, "0"), 1) |
| 135 | assert.strictEqual(yield* redis.send<number>("EXISTS", redisKey, refillKey), 0) |
| 136 | }) |
| 137 | return RateLimiterTest.restartsInterval(key, onExceeded, 5, idle) |
| 138 | }) |
| 139 | } |
| 140 | } |
| 141 | ) |
| 142 | |
| 143 | const PersistedQueueRedisLayer = Layer.mergeAll( |
| 144 | RedisLayer, |
| 145 | PersistedQueue.layer.pipe( |
| 146 | Layer.provideMerge( |
| 147 | PersistedQueue.layerStoreRedis().pipe(Layer.provide(RedisLayer)) |
| 148 | ) |
| 149 | ) |
| 150 | ) |
| 151 | |
| 152 | it.effect("fails the initial connection by default", () => |
| 153 | Effect.gen(function*() { |
| 154 | const port = yield* closedPort |
| 155 | const error = yield* Layer.build(NodeRedis.layer({ |
| 156 | socket: { |
| 157 | host: "127.0.0.1", |
| 158 | port |
| 159 | } |
| 160 | })).pipe(Effect.flip) |
| 161 | |
| 162 | assert.instanceOf(error, Redis.RedisError) |
| 163 | })) |
| 164 | |
| 165 | it.layer(PersistedQueueRedisLayer, { timeout: "30 seconds" })( |
| 166 | "PersistedQueue (NodeRedis)", |
| 167 | (it) => { |
| 168 | it.effect("uses the node-redis protocol default", () => |
| 169 | Effect.gen(function*() { |
| 170 | const redis = yield* NodeRedis.NodeRedis |
| 171 | assert.strictEqual(redis.client.options.RESP, undefined) |
| 172 | })) |
| 173 | |
| 174 | it.effect("receives published messages", () => |
| 175 | Effect.gen(function*() { |
| 176 | const redis = yield* Redis.Redis |
| 177 | const subscription = yield* redis.subscribe("effect-test") |
| 178 | |
| 179 | yield* redis.send("PUBLISH", "effect-test", "hello") |
| 180 | |
| 181 | assert.deepStrictEqual(yield* Queue.take(subscription), { |
| 182 | channel: "effect-test", |
| 183 | message: "hello" |
| 184 | }) |
| 185 | })) |
| 186 | |
| 187 | it.effect("runs blocking reads on a reserved connection without delaying shared commands", () => |
| 188 | Effect.gen(function*() { |
| 189 | const redis = yield* Redis.Redis |
| 190 | const read = yield* redis.reserve.pipe( |
| 191 | Effect.flatMap((connection) => |
| 192 | connection.send("XREAD", "BLOCK", "5000", "STREAMS", "effect-test-reserve", "$") |
| 193 | ), |
| 194 | Effect.scoped, |
| 195 | Effect.forkChild |
| 196 | ) |
| 197 | // XREAD with "$" only sees entries added after it starts blocking |
| 198 | yield* redis.send<string>("CLIENT", "LIST").pipe( |
| 199 | Effect.repeat({ until: (clients) => clients.includes("cmd=xread") }) |
| 200 | ) |
| 201 | yield* redis.send("XADD", "effect-test-reserve", "*", "key", "value") |
| 202 | |
| 203 | assert.include(JSON.stringify(yield* Fiber.join(read)), "value") |
| 204 | })) |
| 205 | |
| 206 | // The shared PersistedQueue suite can only assert that exhausted elements |
| 207 | // are no longer delivered, which is also true if they are silently |
| 208 | // dropped. There is no public API for reading failed elements, so |
| 209 | // verifying they are preserved in the dead-letter list requires |
| 210 | // inspecting Redis directly. |
| 211 | it.effect("moves exhausted elements to the failed list", () => |
| 212 | Effect.gen(function*() { |
| 213 | const redis = yield* NodeRedis.NodeRedis |
| 214 | const queueName = "test-redis-failed" |
| 215 | |
| 216 | const queue = yield* PersistedQueue.make({ |
| 217 | name: queueName, |
| 218 | schema: RedisItem, |
| 219 | maxAttempts: 1 |
| 220 | }) |
| 221 | const id = yield* queue.offer({ n: 42 }) |
| 222 | const error = yield* queue.take(() => Effect.fail("boom")).pipe(Effect.flip) |
| 223 | assert.strictEqual(error, "boom") |
| 224 | |
| 225 | const failed = yield* redis.use((client) => client.lRange(`effectq:${queueName}:failed`, 0, -1)) |
| 226 | assert.strictEqual(failed.length, 1) |
| 227 | const failedItem = JSON.parse(failed[0]) |
| 228 | assert.strictEqual(failedItem.id, id) |
| 229 | assert.deepStrictEqual(failedItem.element, { n: 42 }) |
| 230 | assert.strictEqual(failedItem.attempts, 1) |
| 231 | |
| 232 | const pending = yield* redis.use((client) => client.hLen(`effectq:${queueName}:pending`)) |
| 233 | assert.strictEqual(pending, 0) |
| 234 | })) |
| 235 | |
| 236 | it.effect("success after lease expiry but before reset is not redelivered", () => |
| 237 | Effect.gen(function*() { |
| 238 | // Neither refresh nor reset can run again before this handler succeeds. |
| 239 | const queue = yield* makeOwnershipQueue("effectq-late-success:", "1 hour") |
| 240 | yield* queue.offer({ n: 42 }, { id: "order-1" }) |
| 241 | yield* queue.take((_value, { attempts }) => |
| 242 | Effect.gen(function*() { |
| 243 | assert.strictEqual(attempts, 1) |
| 244 | // Redis owns the lease clock; let it expire without another claimant. |
| 245 | yield* Effect.sleep("1100 millis") |
| 246 | }) |
| 247 | ) |
| 248 | |
| 249 | // Only now allow recovery to run: a recorded success must leave no work. |
| 250 | const recovery = yield* makeOwnershipQueue("effectq-late-success:", "100 millis") |
| 251 | const duplicate = yield* recovery.take((_value, metadata) => Effect.succeed(metadata)).pipe(Effect.forkScoped) |
| 252 | yield* Effect.sleep("1500 millis") |
| 253 | assert.isUndefined(duplicate.pollUnsafe()) |
| 254 | }).pipe(TestClock.withLive), { timeout: 20000 }) |
| 255 | |
| 256 | // it.layer shares its TestClock across concurrent tests. Each paused worker |
| 257 | // needs a private clock so another test cannot renew its lease before recovery. |
| 258 | it.effect("renews an expired lease before recovery revokes ownership", () => |
| 259 | Effect.gen(function*() { |
| 260 | const queue = yield* makeOwnershipQueue("effectq-late-renewal:", "100 millis") |
| 261 | // Stagger the timers: renewal is due at 100ms, reset at 150ms. |
| 262 | yield* TestClock.adjust("50 millis") |
| 263 | yield* queue.offer({ n: 42 }, { id: "order-1" }) |
| 264 | const taken = Latch.makeUnsafe() |
| 265 | const finish = Latch.makeUnsafe() |
| 266 | const claim = yield* queue.take(() => taken.open.pipe(Effect.andThen(finish.await))).pipe(Effect.forkScoped) |
| 267 | yield* taken.await |
| 268 | yield* Effect.sleep("1100 millis").pipe(TestClock.withLive) |
| 269 | // Only renewal runs, after Redis has expired the lease. Give its I/O |
| 270 | // time to complete before introducing a recovery worker. |
| 271 | yield* TestClock.adjust("50 millis") |
| 272 | yield* Effect.sleep("100 millis").pipe(TestClock.withLive) |
| 273 | const recovery = yield* makeOwnershipQueue("effectq-late-renewal:", "100 millis").pipe(TestClock.withLive) |
| 274 | const duplicate = yield* recovery.take((_value, metadata) => Effect.succeed(metadata)).pipe( |
| 275 | TestClock.withLive, |
| 276 | Effect.forkScoped |
| 277 | ) |
| 278 | yield* Effect.sleep("400 millis").pipe(TestClock.withLive) |
| 279 | assert.isUndefined(duplicate.pollUnsafe()) |
| 280 | yield* finish.open |
| 281 | yield* Fiber.join(claim) |
| 282 | yield* Effect.sleep("1500 millis").pipe(TestClock.withLive) |
| 283 | assert.isUndefined(duplicate.pollUnsafe()) |
| 284 | }).pipe(Effect.provide(TestClock.layer())), { timeout: 20000 }) |
| 285 | |
| 286 | it.effect("revoked ownership cannot settle recovered work before reacquisition", () => |
| 287 | Effect.gen(function*() { |
| 288 | const prefix = "effectq-revoked-owner:" |
| 289 | // Store construction pauses renewal. Takes use live time so polling |
| 290 | // and reset keep running even while this test's clock is stopped. |
| 291 | const queue = yield* makeOwnershipQueue(prefix, "100 millis") |
| 292 | yield* queue.offer({ n: 42 }, { id: "order-1" }) |
| 293 | const taken = Latch.makeUnsafe() |
| 294 | const finish = Latch.makeUnsafe() |
| 295 | const stale = yield* queue.take(() => taken.open.pipe(Effect.andThen(finish.await))).pipe( |
| 296 | TestClock.withLive, |
| 297 | Effect.forkScoped |
| 298 | ) |
| 299 | yield* taken.await |
| 300 | // Reset recovers the expired lease, but no taker can acquire it yet. |
| 301 | yield* Effect.sleep("1500 millis").pipe(TestClock.withLive) |
| 302 | yield* TestClock.adjust("100 millis") |
| 303 | yield* Effect.sleep("100 millis").pipe(TestClock.withLive) |
| 304 | yield* finish.open |
| 305 | yield* Fiber.join(stale) |
| 306 | const recovered = yield* queue.take((value, metadata) => Effect.succeed({ value, ...metadata })).pipe( |
| 307 | TestClock.withLive |
| 308 | ) |
| 309 | assert.deepStrictEqual(recovered, { value: { n: 42 }, id: "order-1", attempts: 2 }) |
| 310 | const duplicate = yield* queue.take(Effect.succeed).pipe(TestClock.withLive, Effect.forkScoped) |
| 311 | yield* Effect.sleep("1500 millis").pipe(TestClock.withLive) |
| 312 | assert.isUndefined(duplicate.pollUnsafe()) |
| 313 | }).pipe(Effect.provide(TestClock.layer())), { timeout: 20000 }) |
| 314 | |
| 315 | it.effect( |
| 316 | "stale interruption does not release a replacement or decrement its attempts", |
| 317 | () => |
| 318 | Effect.gen(function*() { |
| 319 | const a = yield* makeOwnershipQueue("effectq-stale-interrupt:", "1 hour") |
| 320 | const b = yield* makeOwnershipQueue("effectq-stale-interrupt:", "100 millis") |
| 321 | yield* a.offer({ n: 42 }, { id: "order-1" }) |
| 322 | const taken = Latch.makeUnsafe() |
| 323 | const stale = yield* a.take(() => taken.open.pipe(Effect.andThen(Effect.never))).pipe(Effect.forkScoped) |
| 324 | yield* taken.await |
| 325 | const replaced = Latch.makeUnsafe() |
| 326 | const replacement = yield* b.take((_value, { attempts }) => |
| 327 | Effect.gen(function*() { |
| 328 | assert.strictEqual(attempts, 2) |
| 329 | yield* replaced.open |
| 330 | return yield* Effect.never |
| 331 | }) |
| 332 | ).pipe(Effect.forkScoped) |
| 333 | yield* replaced.await |
| 334 | yield* Fiber.interrupt(stale) |
| 335 | const duplicate = yield* b.take((_value, metadata) => Effect.succeed(metadata)).pipe(Effect.forkScoped) |
| 336 | yield* Effect.sleep("1500 millis") |
| 337 | assert.isUndefined(duplicate.pollUnsafe()) |
| 338 | yield* Fiber.interrupt(replacement) |
| 339 | assert.strictEqual((yield* Fiber.join(duplicate)).attempts, 2) |
| 340 | }).pipe(TestClock.withLive), |
| 341 | { timeout: 20000 } |
| 342 | ) |
| 343 | |
| 344 | it.effect( |
| 345 | "same-store stale success preserves the replacement and its refresh registration", |
| 346 | () => |
| 347 | Effect.gen(function*() { |
| 348 | const queue = yield* makeOwnershipQueue("effectq-same-store:", "100 millis") |
| 349 | yield* queue.offer({ n: 42 }, { id: "order-1" }) |
| 350 | const aTaken = Latch.makeUnsafe() |
| 351 | const releaseA = Latch.makeUnsafe() |
| 352 | const stale = yield* queue.take(() => aTaken.open.pipe(Effect.andThen(releaseA.await))).pipe( |
| 353 | TestClock.withLive, |
| 354 | Effect.forkScoped |
| 355 | ) |
| 356 | yield* aTaken.await |
| 357 | // Poll/reset use live time; only renewal stays paused until B takes. |
| 358 | const bTaken = Latch.makeUnsafe() |
| 359 | const replacement = yield* queue.take((_value, { attempts }) => |
| 360 | Effect.gen(function*() { |
| 361 | assert.strictEqual(attempts, 2) |
| 362 | yield* bTaken.open |
| 363 | return yield* Effect.never |
| 364 | }) |
| 365 | ).pipe(TestClock.withLive, Effect.forkScoped) |
| 366 | yield* bTaken.await |
| 367 | // Only the replacement can now renew. Keep ticking while Redis I/O |
| 368 | // completes, including while testing the old finalizer below. |
| 369 | yield* Effect.sleep("100 millis").pipe( |
| 370 | TestClock.withLive, |
| 371 | Effect.andThen(TestClock.adjust("100 millis")), |
| 372 | Effect.forever, |
| 373 | Effect.forkScoped |
| 374 | ) |
| 375 | yield* releaseA.open |
| 376 | yield* Fiber.join(stale) |
| 377 | const duplicate = yield* queue.take((_value, metadata) => Effect.succeed(metadata)).pipe( |
| 378 | TestClock.withLive, |
| 379 | Effect.forkScoped |
| 380 | ) |
| 381 | // The old finalizer must not unregister the new acquisition's refresh. |
| 382 | yield* Effect.sleep("1500 millis").pipe(TestClock.withLive) |
| 383 | assert.isUndefined(duplicate.pollUnsafe()) |
| 384 | yield* Fiber.interrupt(replacement) |
| 385 | assert.strictEqual((yield* Fiber.join(duplicate)).attempts, 2) |
| 386 | }).pipe(Effect.provide(TestClock.layer())), |
| 387 | { timeout: 20000 } |
| 388 | ) |
| 389 | |
| 390 | it.effect("recovers elements from crashed workers", () => |
| 391 | Effect.gen(function*() { |
| 392 | const prefix = "effectq-crash:" |
| 393 | const store = yield* PersistedQueue.makeStoreRedis({ |
| 394 | prefix, |
| 395 | pollInterval: "50 millis", |
| 396 | lockRefreshInterval: "100 millis", |
| 397 | lockExpiration: "1 second" |
| 398 | }) |
| 399 | const factory = yield* PersistedQueue.makeFactory.pipe( |
| 400 | Effect.provideService(PersistedQueue.PersistedQueueStore, store) |
| 401 | ) |
| 402 | const queue = yield* factory.make({ name: "crash-recovery", schema: RedisItem }) |
| 403 | const redis = yield* Redis.Redis |
| 404 | |
| 405 | // simulate a worker that claimed the element and then crashed: the |
| 406 | // element sits in the pending hash with a consumed attempt and no lock |
| 407 | yield* redis.send( |
| 408 | "HSET", |
| 409 | `${prefix}crash-recovery:pending`, |
| 410 | "crashed", |
| 411 | JSON.stringify({ id: "crashed", element: { n: 1 } }) |
| 412 | ) |
| 413 | yield* redis.send("HSET", `${prefix}crash-recovery:attempts`, "crashed", "1") |
| 414 | |
| 415 | const result = yield* queue.take((value, metadata) => Effect.succeed([value.n, metadata.attempts])) |
| 416 | assert.deepStrictEqual(result, [1, 2]) |
| 417 | }).pipe(TestClock.withLive), { timeout: 20000 }) |
| 418 | |
| 419 | it.effect("dead-letters elements from workers that crashed on the final attempt", () => |
| 420 | Effect.gen(function*() { |
| 421 | const prefix = "effectq-crash-exhausted:" |
| 422 | const store = yield* PersistedQueue.makeStoreRedis({ |
| 423 | prefix, |
| 424 | pollInterval: "50 millis", |
| 425 | lockRefreshInterval: "100 millis", |
| 426 | lockExpiration: "1 second" |
| 427 | }) |
| 428 | const factory = yield* PersistedQueue.makeFactory.pipe( |
| 429 | Effect.provideService(PersistedQueue.PersistedQueueStore, store) |
| 430 | ) |
| 431 | const queue = yield* factory.make({ name: "crash-exhausted", schema: RedisItem, maxAttempts: 1 }) |
| 432 | const redis = yield* Redis.Redis |
| 433 | |
| 434 | // the final attempt was claimed by a worker that crashed, so no |
| 435 | // finalizer will ever settle this element |
| 436 | yield* redis.send( |
| 437 | "HSET", |
| 438 | `${prefix}crash-exhausted:pending`, |
| 439 | "crashed", |
| 440 | JSON.stringify({ id: "crashed", element: { n: 1 } }) |
| 441 | ) |
| 442 | yield* redis.send("HSET", `${prefix}crash-exhausted:attempts`, "crashed", "1") |
| 443 | |
| 444 | // an active taker runs the periodic reset that dead-letters such |
| 445 | // elements instead of redelivering them |
| 446 | const fiber = yield* queue.take(Effect.succeed).pipe(Effect.forkScoped) |
| 447 | yield* Effect.sleep(1000) |
| 448 | |
| 449 | const failed = yield* redis.send<Array<string>>("LRANGE", `${prefix}crash-exhausted:failed`, "0", "-1") |
| 450 | assert.strictEqual(failed.length, 1) |
| 451 | const failedItem = JSON.parse(failed[0]) |
| 452 | assert.strictEqual(failedItem.id, "crashed") |
| 453 | assert.deepStrictEqual(failedItem.element, { n: 1 }) |
| 454 | assert.strictEqual(failedItem.attempts, 1) |
| 455 | assert.include(failedItem.lastFailure, "Lock expired after final attempt") |
| 456 | |
| 457 | const pending = yield* redis.send<number>("HLEN", `${prefix}crash-exhausted:pending`) |
| 458 | assert.strictEqual(Number(pending), 0) |
| 459 | assert.isUndefined(fiber.pollUnsafe()) |
| 460 | }).pipe(TestClock.withLive), { timeout: 20000 }) |
| 461 | } |
| 462 | ) |
| 463 | |
| 464 | const makeOwnershipQueue = Effect.fnUntraced(function*(prefix: string, lockRefreshInterval: Duration.Input) { |
| 465 | const store = yield* PersistedQueue.makeStoreRedis({ |
| 466 | prefix, |
| 467 | pollInterval: "50 millis", |
| 468 | lockExpiration: "1 second", |
| 469 | lockRefreshInterval |
| 470 | }) |
| 471 | const factory = yield* PersistedQueue.makeFactory.pipe( |
| 472 | Effect.provideService(PersistedQueue.PersistedQueueStore, store) |
| 473 | ) |
| 474 | return yield* factory.make({ name: "ownership", schema: RedisItem }) |
| 475 | }) |
| 476 | |
| 477 | const RedisItem = Schema.Struct({ |
| 478 | n: Schema.Number |
| 479 | }) |
| 480 | |
| 481 | const closedPort = Effect.promise( |
| 482 | () => |
| 483 | new Promise<number>((resolve, reject) => { |
| 484 | const server = createServer() |
| 485 | server.once("error", reject) |
| 486 | server.listen(0, "127.0.0.1", () => { |
| 487 | const address = server.address() |
| 488 | if (address === null || typeof address === "string") { |
| 489 | server.close() |
| 490 | reject(new Error("Could not allocate a TCP port")) |
| 491 | return |
| 492 | } |
| 493 | server.close((error) => { |
| 494 | if (error) { |
| 495 | reject(error) |
| 496 | } else { |
| 497 | resolve(address.port) |
| 498 | } |
| 499 | }) |
| 500 | }) |
| 501 | }) |
| 502 | ) |