The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,148 lines 45.8 KB
Raw
1 import { NodeCrypto, NodeFileSystem } from "@effect/platform-node"
2 import { PgClient } from "@effect/sql-pg"
3 import { SqliteClient } from "@effect/sql-sqlite-node"
4 import { assert, describe, expect, it } from "@effect/vitest"
5 import {
6 Cause,
7 Clock,
8 Duration,
9 Effect,
10 Exit,
11 Fiber,
12 FileSystem,
13 Latch,
14 Layer,
15 Logger,
16 Redacted,
17 Result,
18 Schedule
19 } from "effect"
20 import {
21 ClusterError,
22 MessageStorage,
23 Runner,
24 RunnerAddress,
25 RunnerStorage,
26 ShardId,
27 ShardingConfig,
28 Snowflake,
29 SqlMessageStorage,
30 SqlRunnerStorage
31 } from "effect/cluster"
32 import { Migrator, SqlClient, type SqlConnection, SqlError } from "effect/sql"
33 import { TestClock } from "effect/testing"
34 import { MysqlContainer } from "../fixtures/mysql2-utils.ts"
35 import { PgContainer } from "../fixtures/pg-utils.ts"
36
37 const appRole = "cluster_app"
38
39 const AppClient = Layer.unwrap(
40 Effect.gen(function*() {
41 const container = yield* PgContainer
42 return PgClient.layer({
43 host: container.getHost(),
44 port: container.getPort(),
45 database: container.getDatabase(),
46 username: appRole,
47 password: Redacted.make(appRole)
48 })
49 })
50 )
51
52 const asApp = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
53 Effect.provide(effect, [AppClient, ShardingConfig.layerDefaults, NodeCrypto.layerDigest, Snowflake.layerGenerator])
54
55 const assertMigrationDefect = (exit: Exit.Exit<unknown, unknown>) => {
56 assert(Exit.isFailure(exit))
57 assert.isFalse(Cause.hasFails(exit.cause))
58 assert.isFalse(Cause.hasInterrupts(exit.cause))
59 const error = Cause.findDefect(exit.cause)
60 assert(Result.isSuccess(error))
61 assert(error.success instanceof Migrator.MigrationError)
62 assert.strictEqual(error.success.kind, "Failed")
63 assertAuthorizationError(error.success.cause)
64 }
65
66 const assertAuthorizationError = (error: unknown) => {
67 assert(SqlError.isSqlError(error), `expected a SqlError, got ${error}`)
68 assert.strictEqual(error.reason._tag, "AuthorizationError")
69 }
70
71 const TestLayer = Layer.effectDiscard(Effect.gen(function*() {
72 const sql = yield* SqlClient.SqlClient
73 yield* sql.unsafe(`CREATE ROLE ${appRole} LOGIN PASSWORD '${appRole}'`)
74 yield* sql.unsafe(`REVOKE CREATE ON SCHEMA public FROM PUBLIC`)
75 yield* sql.unsafe(`GRANT USAGE ON SCHEMA public TO ${appRole}`)
76 })).pipe(Layer.provideMerge(PgContainer.layerClient), Layer.provideMerge(PgContainer.layer))
77
78 describe("cluster SQL storage with a DML-only role", () => {
79 // The tests grant on and provision tables in the same schema.
80 it.layer(TestLayer, { concurrent: false, timeout: 120_000 })("pg", (it) => {
81 it.effect("starts from tables migrated by an owner connection", () =>
82 Effect.gen(function*() {
83 yield* Effect.scoped(Layer.build(Layer.mergeAll(
84 SqlMessageStorage.layerMigrations({}),
85 SqlRunnerStorage.layerMigrations({})
86 )))
87 const sql = yield* SqlClient.SqlClient
88 yield* sql.unsafe(`GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO ${appRole}`)
89 yield* sql.unsafe(`GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO ${appRole}`)
90
91 yield* asApp(
92 Effect.gen(function*() {
93 const runners = yield* RunnerStorage.RunnerStorage
94 const runner = Runner.make({
95 address: RunnerAddress.make("localhost", 1234),
96 groups: ["default"],
97 weight: 1
98 })
99 yield* runners.register(runner, true)
100 expect(yield* runners.getRunners).toEqual([[runner, true]])
101
102 const messages = yield* MessageStorage.MessageStorage
103 expect(yield* messages.unprocessedMessages([ShardId.make("default", 1)])).toEqual([])
104 }).pipe(
105 Effect.provide(Layer.mergeAll(SqlMessageStorage.layerStorage({}), SqlRunnerStorage.layerStorage({})))
106 )
107 )
108 }))
109
110 it.effect("surfaces errors inside a migration as defects", () =>
111 Effect.gen(function*() {
112 // Empty history tables the role can use, so the migrator gets past its
113 // own table and fails inside the first migration.
114 for (const table of ["inside_migrations", "inside_runner_migrations"]) {
115 yield* Migrator.make({})({ loader: Migrator.fromRecord({}), table })
116 const sql = yield* SqlClient.SqlClient
117 yield* sql.unsafe(`GRANT SELECT, INSERT, UPDATE, DELETE ON ${table} TO ${appRole}`)
118 }
119
120 assertMigrationDefect(
121 yield* asApp(Effect.scoped(Layer.build(SqlMessageStorage.layerMigrations({ prefix: "inside" })))).pipe(
122 Effect.exit
123 )
124 )
125 assertMigrationDefect(
126 yield* asApp(Effect.scoped(Layer.build(SqlRunnerStorage.layerWith({ prefix: "inside" })))).pipe(Effect.exit)
127 )
128 }).pipe(Effect.timeout("20 seconds"), TestClock.withLive), 30_000)
129 })
130 })
131
132 const StorageLayer = SqlRunnerStorage.layer
133
134 // Allow for CI latency in healthy PostgreSQL queries.
135 const lockOperationInterval = 1000
136 const partitionConfig = {
137 // Keep expiration / 3 above the operation deadline.
138 shardLockExpiration: lockOperationInterval * 10,
139 shardLockRefreshInterval: lockOperationInterval
140 }
141
142 describe("SqlRunnerStorage", () => {
143 // These tests each get an empty database on one shared container, so
144 // container startup counts against the hook timeout, not each test's.
145 it.layer(PgContainer.layer, { timeout: 120_000 })((it) => {
146 it.effect("bounds shard lock operations and rebuilds an unresponsive reserved connection", () => {
147 const partitioned = makePartitionState()
148 const layer = StorageLayer.pipe(
149 Layer.provideMerge(blackholeReservedConnection(partitioned, true)),
150 Layer.provide(ShardingConfig.layer(partitionConfig))
151 )
152
153 return Effect.gen(function*() {
154 const storage = yield* RunnerStorage.RunnerStorage
155 const runner = Runner.make({
156 address: runnerAddress1,
157 groups: ["default"],
158 weight: 1
159 })
160 const shards = [ShardId.make("default", 1)]
161
162 yield* storage.register(runner, true)
163 yield* storage.acquire(runnerAddress1, shards)
164 partitionConnection(partitioned)
165
166 const expectDeadline = Effect.fnUntraced(function*(operation: Effect.Effect<unknown, unknown>) {
167 const [elapsed, exit] = yield* operation.pipe(
168 Effect.exit,
169 Effect.timed
170 )
171 assert(Exit.isFailure(exit))
172 const error = Cause.squash(exit.cause)
173 assert(error instanceof ClusterError.PersistenceError)
174 assert.isBelow(Duration.toMillis(elapsed), lockOperationInterval * 5)
175 yield* Effect.sleep(20)
176 })
177
178 const expectRecovery = Effect.fnUntraced(function*() {
179 restoreConnection(partitioned)
180 // the replacement session lost the previous session's lock, so recovery reacquires it
181 expect(
182 yield* storage.acquire(runnerAddress1, shards).pipe(
183 Effect.retry({ times: 5, schedule: Schedule.spaced(20) })
184 )
185 ).toEqual(shards)
186 })
187
188 yield* expectDeadline(storage.refresh(runnerAddress1, shards))
189 yield* expectDeadline(storage.refresh(runnerAddress1, shards))
190 yield* expectDeadline(storage.refresh(runnerAddress1, shards))
191
192 assert.isAtLeast(partitioned.interruptedQueries, 1)
193 assert.isAtMost(partitioned.maxActiveQueries, 1)
194
195 yield* expectRecovery()
196
197 partitionConnection(partitioned)
198 yield* expectDeadline(storage.acquire(runnerAddress1, [ShardId.make("default", 2)]))
199 yield* expectRecovery()
200
201 partitionConnection(partitioned)
202 yield* expectDeadline(storage.release(runnerAddress1, shards[0]))
203 yield* expectRecovery()
204 yield* storage.release(runnerAddress1, shards[0])
205
206 assert.strictEqual(partitioned.activeQueries, 0)
207 }).pipe(
208 Effect.timeoutOrElse({
209 duration: 30_000,
210 orElse: () =>
211 Effect.die(
212 `timed out exercising shard lock rebuilds (${partitionDiagnostics(partitioned)})`
213 )
214 }),
215 // Ensure layer teardown cannot mask a body failure with Vitest's timeout.
216 Effect.ensuring(Effect.sync(() => {
217 restoreConnection(partitioned)
218 })),
219 Effect.provide(layer),
220 TestClock.withLive
221 )
222 }, 60_000)
223
224 it.effect("empty liveness probe bypasses a wedged reserved connection", () => {
225 const partitioned = makePartitionState()
226 const layer = StorageLayer.pipe(
227 Layer.provideMerge(blackholeReservedConnection(partitioned, true)),
228 Layer.provide(ShardingConfig.layer(partitionConfig))
229 )
230
231 return Effect.gen(function*() {
232 const storage = yield* RunnerStorage.RunnerStorage
233 const runner = Runner.make({
234 address: runnerAddress1,
235 groups: ["default"],
236 weight: 1
237 })
238 const shards = [ShardId.make("default", 1)]
239
240 yield* storage.register(runner, true)
241 yield* storage.acquire(runnerAddress1, shards)
242 partitionConnection(partitioned)
243
244 // shard lock operations are stuck behind the wedged reserved connection
245 const exit = yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
246 assert(Exit.isFailure(exit))
247
248 // the liveness probe runs on the shared pool, so it must succeed while
249 // the reserved connection stays wedged
250 expect(yield* storage.refresh(runnerAddress1, [])).toEqual([])
251
252 restoreConnection(partitioned)
253 // the replacement session lost the previous session's lock, so recovery reacquires it
254 expect(
255 yield* storage.acquire(runnerAddress1, shards).pipe(
256 Effect.retry({ times: 20, schedule: Schedule.spaced(20) })
257 )
258 ).toEqual(shards)
259 }).pipe(
260 Effect.ensuring(Effect.sync(() => {
261 restoreConnection(partitioned)
262 })),
263 Effect.provide(layer),
264 TestClock.withLive
265 )
266 }, 60_000)
267
268 it.effect("recovers when a blackholed query cannot resume after the partition clears", () => {
269 const partitioned = makePartitionState()
270 const layer = StorageLayer.pipe(
271 Layer.provideMerge(blackholeReservedConnection(partitioned, false)),
272 Layer.provide(ShardingConfig.layer({
273 shardLockDisableAdvisory: true,
274 ...partitionConfig
275 }))
276 )
277
278 return Effect.gen(function*() {
279 const storage = yield* RunnerStorage.RunnerStorage
280 const runner = Runner.make({
281 address: runnerAddress1,
282 groups: ["default"],
283 weight: 1
284 })
285 const shards = [ShardId.make("default", 1)]
286
287 yield* storage.register(runner, true)
288 yield* storage.acquire(runnerAddress1, shards)
289 partitionConnection(partitioned)
290 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
291 yield* waitUntil(() => partitioned.activeQueries === 0)
292
293 restoreConnection(partitioned)
294 expect(
295 yield* storage.refresh(runnerAddress1, shards).pipe(
296 Effect.retry({ times: 20, schedule: Schedule.spaced(20) })
297 )
298 ).toEqual(shards)
299 assert.isAtLeast(partitioned.interruptedQueries, 1)
300 assert.isAtMost(partitioned.maxActiveQueries, 1)
301 }).pipe(
302 Effect.ensuring(Effect.sync(() => {
303 restoreConnection(partitioned)
304 })),
305 Effect.provide(layer),
306 TestClock.withLive
307 )
308 }, 60_000)
309
310 it.effect("rebuilds the reserved connection again when a rebuilt connection stops responding", () => {
311 const partitioned = makePartitionState()
312 const layer = StorageLayer.pipe(
313 Layer.provideMerge(blackholeReservedConnection(partitioned, false)),
314 Layer.provide(ShardingConfig.layer(partitionConfig))
315 )
316
317 return Effect.gen(function*() {
318 const storage = yield* RunnerStorage.RunnerStorage
319 const runner = Runner.make({
320 address: runnerAddress1,
321 groups: ["default"],
322 weight: 1
323 })
324 const shards = [ShardId.make("default", 1)]
325
326 yield* storage.register(runner, true)
327 yield* storage.acquire(runnerAddress1, shards)
328
329 // a failing lock operation rebuilds the reserved connection
330 partitioned.failNextQueries = 1
331 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
332 yield* waitUntil(() => partitioned.usableConnections === 2)
333
334 // the rebuilt connection then wedges, without any lock operation
335 // succeeding in between - a further rebuild still has to be attempted
336 const reserved = partitioned.reservedConnections
337 partitionConnection(partitioned)
338 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
339 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
340 restoreConnection(partitioned)
341 yield* waitUntil(() => partitioned.reservedConnections > reserved)
342
343 // the replacement session lost the previous session's lock, so recovery reacquires it
344 expect(
345 yield* storage.acquire(runnerAddress1, shards).pipe(
346 Effect.retry({ times: 5, schedule: Schedule.spaced(20) })
347 )
348 ).toEqual(shards)
349 }).pipe(
350 Effect.ensuring(Effect.sync(() => {
351 restoreConnection(partitioned)
352 })),
353 Effect.provide(layer),
354 TestClock.withLive
355 )
356 }, 60_000)
357
358 it.effect(
359 "rebuilds the reserved connection when releasing the previous one hangs",
360 () =>
361 withStalledRelease((storage, shards, partitioned) =>
362 Effect.gen(function*() {
363 yield* Effect.sleep(lockOperationInterval * 1.5)
364
365 // the stalled release must not disable further rebuilds
366 restoreConnection(partitioned)
367 const reserved = partitioned.reservedConnections
368 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
369 yield* waitUntil(() => partitioned.reservedConnections > reserved)
370
371 expect(
372 yield* storage.refresh(runnerAddress1, shards).pipe(
373 Effect.retry({ times: 20, schedule: Schedule.spaced(20) })
374 )
375 ).toEqual(shards)
376 })
377 ),
378 60_000
379 )
380
381 it.effect(
382 "replaces a pending reserved connection once when operations waiting on it fail",
383 () =>
384 withStalledRelease((storage, shards, partitioned) =>
385 Effect.gen(function*() {
386 restoreConnection(partitioned)
387 const reserved = partitioned.reservedConnections
388
389 // both operations wait on the pending connection and time out together,
390 // but only one of them may replace it
391 const exits = yield* Effect.all([
392 storage.refresh(runnerAddress1, shards).pipe(Effect.exit),
393 storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
394 ], { concurrency: "unbounded" })
395 assert(exits.every(Exit.isFailure))
396 yield* waitUntil(() => partitioned.reservedConnections > reserved)
397 // count before retrying: a retry that misses its deadline under load
398 // rightly replaces the connection again
399 yield* Effect.sleep(lockOperationInterval / 2)
400 expect(partitioned.reservedConnections, partitionDiagnostics(partitioned)).toEqual(reserved + 1)
401 expect(
402 yield* storage.refresh(runnerAddress1, shards).pipe(
403 Effect.retry({ times: 20, schedule: Schedule.spaced(20) })
404 )
405 ).toEqual(shards)
406 })
407 ),
408 60_000
409 )
410
411 it.effect(
412 "replaces a connection that fails an operation which waited for it",
413 () =>
414 withStalledRelease((storage, shards, partitioned) =>
415 Effect.gen(function*() {
416 restoreConnection(partitioned)
417 const waiting = yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit, Effect.forkChild)
418 yield* Effect.sleep(lockOperationInterval / 10)
419
420 // the waiting operation runs on the replacement, which then fails it
421 const reserved = partitioned.reservedConnections
422 partitioned.failNextQueries = 1
423 partitioned.blockRelease = false
424 yield* waitUntil(() => partitioned.reservedConnections > reserved)
425 assert(Exit.isFailure(yield* Fiber.join(waiting)))
426
427 // the failing replacement is replaced without another operation failing on it
428 yield* waitUntil(() => partitioned.reservedConnections > reserved + 1)
429 expect(
430 yield* storage.refresh(runnerAddress1, shards).pipe(
431 Effect.retry({ times: 20, schedule: Schedule.spaced(20) })
432 )
433 ).toEqual(shards)
434 })
435 ),
436 60_000
437 )
438
439 it.effect("isolates advisory shard locks by prefix", () =>
440 Effect.gen(function*() {
441 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" })
442 const storageB = yield* SqlRunnerStorage.make({ prefix: "other" })
443 const shard = ShardId.make("default", 1)
444
445 expect(yield* storageA.acquire(runnerAddress1, [shard])).toEqual([shard])
446 expect(yield* storageB.acquire(runnerAddress2, [shard])).toEqual([shard])
447 }).pipe(
448 // Release the advisory-lock connections before the PostgreSQL layer.
449 Effect.scoped,
450 Effect.provide(PgContainer.layerClientDatabase),
451 Effect.provide(ShardingConfig.layer())
452 ), 60_000)
453
454 it.effect("excludes other storages using the same prefix", () =>
455 Effect.gen(function*() {
456 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" })
457 const storageB = yield* SqlRunnerStorage.make({ prefix: "cluster" })
458 const shard = ShardId.make("default", 1)
459
460 expect(yield* storageA.acquire(runnerAddress1, [shard])).toEqual([shard])
461 expect(yield* storageB.acquire(runnerAddress2, [shard])).toEqual([])
462 }).pipe(
463 // Release the advisory-lock connections before the PostgreSQL layer.
464 Effect.scoped,
465 Effect.provide(PgContainer.layerClientDatabase),
466 Effect.provide(ShardingConfig.layer())
467 ), 60_000)
468
469 it.effect("releases a frozen runner's advisory shard locks after shardLockExpiration", () => {
470 const shardLockExpiration = lockOperationInterval * 3
471 return Effect.gen(function*() {
472 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" })
473 const storageB = yield* SqlRunnerStorage.make({ prefix: "cluster" })
474 const shard = ShardId.make("default", 1)
475
476 expect(yield* storageA.acquire(runnerAddress1, [shard])).toEqual([shard])
477 // storageA freezes: it stops refreshing, but its lock session stays open
478 const frozenAt = yield* Clock.currentTimeMillis
479 expect(yield* storageB.acquire(runnerAddress2, [shard])).toEqual([])
480
481 yield* storageB.acquire(runnerAddress2, [shard]).pipe(
482 Effect.repeat({
483 until: (acquired) => acquired.length > 0,
484 schedule: Schedule.spaced(100)
485 }),
486 Effect.timeoutOrElse({
487 duration: shardLockExpiration * 3,
488 orElse: () => Effect.die("the frozen runner still holds its advisory shard lock")
489 })
490 )
491 const elapsed = (yield* Clock.currentTimeMillis) - frozenAt
492 assert.isAtLeast(elapsed, shardLockExpiration * 0.9)
493 assert.isBelow(elapsed, shardLockExpiration * 2)
494 }).pipe(
495 // Release the advisory-lock connections before the PostgreSQL layer.
496 Effect.scoped,
497 Effect.provide(PgContainer.layerClientDatabase),
498 Effect.provide(ShardingConfig.layer({ shardLockExpiration })),
499 TestClock.withLive
500 )
501 }, 60_000)
502
503 it.effect("acquires cleanly after the server ends the lock session of a runner without shards", () => {
504 const logs: Array<string> = []
505 const logger = Logger.make((options) => {
506 if (options.logLevel !== "Debug" && options.logLevel !== "Trace") {
507 logs.push(Logger.formatJson.log(options))
508 }
509 })
510 return Effect.gen(function*() {
511 const sql = yield* SqlClient.SqlClient
512 const storage = yield* SqlRunnerStorage.make({ prefix: "cluster" })
513 const runner = Runner.make({
514 address: runnerAddress1,
515 groups: ["default"],
516 weight: 1
517 })
518 const shard = ShardId.make("default", 1)
519
520 yield* storage.register(runner, true)
521 // Stands in for idle_session_timeout ending the idle lock session.
522 // Every other session on this database is the lock session.
523 const terminated = yield* sql`SELECT pg_terminate_backend(pid, 5000) FROM pg_stat_activity
524 WHERE datname = current_database() AND pid <> pg_backend_pid()`.values
525 expect(terminated).toEqual([[true]])
526
527 expect(yield* storage.acquire(runnerAddress1, [shard])).toEqual([shard])
528 expect(yield* storage.refresh(runnerAddress1, [shard])).toEqual([shard])
529 yield* storage.release(runnerAddress1, shard)
530 expect(logs).toEqual([])
531 }).pipe(
532 // Release the advisory-lock connections before the PostgreSQL layer.
533 Effect.scoped,
534 Effect.provide(PgContainer.layerClientDatabase),
535 Effect.provide(ShardingConfig.layer()),
536 Effect.provide(Logger.layer([logger])),
537 TestClock.withLive
538 )
539 }, 60_000)
540
541 it.effect(
542 "restores idle_session_timeout before returning the lock connection to the pool",
543 () =>
544 Effect.gen(function*() {
545 const sql = yield* SqlClient.SqlClient
546
547 const lockPid = yield* Effect.gen(function*() {
548 const storage = yield* SqlRunnerStorage.make({ prefix: "cluster" })
549 const shard = ShardId.make("default", 1)
550 expect(yield* storage.acquire(runnerAddress1, [shard])).toEqual([shard])
551 const holders = yield* sql`SELECT DISTINCT pid FROM pg_locks
552 WHERE locktype = 'advisory' AND granted
553 AND database = (SELECT oid FROM pg_database WHERE datname = current_database())`.values
554 expect(holders).toHaveLength(1)
555 return Number(holders[0][0])
556 }).pipe(Effect.scoped)
557
558 // Hold pooled connections until the pool hands out the former lock connection.
559 const [timeout, resetTimeout] = yield* Effect.gen(function*() {
560 for (let i = 0; i < 10; i++) {
561 const conn = yield* sql.reserve
562 const [[pid, setting, resetVal]] = yield* conn.executeValues(
563 "SELECT pg_backend_pid(), setting, reset_val FROM pg_settings WHERE name = 'idle_session_timeout'",
564 []
565 )
566 if (Number(pid) === lockPid) return [setting, resetVal]
567 }
568 return yield* Effect.die("the pool did not hand out the former lock connection")
569 }).pipe(Effect.scoped)
570 expect(timeout).toEqual(resetTimeout)
571 }).pipe(
572 Effect.provide(PgContainer.layerClientDatabase),
573 Effect.provide(ShardingConfig.layer())
574 ),
575 60_000
576 )
577
578 it.effect("reports a shard lost when a failed acquisition replaces the lock session holding it", () => {
579 const partitioned = makePartitionState()
580 return Effect.gen(function*() {
581 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" }).pipe(
582 Effect.provide(wrapReservedConnections(partitioned, true))
583 )
584 const storageB = yield* SqlRunnerStorage.make({ prefix: "cluster" })
585 const held = ShardId.make("default", 1)
586 const other = ShardId.make("default", 2)
587
588 expect(yield* storageA.acquire(runnerAddress1, [held])).toEqual([held])
589 // the connection is lost while acquiring another shard, taking the
590 // session's lock on the held shard with it
591 partitioned.failNextQueries = 1
592 yield* storageA.acquire(runnerAddress1, [other]).pipe(Effect.exit)
593
594 yield* acquireHandoff(storageB, held, lockOperationInterval * 10)
595 expect(yield* refreshLikeSharding(storageA, held)).toEqual([])
596 }).pipe(
597 // Release the advisory-lock connections before the PostgreSQL layer.
598 Effect.scoped,
599 Effect.provide(PgContainer.layerClientDatabase),
600 Effect.provide(ShardingConfig.layer(partitionConfig)),
601 TestClock.withLive
602 )
603 }, 60_000)
604
605 for (const acquireFirst of [true, false]) {
606 it.effect(
607 `a resumed frozen runner reports a shard lost after it was handed off (${
608 acquireFirst ? "acquire" : "refresh"
609 } first)`,
610 () => {
611 const shardLockExpiration = lockOperationInterval * 3
612 return Effect.gen(function*() {
613 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" })
614 const storageB = yield* SqlRunnerStorage.make({ prefix: "cluster" })
615 const held = ShardId.make("default", 1)
616 const other = ShardId.make("default", 2)
617
618 expect(yield* storageA.acquire(runnerAddress1, [held])).toEqual([held])
619 // storageA freezes until the shard has passed through storageB
620 yield* acquireHandoff(storageB, held, shardLockExpiration * 3)
621
622 if (acquireFirst) {
623 yield* storageA.acquire(runnerAddress1, [other]).pipe(Effect.exit)
624 }
625 expect(yield* refreshLikeSharding(storageA, held)).toEqual([])
626 }).pipe(
627 // Release the advisory-lock connections before the PostgreSQL layer.
628 Effect.scoped,
629 Effect.provide(PgContainer.layerClientDatabase),
630 Effect.provide(ShardingConfig.layer({ shardLockExpiration })),
631 TestClock.withLive
632 )
633 },
634 60_000
635 )
636 }
637
638 it.effect(
639 "pg (no advisory) does not deadlock acquisition with refresh or bulk release",
640 () =>
641 Effect.gen(function*() {
642 const sql = yield* SqlClient.SqlClient
643 const storageA = yield* SqlRunnerStorage.make({ prefix: "cluster" })
644 const storageB = yield* SqlRunnerStorage.make({ prefix: "cluster" })
645 const shards = Array.from({ length: 12 }, (_, i) => ShardId.make("default", i + 1))
646 const reversed = [...shards].reverse()
647 // Rewrite each lease in reverse so the physical row order differs from
648 // the sorted lock order.
649 const acquireShuffled = Effect.gen(function*() {
650 yield* storageA.acquire(runnerAddress1, shards)
651 for (const shard of reversed) {
652 yield* storageA.refresh(runnerAddress1, [shard])
653 }
654 })
655
656 // Slow each row write so the concurrent statements overlap.
657 yield* sql`CREATE FUNCTION delay_row() RETURNS trigger LANGUAGE plpgsql AS $$
658 BEGIN PERFORM pg_sleep(0.003); RETURN COALESCE(NEW, OLD); END $$`
659 yield* sql`CREATE TRIGGER delay_row BEFORE UPDATE OR DELETE ON cluster_locks
660 FOR EACH ROW EXECUTE FUNCTION delay_row()`
661
662 for (let i = 0; i < 8; i++) {
663 yield* acquireShuffled
664 yield* sql`UPDATE cluster_locks SET acquired_at = NOW() - INTERVAL '1 hour'`
665 yield* Effect.all([
666 storageA.refresh(runnerAddress1, shards),
667 storageB.acquire(runnerAddress2, reversed)
668 ], { concurrency: 2 })
669 yield* storageA.releaseAll(runnerAddress1)
670 yield* storageB.releaseAll(runnerAddress2)
671
672 yield* acquireShuffled
673 yield* Effect.all([
674 storageA.releaseAll(runnerAddress1),
675 storageB.acquire(runnerAddress2, reversed)
676 ], { concurrency: 2 })
677 yield* storageB.releaseAll(runnerAddress2)
678 }
679 }).pipe(
680 Effect.scoped,
681 Effect.provide(PgContainer.layerClientDatabase),
682 Effect.provide(ShardingConfig.layer({ shardLockDisableAdvisory: true })),
683 TestClock.withLive
684 ),
685 60_000
686 )
687 })
688 ;([
689 ["pg", Layer.orDie(PgContainer.layerClient)],
690 ["mysql", Layer.orDie(MysqlContainer.layerClient)],
691 ["vitess", Layer.orDie(MysqlContainer.layerClientVitess)],
692 ["sqlite", Layer.orDie(SqliteLayer)]
693 ] as const).flatMap(([label, layer]) =>
694 [
695 [label, StorageLayer.pipe(Layer.provideMerge(layer), Layer.provide(ShardingConfig.layer()))],
696 [
697 label + " (no advisory)",
698 StorageLayer.pipe(
699 Layer.provideMerge(layer),
700 Layer.provide(ShardingConfig.layer({
701 shardLockDisableAdvisory: true
702 }))
703 )
704 ]
705 ] as const
706 ).forEach(([label, layer]) => {
707 // Both tests update the same runner rows.
708 it.layer(layer, {
709 concurrent: false,
710 timeout: 60000
711 })(label, (it) => {
712 it.effect("getRunners", () =>
713 Effect.gen(function*() {
714 const storage = yield* RunnerStorage.RunnerStorage
715
716 const runner = Runner.make({
717 address: runnerAddress1,
718 groups: ["default"],
719 weight: 1
720 })
721 const machineId = yield* storage.register(runner, true)
722 yield* storage.register(runner, true)
723 expect(machineId).toEqual(1)
724 expect(yield* storage.getRunners).toEqual([[runner, true]])
725
726 yield* storage.setRunnerHealth(runnerAddress1, false)
727 expect(yield* storage.getRunners).toEqual([[runner, false]])
728
729 yield* storage.unregister(runnerAddress1)
730 expect(yield* storage.getRunners).toEqual([])
731 }), { timeout: 30_000 })
732
733 it.effect("acquireShards", () =>
734 Effect.gen(function*() {
735 const storage = yield* RunnerStorage.RunnerStorage
736
737 let acquired = yield* storage.acquire(runnerAddress1, [
738 ShardId.make("default", 1),
739 ShardId.make("default", 2),
740 ShardId.make("default", 3)
741 ])
742 expect(acquired.map((_) => _.id)).toEqual([1, 2, 3])
743 acquired = yield* storage.acquire(runnerAddress1, [
744 ShardId.make("default", 1),
745 ShardId.make("default", 2),
746 ShardId.make("default", 3)
747 ])
748 expect(acquired.map((_) => _.id)).toEqual([1, 2, 3])
749 acquired = yield* storage.acquire(runnerAddress1, [ShardId.make("default", 4)])
750 expect(acquired.map((_) => _.id)).toEqual([4])
751
752 const refreshed = yield* storage.refresh(runnerAddress1, [
753 ShardId.make("default", 1),
754 ShardId.make("default", 2),
755 ShardId.make("default", 3)
756 ])
757 expect(refreshed.map((_) => _.id).sort((a, b) => a - b)).toEqual([1, 2, 3])
758
759 // smoke test release
760 yield* storage.release(runnerAddress1, ShardId.make("default", 2))
761 }))
762
763 it.effect("releases a shard reacquired after releasing all shards", () =>
764 Effect.gen(function*() {
765 const storage = yield* SqlRunnerStorage.make({})
766 const shards = [ShardId.make("default", 12)]
767
768 yield* storage.releaseAll(runnerAddress1)
769 expect(yield* storage.acquire(runnerAddress1, shards)).toEqual(shards)
770 // Refreshing must not take the lock a second time.
771 expect(yield* storage.refresh(runnerAddress1, shards)).toEqual(shards)
772 yield* storage.release(runnerAddress1, shards[0])
773
774 const other = yield* SqlRunnerStorage.make({})
775 expect(yield* other.acquire(runnerAddress2, shards)).toEqual(shards)
776 }).pipe(
777 Effect.scoped,
778 Effect.provide(ShardingConfig.layer({ shardLockDisableAdvisory: label.endsWith("(no advisory)") }))
779 ))
780
781 it.effect("refresh reports a shard lost after it was handed off", () =>
782 Effect.gen(function*() {
783 const storage = yield* SqlRunnerStorage.make({})
784 const other = yield* SqlRunnerStorage.make({})
785 const shards = [ShardId.make("default", 11)]
786
787 expect(yield* storage.acquire(runnerAddress1, shards)).toEqual(shards)
788 yield* storage.release(runnerAddress1, shards[0])
789 expect(yield* other.acquire(runnerAddress2, shards)).toEqual(shards)
790 yield* other.release(runnerAddress2, shards[0])
791
792 // Refreshing must not take back a lock that is no longer held.
793 expect(yield* storage.refresh(runnerAddress1, shards)).toEqual([])
794 expect(yield* storage.acquire(runnerAddress1, shards)).toEqual(shards)
795 }).pipe(
796 Effect.scoped,
797 Effect.provide(ShardingConfig.layer({ shardLockDisableAdvisory: label.endsWith("(no advisory)") }))
798 ))
799
800 if (label === "pg") {
801 it.effect("runner migrations adopt existing tables without migration records", () =>
802 Effect.gen(function*() {
803 const sql = yield* SqlClient.SqlClient
804
805 // Advisory-lock deployments had no locks table or migration history.
806 yield* sql`CREATE TABLE legacy_runners (
807 machine_id SERIAL PRIMARY KEY,
808 address VARCHAR(255) NOT NULL,
809 runner TEXT NOT NULL,
810 healthy BOOLEAN NOT NULL DEFAULT TRUE,
811 last_heartbeat TIMESTAMP NOT NULL DEFAULT NOW(),
812 UNIQUE(address)
813 )`
814 yield* sql`INSERT INTO legacy_runners (address, runner) VALUES ('legacy:1', 'legacy')`
815 const rows = yield* sql`SELECT machine_id, address, runner, healthy FROM legacy_runners`
816
817 yield* Effect.scoped(Layer.build(SqlRunnerStorage.layerMigrations({ prefix: "legacy" })))
818
819 expect(yield* sql`SELECT machine_id, address, runner, healthy FROM legacy_runners`).toEqual(rows)
820 expect(yield* sql`SELECT * FROM legacy_locks`).toEqual([])
821 }))
822 }
823
824 if (label === "mysql" || label === "vitess") {
825 // MySQL session timeouts are whole seconds.
826 const shardLockExpiration = lockOperationInterval * 3
827
828 // vtgate ignores SET SESSION wait_timeout and keeps its own lock
829 // connection alive, so behind Vitess a frozen runner keeps its locks.
830 if (label === "mysql") {
831 it.effect(
832 "releases a frozen runner's named shard locks after shardLockExpiration",
833 () =>
834 Effect.gen(function*() {
835 const storageA = yield* SqlRunnerStorage.make({ prefix: "frozen" })
836 const storageB = yield* SqlRunnerStorage.make({ prefix: "frozen" })
837 const shard = ShardId.make("default", 1)
838
839 expect(yield* storageA.acquire(runnerAddress1, [shard])).toEqual([shard])
840 // storageA freezes: it stops refreshing, but its lock connection stays open
841 const frozenAt = yield* Clock.currentTimeMillis
842 expect(yield* storageB.acquire(runnerAddress2, [shard])).toEqual([])
843
844 yield* storageB.acquire(runnerAddress2, [shard]).pipe(
845 Effect.repeat({
846 until: (acquired) => acquired.length > 0,
847 schedule: Schedule.spaced(100)
848 }),
849 Effect.timeoutOrElse({
850 duration: shardLockExpiration * 3,
851 orElse: () => Effect.die("the frozen runner still holds its named shard lock")
852 })
853 )
854 const elapsed = (yield* Clock.currentTimeMillis) - frozenAt
855 assert.isAtLeast(elapsed, shardLockExpiration * 0.9)
856 assert.isBelow(elapsed, shardLockExpiration * 2)
857 }).pipe(
858 Effect.scoped,
859 Effect.provide(ShardingConfig.layer({ shardLockExpiration })),
860 TestClock.withLive
861 ),
862 60_000
863 )
864
865 it.effect(
866 "restores wait_timeout before returning the lock connection to the pool",
867 () =>
868 Effect.gen(function*() {
869 const sql = yield* SqlClient.SqlClient
870 const [[defaultTimeout]] = yield* sql`SELECT @@GLOBAL.wait_timeout`.values
871
872 const lockConnectionId = yield* Effect.gen(function*() {
873 const storage = yield* SqlRunnerStorage.make({ prefix: "pooled" })
874 const shard = ShardId.make("default", 1)
875 expect(yield* storage.acquire(runnerAddress1, [shard])).toEqual([shard])
876 const [[id]] = yield* sql`SELECT IS_USED_LOCK(${`pooled.${shard.toString()}`})`.values
877 return Number(id)
878 }).pipe(Effect.scoped)
879
880 // The pool hands out the connection released last first.
881 const [[id, timeout]] = yield* sql.reserve.pipe(
882 Effect.flatMap((conn) => conn.executeValues("SELECT CONNECTION_ID(), @@SESSION.wait_timeout", [])),
883 Effect.scoped
884 )
885 expect(Number(id)).toEqual(lockConnectionId)
886 expect(Number(timeout)).toEqual(Number(defaultTimeout))
887 }).pipe(
888 Effect.provide(ShardingConfig.layer({ shardLockExpiration })),
889 TestClock.withLive
890 ),
891 60_000
892 )
893 }
894
895 it.effect("acquires cleanly after a runner without shards leaves its lock connection idle", () => {
896 const logs: Array<string> = []
897 const logger = Logger.make((options) => {
898 if (options.logLevel !== "Debug" && options.logLevel !== "Trace") {
899 logs.push(Logger.formatJson.log(options))
900 }
901 })
902 return Effect.gen(function*() {
903 const storage = yield* SqlRunnerStorage.make({ prefix: "idle" })
904 const runner = Runner.make({
905 address: runnerAddress1,
906 groups: ["default"],
907 weight: 1
908 })
909 const shard = ShardId.make("default", 1)
910
911 yield* storage.register(runner, true)
912 // Without shards, only the liveness probe runs, and it uses the shared
913 // pool, so the lock connection stays idle past shardLockExpiration.
914 const idleUntil = (yield* Clock.currentTimeMillis) + shardLockExpiration * 1.5
915 while ((yield* Clock.currentTimeMillis) < idleUntil) {
916 expect(yield* storage.refresh(runnerAddress1, [])).toEqual([])
917 yield* Effect.sleep(lockOperationInterval)
918 }
919
920 expect(yield* storage.acquire(runnerAddress1, [shard])).toEqual([shard])
921 expect(yield* storage.refresh(runnerAddress1, [shard])).toEqual([shard])
922 yield* storage.release(runnerAddress1, shard)
923 expect(logs).toEqual([])
924 }).pipe(
925 Effect.scoped,
926 Effect.provide(ShardingConfig.layer({ shardLockExpiration })),
927 Effect.provide(Logger.layer([logger])),
928 TestClock.withLive
929 )
930 }, 60_000)
931 }
932 })
933 })
934 })
935
936 const runnerAddress1 = RunnerAddress.make("localhost", 1234)
937 const runnerAddress2 = RunnerAddress.make("localhost", 5678)
938
939 interface PartitionState {
940 connectionAvailable: Latch.Latch
941 blockRelease: boolean
942 activeQueries: number
943 maxActiveQueries: number
944 interruptedQueries: number
945 failNextQueries: number
946 reservedConnections: number
947 usableConnections: number
948 }
949
950 // Registers a runner holding one shard, then wedges the reserved connection
951 // and blocks its release, so the first rebuild stalls closing the previous
952 // scope. `body` runs with the stall in place and may lift it through
953 // `partitioned`; it is lifted afterwards so the layer can be torn down.
954 const withStalledRelease = (
955 body: (
956 storage: RunnerStorage.RunnerStorage["Service"],
957 shards: ReadonlyArray<ShardId.ShardId>,
958 partitioned: PartitionState
959 ) => Effect.Effect<void, unknown>
960 ) => {
961 const partitioned = makePartitionState()
962 const layer = StorageLayer.pipe(
963 Layer.provideMerge(blackholeReservedConnection(partitioned, false)),
964 Layer.provide(ShardingConfig.layer({
965 shardLockDisableAdvisory: true,
966 ...partitionConfig
967 }))
968 )
969 return Effect.gen(function*() {
970 const storage = yield* RunnerStorage.RunnerStorage
971 const runner = Runner.make({
972 address: runnerAddress1,
973 groups: ["default"],
974 weight: 1
975 })
976 const shards = [ShardId.make("default", 1)]
977 yield* storage.register(runner, true)
978 yield* storage.acquire(runnerAddress1, shards)
979 partitionConnection(partitioned)
980 partitioned.blockRelease = true
981 yield* storage.refresh(runnerAddress1, shards).pipe(Effect.exit)
982 yield* body(storage, shards, partitioned)
983 }).pipe(
984 Effect.ensuring(Effect.sync(() => {
985 restoreConnection(partitioned)
986 partitioned.blockRelease = false
987 })),
988 Effect.provide(layer),
989 TestClock.withLive
990 )
991 }
992
993 const makePartitionState = (): PartitionState => ({
994 connectionAvailable: Latch.makeUnsafe(true),
995 blockRelease: false,
996 activeQueries: 0,
997 maxActiveQueries: 0,
998 interruptedQueries: 0,
999 failNextQueries: 0,
1000 reservedConnections: 0,
1001 usableConnections: 0
1002 })
1003
1004 const partitionConnection = (partitioned: PartitionState) => {
1005 Latch.closeUnsafe(partitioned.connectionAvailable)
1006 }
1007
1008 const restoreConnection = (partitioned: PartitionState) => {
1009 Latch.openUnsafe(partitioned.connectionAvailable)
1010 }
1011
1012 const partitionDiagnostics = (partitioned: PartitionState) =>
1013 [
1014 `active=${partitioned.activeQueries}`,
1015 `interrupted=${partitioned.interruptedQueries}`,
1016 `reserved=${partitioned.reservedConnections}`,
1017 `usable=${partitioned.usableConnections}`
1018 ].join(", ")
1019
1020 // Another runner takes the shard once it becomes free and hands it off again,
1021 // so it is free when the previous owner next checks it.
1022 const acquireHandoff = Effect.fnUntraced(function*(
1023 storage: RunnerStorage.RunnerStorage["Service"],
1024 shard: ShardId.ShardId,
1025 timeout: number
1026 ) {
1027 yield* storage.acquire(runnerAddress2, [shard]).pipe(
1028 Effect.repeat({
1029 until: (acquired) => acquired.length > 0,
1030 schedule: Schedule.spaced(100)
1031 }),
1032 Effect.timeoutOrElse({
1033 duration: timeout,
1034 orElse: () => Effect.die("the previous owner still holds the shard lock")
1035 })
1036 )
1037 yield* storage.release(runnerAddress2, shard)
1038 })
1039
1040 // Sharding retries a failed refresh before acting on its result.
1041 const refreshLikeSharding = (storage: RunnerStorage.RunnerStorage["Service"], shard: ShardId.ShardId) =>
1042 storage.refresh(runnerAddress1, [shard]).pipe(
1043 Effect.retry({ times: 5, schedule: Schedule.spaced(50) })
1044 )
1045
1046 const waitUntil = Effect.fnUntraced(
1047 function*(predicate: () => boolean) {
1048 while (!predicate()) {
1049 yield* Effect.sleep(20)
1050 }
1051 },
1052 Effect.timeoutOrElse({
1053 duration: 10_000,
1054 orElse: () => Effect.die("timed out waiting for condition")
1055 })
1056 )
1057
1058 const blackholeReservedConnection = (partitioned: PartitionState, resumePending: boolean) =>
1059 wrapReservedConnections(partitioned, resumePending).pipe(Layer.provide(PgContainer.layerClientDatabase))
1060
1061 // Wraps the reserved connections of the SqlClient in context.
1062 const wrapReservedConnections = (partitioned: PartitionState, resumePending: boolean) =>
1063 Layer.effect(
1064 SqlClient.SqlClient,
1065 Effect.gen(function*() {
1066 const sql = yield* SqlClient.SqlClient
1067 const wrapConnection = (connection: SqlConnection.Connection): SqlConnection.Connection => {
1068 let usable = false
1069 const execute = <A, E extends SqlError.SqlError, R>(effect: Effect.Effect<A, E, R>) =>
1070 Effect.suspend((): Effect.Effect<A, E | SqlError.SqlError, R> => {
1071 if (partitioned.failNextQueries > 0) {
1072 partitioned.failNextQueries--
1073 return Effect.fail(
1074 new SqlError.SqlError({
1075 reason: new SqlError.ConnectionError({ cause: new Error("connection lost") })
1076 })
1077 )
1078 }
1079 partitioned.activeQueries++
1080 partitioned.maxActiveQueries = Math.max(partitioned.maxActiveQueries, partitioned.activeQueries)
1081 return (!Latch.isOpen(partitioned.connectionAvailable)
1082 ? resumePending
1083 ? Effect.andThen(Latch.await(partitioned.connectionAvailable), effect)
1084 : Effect.never
1085 : effect).pipe(
1086 Effect.onExit((exit) =>
1087 Effect.sync(() => {
1088 partitioned.activeQueries--
1089 if (Exit.hasInterrupts(exit)) {
1090 partitioned.interruptedQueries++
1091 }
1092 if (Exit.isSuccess(exit) && !usable) {
1093 usable = true
1094 partitioned.usableConnections++
1095 }
1096 })
1097 )
1098 )
1099 })
1100 return {
1101 ...connection,
1102 execute: (...args) => execute(connection.execute(...args)),
1103 executeRaw: (...args) => execute(connection.executeRaw(...args)),
1104 executeValues: (...args) => execute(connection.executeValues(...args)),
1105 executeValuesUnprepared: (...args) => execute(connection.executeValuesUnprepared(...args)),
1106 executeUnprepared: (...args) => execute(connection.executeUnprepared(...args))
1107 }
1108 }
1109 let client: SqlClient.SqlClient
1110 client = new Proxy(sql, {
1111 get(target, property, receiver) {
1112 if (property === "reserve") {
1113 return Effect.andThen(
1114 // simulates a driver that stalls uninterruptibly while tearing
1115 // down a reserved connection, so closing its scope cannot be
1116 // interrupted
1117 Effect.addFinalizer(() =>
1118 Effect.uninterruptible(Effect.suspend(function waitForRelease(): Effect.Effect<void> {
1119 if (!partitioned.blockRelease) return Effect.void
1120 return Effect.andThen(
1121 Effect.promise(() => new Promise<void>((resolve) => setTimeout(resolve, 5))),
1122 waitForRelease
1123 )
1124 }))
1125 ),
1126 Effect.map(target.reserve, (connection) => {
1127 partitioned.reservedConnections++
1128 return wrapConnection(connection)
1129 })
1130 )
1131 }
1132 if (property === "withoutTransforms") {
1133 return () => client
1134 }
1135 return Reflect.get(target, property, receiver)
1136 }
1137 })
1138 return client
1139 })
1140 )
1141
1142 const SqliteLayer = Effect.gen(function*() {
1143 const fs = yield* FileSystem.FileSystem
1144 const dir = yield* fs.makeTempDirectoryScoped()
1145 return SqliteClient.layer({
1146 filename: dir + "/test.db"
1147 })
1148 }).pipe(Layer.unwrap, Layer.provide(NodeFileSystem.layer))