The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { NodeCrypto } from "@effect/platform-node" |
| 2 | import { assert, describe, it } from "@effect/vitest" |
| 3 | import { Clock, Context, Deferred, Effect, Latch, Layer, Option, Schema } from "effect" |
| 4 | import { |
| 5 | ClusterSchema, |
| 6 | Entity, |
| 7 | EntityId, |
| 8 | MessageStorage, |
| 9 | RunnerAddress, |
| 10 | RunnerHealth, |
| 11 | Runners, |
| 12 | Sharding, |
| 13 | ShardingConfig, |
| 14 | type Snowflake, |
| 15 | SqlMessageStorage, |
| 16 | SqlRunnerStorage |
| 17 | } from "effect/cluster" |
| 18 | import { Rpc } from "effect/rpc" |
| 19 | import { SqlClient } from "effect/sql" |
| 20 | import { TestClock } from "effect/testing" |
| 21 | import { PgContainer } from "../fixtures/pg-utils.ts" |
| 22 | |
| 23 | const Hold = Entity.make("Hold", [ |
| 24 | Rpc.make("Hold", { success: Schema.String }) |
| 25 | ]).annotateRpcs(ClusterSchema.Persisted, true) |
| 26 | |
| 27 | interface RunnerProbe { |
| 28 | readonly port: number |
| 29 | readonly entered: Deferred.Deferred<Snowflake.Snowflake> |
| 30 | readonly gate: Latch.Latch |
| 31 | readonly returned: Deferred.Deferred<void> |
| 32 | } |
| 33 | |
| 34 | const makeProbe = (port: number): RunnerProbe => ({ |
| 35 | port, |
| 36 | entered: Deferred.makeUnsafe(), |
| 37 | gate: Latch.makeUnsafe(false), |
| 38 | returned: Deferred.makeUnsafe() |
| 39 | }) |
| 40 | |
| 41 | const HoldLayer = (probe: RunnerProbe) => |
| 42 | Hold.toLayer(Effect.gen(function*() { |
| 43 | const address = yield* Entity.CurrentRunnerAddress |
| 44 | return Hold.of({ |
| 45 | Hold: Effect.fnUntraced(function*(envelope) { |
| 46 | yield* Deferred.succeed(probe.entered, envelope.requestId) |
| 47 | yield* probe.gate.await |
| 48 | yield* Deferred.succeed(probe.returned, void 0) |
| 49 | return String(address.port) |
| 50 | }) |
| 51 | }) |
| 52 | })) |
| 53 | |
| 54 | // A clock whose timers stop firing while frozen, as in a paused process: the |
| 55 | // runner's lock refreshes, heartbeats and storage polls stall, while work that |
| 56 | // does not wait on a timer, such as a running handler, carries on. |
| 57 | const makeFreezableClock = () => { |
| 58 | const live = Clock.Clock.defaultValue() |
| 59 | const thawed = Latch.makeUnsafe(true) |
| 60 | const clock: Clock.Clock = { |
| 61 | currentTimeMillisUnsafe: () => live.currentTimeMillisUnsafe(), |
| 62 | currentTimeMillis: live.currentTimeMillis, |
| 63 | currentTimeNanosUnsafe: () => live.currentTimeNanosUnsafe(), |
| 64 | currentTimeNanos: live.currentTimeNanos, |
| 65 | monotonicTimeNanosUnsafe: () => live.monotonicTimeNanosUnsafe(), |
| 66 | monotonicTimeNanos: live.monotonicTimeNanos, |
| 67 | sleep: (duration) => Effect.andThen(live.sleep(duration), thawed.await) |
| 68 | } |
| 69 | return { |
| 70 | clock, |
| 71 | freeze: Effect.sync(() => { |
| 72 | thawed.closeUnsafe() |
| 73 | }), |
| 74 | thaw: thawed.open |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | const RunnerLayer = (probe: RunnerProbe, config: Partial<ShardingConfig.ShardingConfig["Service"]>) => |
| 79 | HoldLayer(probe).pipe( |
| 80 | Layer.provideMerge(Sharding.layer), |
| 81 | Layer.provide([SqlRunnerStorage.layer, RunnerHealth.layerNoop, Runners.layerNoop]), |
| 82 | Layer.provideMerge(SqlMessageStorage.layer), |
| 83 | Layer.provide(NodeCrypto.layer), |
| 84 | Layer.provide(ShardingConfig.layer({ |
| 85 | ...config, |
| 86 | runnerAddress: Option.some(RunnerAddress.make("localhost", probe.port)), |
| 87 | shardsPerGroup: 1, |
| 88 | // Short enough for B to take over quickly, with lock operation deadlines |
| 89 | // that tolerate CI latency. |
| 90 | shardLockExpiration: 3000, |
| 91 | shardLockRefreshInterval: 1000, |
| 92 | entityMessagePollInterval: 100, |
| 93 | refreshAssignmentsInterval: 100, |
| 94 | entityTerminationTimeout: 0 |
| 95 | })) |
| 96 | ) |
| 97 | |
| 98 | const waitFor = Effect.fnUntraced(function*<A, E, R>(effect: Effect.Effect<A | undefined, E, R>) { |
| 99 | while (true) { |
| 100 | const value = yield* effect |
| 101 | if (value !== undefined) return value |
| 102 | yield* Effect.sleep(20) |
| 103 | } |
| 104 | }) |
| 105 | |
| 106 | const staleOwnerTest = Effect.fnUntraced( |
| 107 | function*(options: { readonly advisory: boolean }) { |
| 108 | const sql = yield* SqlClient.SqlClient |
| 109 | const config = { shardLockDisableAdvisory: !options.advisory } |
| 110 | const pauseA = makeFreezableClock() |
| 111 | const probeA = makeProbe(1234) |
| 112 | const probeB = makeProbe(5678) |
| 113 | |
| 114 | const contextA = yield* Layer.build(RunnerLayer(probeA, config)).pipe( |
| 115 | Effect.provideService(Clock.Clock, pauseA.clock) |
| 116 | ) |
| 117 | const storage = Context.get(contextA, MessageStorage.MessageStorage) |
| 118 | yield* Hold.client.pipe( |
| 119 | Effect.flatMap((makeClient) => makeClient("1").Hold(void 0, { discard: true })), |
| 120 | Effect.provide(contextA) |
| 121 | ) |
| 122 | const requestId = yield* Deferred.await(probeA.entered).pipe( |
| 123 | Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner A never started the request") }) |
| 124 | ) |
| 125 | |
| 126 | // A is paused, and its lock session is lost. |
| 127 | yield* pauseA.freeze |
| 128 | if (options.advisory) { |
| 129 | yield* sql`SELECT pg_terminate_backend(pid, 10000) FROM pg_locks |
| 130 | WHERE locktype = 'advisory' AND objsubid = 2 |
| 131 | AND database = (SELECT oid FROM pg_database WHERE datname = current_database())` |
| 132 | } |
| 133 | |
| 134 | const contextB = yield* Layer.build(RunnerLayer(probeB, config)) |
| 135 | const shardingB = Context.get(contextB, Sharding.Sharding) |
| 136 | const shardId = shardingB.getShardId(EntityId.make("1"), "default") |
| 137 | yield* waitFor(Effect.sync(() => shardingB.hasShardId(shardId) || undefined)).pipe( |
| 138 | Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never acquired the shard") }) |
| 139 | ) |
| 140 | yield* Deferred.await(probeB.entered).pipe( |
| 141 | Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never started the request") }) |
| 142 | ) |
| 143 | |
| 144 | // A's handler finishes before A's lock refresh has run again. |
| 145 | yield* probeA.gate.open |
| 146 | yield* Deferred.await(probeA.returned) |
| 147 | |
| 148 | const storedExit = storage.repliesForUnfiltered([requestId]).pipe( |
| 149 | Effect.map((replies) => replies.find((reply) => reply._tag === "WithExit")) |
| 150 | ) |
| 151 | // An unfenced write lands within milliseconds of the handler returning. |
| 152 | const stale = yield* waitFor(storedExit).pipe(Effect.timeoutOption(2000)) |
| 153 | assert.isTrue(Option.isNone(stale), "the stale owner stored its reply") |
| 154 | |
| 155 | yield* probeB.gate.open |
| 156 | const reply = yield* waitFor(storedExit).pipe( |
| 157 | Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner B never stored its reply") }) |
| 158 | ) |
| 159 | assert(reply._tag === "WithExit") |
| 160 | assert.deepStrictEqual(reply.exit, { _tag: "Success", value: "5678" }) |
| 161 | |
| 162 | // Resume A so it can shut down. It may legitimately own the shard again |
| 163 | // from here, so nothing about it is asserted. |
| 164 | yield* pauseA.thaw |
| 165 | }, |
| 166 | // Release both runners before the database. |
| 167 | Effect.scoped, |
| 168 | Effect.provide(PgContainer.layerClientDatabase), |
| 169 | TestClock.withLive |
| 170 | ) |
| 171 | |
| 172 | describe("Sharding shard fencing", { concurrent: false }, () => { |
| 173 | // Each test runs two runners against live lock deadlines, so keep them apart. |
| 174 | it.layer(PgContainer.layer, { concurrent: false, timeout: 120_000 })((it) => { |
| 175 | it.effect( |
| 176 | "a runner that lost its advisory lock cannot store a reply", |
| 177 | () => staleOwnerTest({ advisory: true }), |
| 178 | 60_000 |
| 179 | ) |
| 180 | |
| 181 | it.effect("a runner that lost its lease cannot store a reply", () => staleOwnerTest({ advisory: false }), 60_000) |
| 182 | }) |
| 183 | }) |