@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 183 lines 6.69 KB
Raw
1 import { NodeCrypto } from "@effect/platform-node"
2 import { assert, describe, it } from "@effect/vitest"
3 import { Clock, Context, Deferred, Effect, Latch, Layer, Option, Schema } from "effect"
4 import {
5 ClusterSchema,
6 Entity,
7 EntityId,
8 MessageStorage,
9 RunnerAddress,
10 RunnerHealth,
11 Runners,
12 Sharding,
13 ShardingConfig,
14 type Snowflake,
15 SqlMessageStorage,
16 SqlRunnerStorage
17 } from "effect/cluster"
18 import { Rpc } from "effect/rpc"
19 import { SqlClient } from "effect/sql"
20 import { TestClock } from "effect/testing"
21 import { PgContainer } from "../fixtures/pg-utils.ts"
22
23 const Hold = Entity.make("Hold", [
24 Rpc.make("Hold", { success: Schema.String })
25 ]).annotateRpcs(ClusterSchema.Persisted, true)
26
27 interface RunnerProbe {
28 readonly port: number
29 readonly entered: Deferred.Deferred<Snowflake.Snowflake>
30 readonly gate: Latch.Latch
31 readonly returned: Deferred.Deferred<void>
32 }
33
34 const makeProbe = (port: number): RunnerProbe => ({
35 port,
36 entered: Deferred.makeUnsafe(),
37 gate: Latch.makeUnsafe(false),
38 returned: Deferred.makeUnsafe()
39 })
40
41 const HoldLayer = (probe: RunnerProbe) =>
42 Hold.toLayer(Effect.gen(function*() {
43 const address = yield* Entity.CurrentRunnerAddress
44 return Hold.of({
45 Hold: Effect.fnUntraced(function*(envelope) {
46 yield* Deferred.succeed(probe.entered, envelope.requestId)
47 yield* probe.gate.await
48 yield* Deferred.succeed(probe.returned, void 0)
49 return String(address.port)
50 })
51 })
52 }))
53
54 // A clock whose timers stop firing while frozen, as in a paused process: the
55 // runner's lock refreshes, heartbeats and storage polls stall, while work that
56 // does not wait on a timer, such as a running handler, carries on.
57 const makeFreezableClock = () => {
58 const live = Clock.Clock.defaultValue()
59 const thawed = Latch.makeUnsafe(true)
60 const clock: Clock.Clock = {
61 currentTimeMillisUnsafe: () => live.currentTimeMillisUnsafe(),
62 currentTimeMillis: live.currentTimeMillis,
63 currentTimeNanosUnsafe: () => live.currentTimeNanosUnsafe(),
64 currentTimeNanos: live.currentTimeNanos,
65 monotonicTimeNanosUnsafe: () => live.monotonicTimeNanosUnsafe(),
66 monotonicTimeNanos: live.monotonicTimeNanos,
67 sleep: (duration) => Effect.andThen(live.sleep(duration), thawed.await)
68 }
69 return {
70 clock,
71 freeze: Effect.sync(() => {
72 thawed.closeUnsafe()
73 }),
74 thaw: thawed.open
75 }
76 }
77
78 const RunnerLayer = (probe: RunnerProbe, config: Partial<ShardingConfig.ShardingConfig["Service"]>) =>
79 HoldLayer(probe).pipe(
80 Layer.provideMerge(Sharding.layer),
81 Layer.provide([SqlRunnerStorage.layer, RunnerHealth.layerNoop, Runners.layerNoop]),
82 Layer.provideMerge(SqlMessageStorage.layer),
83 Layer.provide(NodeCrypto.layer),
84 Layer.provide(ShardingConfig.layer({
85 ...config,
86 runnerAddress: Option.some(RunnerAddress.make("localhost", probe.port)),
87 shardsPerGroup: 1,
88 // Short enough for B to take over quickly, with lock operation deadlines
89 // that tolerate CI latency.
90 shardLockExpiration: 3000,
91 shardLockRefreshInterval: 1000,
92 entityMessagePollInterval: 100,
93 refreshAssignmentsInterval: 100,
94 entityTerminationTimeout: 0
95 }))
96 )
97
98 const waitFor = Effect.fnUntraced(function*<A, E, R>(effect: Effect.Effect<A | undefined, E, R>) {
99 while (true) {
100 const value = yield* effect
101 if (value !== undefined) return value
102 yield* Effect.sleep(20)
103 }
104 })
105
106 const staleOwnerTest = Effect.fnUntraced(
107 function*(options: { readonly advisory: boolean }) {
108 const sql = yield* SqlClient.SqlClient
109 const config = { shardLockDisableAdvisory: !options.advisory }
110 const pauseA = makeFreezableClock()
111 const probeA = makeProbe(1234)
112 const probeB = makeProbe(5678)
113
114 const contextA = yield* Layer.build(RunnerLayer(probeA, config)).pipe(
115 Effect.provideService(Clock.Clock, pauseA.clock)
116 )
117 const storage = Context.get(contextA, MessageStorage.MessageStorage)
118 yield* Hold.client.pipe(
119 Effect.flatMap((makeClient) => makeClient("1").Hold(void 0, { discard: true })),
120 Effect.provide(contextA)
121 )
122 const requestId = yield* Deferred.await(probeA.entered).pipe(
123 Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner A never started the request") })
124 )
125
126 // A is paused, and its lock session is lost.
127 yield* pauseA.freeze
128 if (options.advisory) {
129 yield* sql`SELECT pg_terminate_backend(pid, 10000) FROM pg_locks
130 WHERE locktype = 'advisory' AND objsubid = 2
131 AND database = (SELECT oid FROM pg_database WHERE datname = current_database())`
132 }
133
134 const contextB = yield* Layer.build(RunnerLayer(probeB, config))
135 const shardingB = Context.get(contextB, Sharding.Sharding)
136 const shardId = shardingB.getShardId(EntityId.make("1"), "default")
137 yield* waitFor(Effect.sync(() => shardingB.hasShardId(shardId) || undefined)).pipe(
138 Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never acquired the shard") })
139 )
140 yield* Deferred.await(probeB.entered).pipe(
141 Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never started the request") })
142 )
143
144 // A's handler finishes before A's lock refresh has run again.
145 yield* probeA.gate.open
146 yield* Deferred.await(probeA.returned)
147
148 const storedExit = storage.repliesForUnfiltered([requestId]).pipe(
149 Effect.map((replies) => replies.find((reply) => reply._tag === "WithExit"))
150 )
151 // An unfenced write lands within milliseconds of the handler returning.
152 const stale = yield* waitFor(storedExit).pipe(Effect.timeoutOption(2000))
153 assert.isTrue(Option.isNone(stale), "the stale owner stored its reply")
154
155 yield* probeB.gate.open
156 const reply = yield* waitFor(storedExit).pipe(
157 Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner B never stored its reply") })
158 )
159 assert(reply._tag === "WithExit")
160 assert.deepStrictEqual(reply.exit, { _tag: "Success", value: "5678" })
161
162 // Resume A so it can shut down. It may legitimately own the shard again
163 // from here, so nothing about it is asserted.
164 yield* pauseA.thaw
165 },
166 // Release both runners before the database.
167 Effect.scoped,
168 Effect.provide(PgContainer.layerClientDatabase),
169 TestClock.withLive
170 )
171
172 describe("Sharding shard fencing", { concurrent: false }, () => {
173 // Each test runs two runners against live lock deadlines, so keep them apart.
174 it.layer(PgContainer.layer, { concurrent: false, timeout: 120_000 })((it) => {
175 it.effect(
176 "a runner that lost its advisory lock cannot store a reply",
177 () => staleOwnerTest({ advisory: true }),
178 60_000
179 )
180
181 it.effect("a runner that lost its lease cannot store a reply", () => staleOwnerTest({ advisory: false }), 60_000)
182 })
183 })