@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 304 lines 12.1 KB
Raw
1 import { assert, describe, it } from "@effect/vitest"
2 import { Clock, Context, Cron, DateTime, Duration, Effect, Latch, Layer } from "effect"
3 import { ClusterCron, ClusterSchema, Entity } from "effect/cluster"
4 import { type Backend, make } from "./harness.ts"
5
6 interface Tick {
7 readonly at: number
8 readonly runner: string
9 readonly scheduled: string
10 }
11
12 const everySecond = Cron.parseUnsafe("* * * * * *", "UTC")
13 const testConfig = { shardsPerGroup: 12 } as const
14
15 const addressString = (address: { readonly host: string; readonly port: number }) => `${address.host}:${address.port}`
16
17 const recordTick = (ticks: Array<Tick>) =>
18 Effect.contextWith((context: Context.Context<never>) =>
19 Effect.gen(function*() {
20 const runner = Context.getUnsafe(context, Entity.CurrentRunnerAddress)
21 const at = yield* Clock.currentTimeMillis
22 const tick = {
23 at,
24 runner: addressString(runner),
25 scheduled: DateTime.formatIso(Context.getUnsafe(context, ClusterCron.CurrentScheduledTime))
26 }
27 const isFirst = ticks.length === 0
28 ticks.push(tick)
29 return isFirst
30 })
31 )
32
33 const cronProbe = (name: string, shardGroup = "default") =>
34 Entity.make(`ClusterCron/${name}`, []).annotate(ClusterSchema.ShardGroup, () => shardGroup)
35
36 const nextScheduled = (cron: Cron.Cron, after: DateTime.DateTime.Input) =>
37 DateTime.formatIso(DateTime.fromDateUnsafe(Cron.next(cron, after)))
38
39 const assertScheduledFromExecutionTime = (
40 cron: Cron.Cron,
41 ticks: ReadonlyArray<Tick>,
42 startIndex = 1
43 ) => {
44 for (let index = startIndex; index < ticks.length; index++) {
45 assert.strictEqual(ticks[index].scheduled, nextScheduled(cron, ticks[index - 1].at))
46 }
47 }
48
49 const assertScheduledFromPrevious = (cron: Cron.Cron, ticks: ReadonlyArray<Tick>) => {
50 for (let index = 1; index < ticks.length; index++) {
51 assert.strictEqual(ticks[index].scheduled, nextScheduled(cron, ticks[index - 1].scheduled))
52 }
53 }
54
55 describe("cluster cron integration", () => {
56 for (const backend of ["pg", "mysql"] satisfies ReadonlyArray<Backend>) {
57 it.live(`${backend}: runs once per scheduled instant and continues after failure`, () =>
58 Effect.gen(function*() {
59 const ticks: Array<Tick> = []
60 let failingAttempts = 0
61 let successfulAttempts = 0
62 const cron = ClusterCron.make({
63 name: `basic-${backend}`,
64 cron: everySecond,
65 execute: recordTick(ticks)
66 })
67 const failingCron = ClusterCron.make({
68 name: `failure-${backend}`,
69 cron: everySecond,
70 execute: Effect.suspend(() => {
71 failingAttempts++
72 if (failingAttempts === 1) return Effect.fail("expected cron failure")
73 successfulAttempts++
74 return Effect.void
75 })
76 })
77 const cluster = yield* make({
78 backend,
79 config: testConfig,
80 entities: Layer.merge(cron, failingCron)
81 })
82 yield* cluster.start(3)
83 yield* cluster.waitForStableAssignments()
84 yield* cluster.waitUntil(
85 "The cron jobs did not continue through four scheduled instants",
86 Effect.sync(() => ticks.length >= 4 && failingAttempts >= 4)
87 )
88
89 const firstFour = ticks.slice(0, 4)
90 assert.strictEqual(new Set(firstFour.map((tick) => tick.scheduled)).size, firstFour.length)
91 assertScheduledFromExecutionTime(everySecond, firstFour)
92 assert.strictEqual(successfulAttempts, failingAttempts - 1)
93 }))
94
95 it.live(`${backend}: calculates the next run from the previous instant or the current time`, () =>
96 Effect.gen(function*() {
97 const previousTicks: Array<Tick> = []
98 const currentTicks: Array<Tick> = []
99 const gate = Latch.makeUnsafe()
100 // Hold an execution only once the assignments are stable, as a held
101 // execution on a moving shard would keep that shard from being released.
102 let armed = false
103 let entered = 0
104 const blockedExecution = Effect.fnUntraced(
105 function*(ticks: Array<Tick>) {
106 if (!armed) return
107 if (yield* recordTick(ticks)) {
108 entered++
109 yield* gate.await
110 }
111 },
112 Effect.uninterruptible
113 )
114 const previousCron = ClusterCron.make({
115 name: `previous-${backend}`,
116 cron: everySecond,
117 calculateNextRunFromPrevious: true,
118 execute: blockedExecution(previousTicks)
119 })
120 const currentCron = ClusterCron.make({
121 name: `current-${backend}`,
122 cron: everySecond,
123 execute: blockedExecution(currentTicks)
124 })
125 const cluster = yield* make({
126 backend,
127 config: testConfig,
128 entities: Layer.merge(previousCron, currentCron)
129 })
130 yield* cluster.start(3)
131 yield* cluster.waitForStableAssignments()
132 armed = true
133 yield* cluster.waitUntil(
134 "Both cron executions did not enter their first run",
135 Effect.sync(() => entered === 2)
136 )
137 const blockedAt = yield* Clock.currentTimeMillis
138 yield* cluster.waitUntil(
139 "The cron executions were not held across several scheduled instants",
140 Effect.map(Clock.currentTimeMillis, (now) => now >= blockedAt + 3_200),
141 "5 seconds"
142 )
143 gate.openUnsafe()
144 yield* cluster.waitUntil(
145 "The cron jobs did not resume after their first execution",
146 Effect.sync(() => previousTicks.length >= 4 && currentTicks.length >= 4),
147 "12 seconds"
148 )
149
150 const previousSecond = DateTime.toEpochMillis(DateTime.makeUnsafe(previousTicks[1].scheduled))
151 const currentSecond = DateTime.toEpochMillis(DateTime.makeUnsafe(currentTicks[1].scheduled))
152 assertScheduledFromPrevious(everySecond, previousTicks)
153 assertScheduledFromExecutionTime(everySecond, currentTicks, 2)
154 assert.isAtLeast(previousTicks[1].at - previousSecond, 2_000)
155 assert.isAtMost(currentTicks[1].at - currentSecond, 1_000)
156 }))
157
158 it.live(`${backend}: catches up or skips stale runs and preserves the schedule across restart`, () =>
159 Effect.gen(function*() {
160 const catchUpTicks: Array<Tick> = []
161 const skipTicks: Array<Tick> = []
162 const catchUpCron = ClusterCron.make({
163 name: `catch-up-${backend}`,
164 cron: everySecond,
165 calculateNextRunFromPrevious: true,
166 execute: recordTick(catchUpTicks)
167 })
168 const skipCron = ClusterCron.make({
169 name: `skip-stale-${backend}`,
170 cron: everySecond,
171 calculateNextRunFromPrevious: true,
172 skipIfOlderThan: Duration.millis(500),
173 execute: recordTick(skipTicks)
174 })
175 const cluster = yield* make({
176 backend,
177 config: testConfig,
178 entities: Layer.merge(catchUpCron, skipCron)
179 })
180 const runners = yield* cluster.start(3)
181 yield* cluster.waitForStableAssignments()
182 yield* cluster.waitUntil(
183 "The cron schedules did not start",
184 Effect.sync(() => catchUpTicks.length >= 2 && skipTicks.length >= 2)
185 )
186 for (const runner of runners) {
187 yield* cluster.kill(runner)
188 }
189 const catchUpBefore = catchUpTicks.length
190 const skipBefore = skipTicks.length
191 const lastScheduledBeforeRestart = catchUpTicks[catchUpBefore - 1].scheduled
192 const stoppedAt = yield* Clock.currentTimeMillis
193 yield* cluster.waitUntil(
194 "The cluster downtime window did not elapse",
195 Effect.map(Clock.currentTimeMillis, (now) => now >= stoppedAt + 3_200),
196 "5 seconds"
197 )
198 assert.strictEqual(catchUpTicks.length, catchUpBefore)
199 assert.strictEqual(skipTicks.length, skipBefore)
200
201 yield* cluster.start(3)
202 yield* cluster.waitForStableAssignments()
203 yield* cluster.waitUntil(
204 "The restarted cluster did not expose the catch-up and stale-skip difference",
205 Effect.sync(() => {
206 const caughtUp = catchUpTicks.length - catchUpBefore
207 const skipped = skipTicks.length - skipBefore
208 return caughtUp >= 3 && caughtUp >= skipped + 2
209 }),
210 "15 seconds"
211 )
212 yield* cluster.waitUntil(
213 "The stale-skipping cron did not resume",
214 Effect.sync(() => skipTicks.length > skipBefore)
215 )
216
217 const firstCatchUp = catchUpTicks[catchUpBefore]
218 const firstAfterSkip = skipTicks[skipBefore]
219 const firstAfterSkipScheduledAt = DateTime.toEpochMillis(DateTime.makeUnsafe(firstAfterSkip.scheduled))
220 assert.strictEqual(
221 firstCatchUp.scheduled,
222 nextScheduled(everySecond, lastScheduledBeforeRestart)
223 )
224 assert.isAtLeast(firstCatchUp.at - DateTime.toEpochMillis(DateTime.makeUnsafe(firstCatchUp.scheduled)), 2_000)
225 assert.isAtMost(firstAfterSkip.at - firstAfterSkipScheduledAt, 750)
226 assertScheduledFromPrevious(everySecond, catchUpTicks)
227 }))
228
229 it.live(`${backend}: resumes without duplicate or missing ticks after the singleton owner dies`, () =>
230 Effect.gen(function*() {
231 const name = `owner-failover-${backend}`
232 const ticks: Array<Tick> = []
233 const cron = ClusterCron.make({
234 name,
235 cron: everySecond,
236 calculateNextRunFromPrevious: true,
237 execute: recordTick(ticks)
238 })
239 const cluster = yield* make({ backend, config: testConfig, entities: cron })
240 yield* cluster.start(3)
241 yield* cluster.waitForStableAssignments()
242 yield* cluster.waitUntil(
243 "The cron did not execute before failover",
244 Effect.sync(() => ticks.length >= 2)
245 )
246 const owner = yield* cluster.ownerOfEntity(cronProbe(name), name)
247 assert.isDefined(owner)
248 yield* cluster.kill(owner!)
249 const afterKill = ticks.length
250 yield* cluster.waitUntil(
251 "The cron did not resume after its singleton owner died",
252 Effect.sync(() => ticks.length >= afterKill + 3),
253 "12 seconds"
254 )
255 yield* cluster.waitForStableAssignments()
256
257 assertScheduledFromPrevious(everySecond, ticks)
258 assert.strictEqual(new Set(ticks.map((tick) => tick.scheduled)).size, ticks.length)
259 assert.isTrue(ticks.slice(afterKill).every((tick) => tick.runner !== addressString(owner!.address)))
260 }))
261
262 it.live(`${backend}: assigns cron singletons and executions to their shard groups`, () =>
263 Effect.gen(function*() {
264 const defaultName = `default-group-${backend}`
265 const specialName = `special-group-${backend}`
266 const defaultTicks: Array<Tick> = []
267 const specialTicks: Array<Tick> = []
268 const defaultCron = ClusterCron.make({
269 name: defaultName,
270 cron: everySecond,
271 execute: recordTick(defaultTicks)
272 })
273 const specialCron = ClusterCron.make({
274 name: specialName,
275 cron: everySecond,
276 shardGroup: "special",
277 execute: recordTick(specialTicks)
278 })
279 const cluster = yield* make({
280 backend,
281 config: {
282 availableShardGroups: ["default", "special"],
283 shardsPerGroup: 12
284 },
285 entities: Layer.merge(defaultCron, specialCron)
286 })
287 const [defaultRunner] = yield* cluster.start(1, { assignedShardGroups: ["default"] })
288 const [specialRunner] = yield* cluster.start(1, { assignedShardGroups: ["special"] })
289 yield* cluster.waitForStableAssignments()
290 yield* cluster.waitUntil(
291 "The shard-group cron jobs did not execute",
292 Effect.sync(() => defaultTicks.length >= 2 && specialTicks.length >= 2)
293 )
294
295 assert.strictEqual(yield* cluster.ownerOfEntity(cronProbe(defaultName), defaultName), defaultRunner)
296 assert.strictEqual(
297 yield* cluster.ownerOfEntity(cronProbe(specialName, "special"), specialName),
298 specialRunner
299 )
300 assert.isTrue(defaultTicks.every((tick) => tick.runner === addressString(defaultRunner.address)))
301 assert.isTrue(specialTicks.every((tick) => tick.runner === addressString(specialRunner.address)))
302 }))
303 }
304 })