The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { assert, describe, it } from "@effect/vitest" |
| 2 | import { Clock, Context, Cron, DateTime, Duration, Effect, Latch, Layer } from "effect" |
| 3 | import { ClusterCron, ClusterSchema, Entity } from "effect/cluster" |
| 4 | import { type Backend, make } from "./harness.ts" |
| 5 | |
| 6 | interface Tick { |
| 7 | readonly at: number |
| 8 | readonly runner: string |
| 9 | readonly scheduled: string |
| 10 | } |
| 11 | |
| 12 | const everySecond = Cron.parseUnsafe("* * * * * *", "UTC") |
| 13 | const testConfig = { shardsPerGroup: 12 } as const |
| 14 | |
| 15 | const addressString = (address: { readonly host: string; readonly port: number }) => `${address.host}:${address.port}` |
| 16 | |
| 17 | const recordTick = (ticks: Array<Tick>) => |
| 18 | Effect.contextWith((context: Context.Context<never>) => |
| 19 | Effect.gen(function*() { |
| 20 | const runner = Context.getUnsafe(context, Entity.CurrentRunnerAddress) |
| 21 | const at = yield* Clock.currentTimeMillis |
| 22 | const tick = { |
| 23 | at, |
| 24 | runner: addressString(runner), |
| 25 | scheduled: DateTime.formatIso(Context.getUnsafe(context, ClusterCron.CurrentScheduledTime)) |
| 26 | } |
| 27 | const isFirst = ticks.length === 0 |
| 28 | ticks.push(tick) |
| 29 | return isFirst |
| 30 | }) |
| 31 | ) |
| 32 | |
| 33 | const cronProbe = (name: string, shardGroup = "default") => |
| 34 | Entity.make(`ClusterCron/${name}`, []).annotate(ClusterSchema.ShardGroup, () => shardGroup) |
| 35 | |
| 36 | const nextScheduled = (cron: Cron.Cron, after: DateTime.DateTime.Input) => |
| 37 | DateTime.formatIso(DateTime.fromDateUnsafe(Cron.next(cron, after))) |
| 38 | |
| 39 | const assertScheduledFromExecutionTime = ( |
| 40 | cron: Cron.Cron, |
| 41 | ticks: ReadonlyArray<Tick>, |
| 42 | startIndex = 1 |
| 43 | ) => { |
| 44 | for (let index = startIndex; index < ticks.length; index++) { |
| 45 | assert.strictEqual(ticks[index].scheduled, nextScheduled(cron, ticks[index - 1].at)) |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | const assertScheduledFromPrevious = (cron: Cron.Cron, ticks: ReadonlyArray<Tick>) => { |
| 50 | for (let index = 1; index < ticks.length; index++) { |
| 51 | assert.strictEqual(ticks[index].scheduled, nextScheduled(cron, ticks[index - 1].scheduled)) |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | describe("cluster cron integration", () => { |
| 56 | for (const backend of ["pg", "mysql"] satisfies ReadonlyArray<Backend>) { |
| 57 | it.live(`${backend}: runs once per scheduled instant and continues after failure`, () => |
| 58 | Effect.gen(function*() { |
| 59 | const ticks: Array<Tick> = [] |
| 60 | let failingAttempts = 0 |
| 61 | let successfulAttempts = 0 |
| 62 | const cron = ClusterCron.make({ |
| 63 | name: `basic-${backend}`, |
| 64 | cron: everySecond, |
| 65 | execute: recordTick(ticks) |
| 66 | }) |
| 67 | const failingCron = ClusterCron.make({ |
| 68 | name: `failure-${backend}`, |
| 69 | cron: everySecond, |
| 70 | execute: Effect.suspend(() => { |
| 71 | failingAttempts++ |
| 72 | if (failingAttempts === 1) return Effect.fail("expected cron failure") |
| 73 | successfulAttempts++ |
| 74 | return Effect.void |
| 75 | }) |
| 76 | }) |
| 77 | const cluster = yield* make({ |
| 78 | backend, |
| 79 | config: testConfig, |
| 80 | entities: Layer.merge(cron, failingCron) |
| 81 | }) |
| 82 | yield* cluster.start(3) |
| 83 | yield* cluster.waitForStableAssignments() |
| 84 | yield* cluster.waitUntil( |
| 85 | "The cron jobs did not continue through four scheduled instants", |
| 86 | Effect.sync(() => ticks.length >= 4 && failingAttempts >= 4) |
| 87 | ) |
| 88 | |
| 89 | const firstFour = ticks.slice(0, 4) |
| 90 | assert.strictEqual(new Set(firstFour.map((tick) => tick.scheduled)).size, firstFour.length) |
| 91 | assertScheduledFromExecutionTime(everySecond, firstFour) |
| 92 | assert.strictEqual(successfulAttempts, failingAttempts - 1) |
| 93 | })) |
| 94 | |
| 95 | it.live(`${backend}: calculates the next run from the previous instant or the current time`, () => |
| 96 | Effect.gen(function*() { |
| 97 | const previousTicks: Array<Tick> = [] |
| 98 | const currentTicks: Array<Tick> = [] |
| 99 | const gate = Latch.makeUnsafe() |
| 100 | // Hold an execution only once the assignments are stable, as a held |
| 101 | // execution on a moving shard would keep that shard from being released. |
| 102 | let armed = false |
| 103 | let entered = 0 |
| 104 | const blockedExecution = Effect.fnUntraced( |
| 105 | function*(ticks: Array<Tick>) { |
| 106 | if (!armed) return |
| 107 | if (yield* recordTick(ticks)) { |
| 108 | entered++ |
| 109 | yield* gate.await |
| 110 | } |
| 111 | }, |
| 112 | Effect.uninterruptible |
| 113 | ) |
| 114 | const previousCron = ClusterCron.make({ |
| 115 | name: `previous-${backend}`, |
| 116 | cron: everySecond, |
| 117 | calculateNextRunFromPrevious: true, |
| 118 | execute: blockedExecution(previousTicks) |
| 119 | }) |
| 120 | const currentCron = ClusterCron.make({ |
| 121 | name: `current-${backend}`, |
| 122 | cron: everySecond, |
| 123 | execute: blockedExecution(currentTicks) |
| 124 | }) |
| 125 | const cluster = yield* make({ |
| 126 | backend, |
| 127 | config: testConfig, |
| 128 | entities: Layer.merge(previousCron, currentCron) |
| 129 | }) |
| 130 | yield* cluster.start(3) |
| 131 | yield* cluster.waitForStableAssignments() |
| 132 | armed = true |
| 133 | yield* cluster.waitUntil( |
| 134 | "Both cron executions did not enter their first run", |
| 135 | Effect.sync(() => entered === 2) |
| 136 | ) |
| 137 | const blockedAt = yield* Clock.currentTimeMillis |
| 138 | yield* cluster.waitUntil( |
| 139 | "The cron executions were not held across several scheduled instants", |
| 140 | Effect.map(Clock.currentTimeMillis, (now) => now >= blockedAt + 3_200), |
| 141 | "5 seconds" |
| 142 | ) |
| 143 | gate.openUnsafe() |
| 144 | yield* cluster.waitUntil( |
| 145 | "The cron jobs did not resume after their first execution", |
| 146 | Effect.sync(() => previousTicks.length >= 4 && currentTicks.length >= 4), |
| 147 | "12 seconds" |
| 148 | ) |
| 149 | |
| 150 | const previousSecond = DateTime.toEpochMillis(DateTime.makeUnsafe(previousTicks[1].scheduled)) |
| 151 | const currentSecond = DateTime.toEpochMillis(DateTime.makeUnsafe(currentTicks[1].scheduled)) |
| 152 | assertScheduledFromPrevious(everySecond, previousTicks) |
| 153 | assertScheduledFromExecutionTime(everySecond, currentTicks, 2) |
| 154 | assert.isAtLeast(previousTicks[1].at - previousSecond, 2_000) |
| 155 | assert.isAtMost(currentTicks[1].at - currentSecond, 1_000) |
| 156 | })) |
| 157 | |
| 158 | it.live(`${backend}: catches up or skips stale runs and preserves the schedule across restart`, () => |
| 159 | Effect.gen(function*() { |
| 160 | const catchUpTicks: Array<Tick> = [] |
| 161 | const skipTicks: Array<Tick> = [] |
| 162 | const catchUpCron = ClusterCron.make({ |
| 163 | name: `catch-up-${backend}`, |
| 164 | cron: everySecond, |
| 165 | calculateNextRunFromPrevious: true, |
| 166 | execute: recordTick(catchUpTicks) |
| 167 | }) |
| 168 | const skipCron = ClusterCron.make({ |
| 169 | name: `skip-stale-${backend}`, |
| 170 | cron: everySecond, |
| 171 | calculateNextRunFromPrevious: true, |
| 172 | skipIfOlderThan: Duration.millis(500), |
| 173 | execute: recordTick(skipTicks) |
| 174 | }) |
| 175 | const cluster = yield* make({ |
| 176 | backend, |
| 177 | config: testConfig, |
| 178 | entities: Layer.merge(catchUpCron, skipCron) |
| 179 | }) |
| 180 | const runners = yield* cluster.start(3) |
| 181 | yield* cluster.waitForStableAssignments() |
| 182 | yield* cluster.waitUntil( |
| 183 | "The cron schedules did not start", |
| 184 | Effect.sync(() => catchUpTicks.length >= 2 && skipTicks.length >= 2) |
| 185 | ) |
| 186 | for (const runner of runners) { |
| 187 | yield* cluster.kill(runner) |
| 188 | } |
| 189 | const catchUpBefore = catchUpTicks.length |
| 190 | const skipBefore = skipTicks.length |
| 191 | const lastScheduledBeforeRestart = catchUpTicks[catchUpBefore - 1].scheduled |
| 192 | const stoppedAt = yield* Clock.currentTimeMillis |
| 193 | yield* cluster.waitUntil( |
| 194 | "The cluster downtime window did not elapse", |
| 195 | Effect.map(Clock.currentTimeMillis, (now) => now >= stoppedAt + 3_200), |
| 196 | "5 seconds" |
| 197 | ) |
| 198 | assert.strictEqual(catchUpTicks.length, catchUpBefore) |
| 199 | assert.strictEqual(skipTicks.length, skipBefore) |
| 200 | |
| 201 | yield* cluster.start(3) |
| 202 | yield* cluster.waitForStableAssignments() |
| 203 | yield* cluster.waitUntil( |
| 204 | "The restarted cluster did not expose the catch-up and stale-skip difference", |
| 205 | Effect.sync(() => { |
| 206 | const caughtUp = catchUpTicks.length - catchUpBefore |
| 207 | const skipped = skipTicks.length - skipBefore |
| 208 | return caughtUp >= 3 && caughtUp >= skipped + 2 |
| 209 | }), |
| 210 | "15 seconds" |
| 211 | ) |
| 212 | yield* cluster.waitUntil( |
| 213 | "The stale-skipping cron did not resume", |
| 214 | Effect.sync(() => skipTicks.length > skipBefore) |
| 215 | ) |
| 216 | |
| 217 | const firstCatchUp = catchUpTicks[catchUpBefore] |
| 218 | const firstAfterSkip = skipTicks[skipBefore] |
| 219 | const firstAfterSkipScheduledAt = DateTime.toEpochMillis(DateTime.makeUnsafe(firstAfterSkip.scheduled)) |
| 220 | assert.strictEqual( |
| 221 | firstCatchUp.scheduled, |
| 222 | nextScheduled(everySecond, lastScheduledBeforeRestart) |
| 223 | ) |
| 224 | assert.isAtLeast(firstCatchUp.at - DateTime.toEpochMillis(DateTime.makeUnsafe(firstCatchUp.scheduled)), 2_000) |
| 225 | assert.isAtMost(firstAfterSkip.at - firstAfterSkipScheduledAt, 750) |
| 226 | assertScheduledFromPrevious(everySecond, catchUpTicks) |
| 227 | })) |
| 228 | |
| 229 | it.live(`${backend}: resumes without duplicate or missing ticks after the singleton owner dies`, () => |
| 230 | Effect.gen(function*() { |
| 231 | const name = `owner-failover-${backend}` |
| 232 | const ticks: Array<Tick> = [] |
| 233 | const cron = ClusterCron.make({ |
| 234 | name, |
| 235 | cron: everySecond, |
| 236 | calculateNextRunFromPrevious: true, |
| 237 | execute: recordTick(ticks) |
| 238 | }) |
| 239 | const cluster = yield* make({ backend, config: testConfig, entities: cron }) |
| 240 | yield* cluster.start(3) |
| 241 | yield* cluster.waitForStableAssignments() |
| 242 | yield* cluster.waitUntil( |
| 243 | "The cron did not execute before failover", |
| 244 | Effect.sync(() => ticks.length >= 2) |
| 245 | ) |
| 246 | const owner = yield* cluster.ownerOfEntity(cronProbe(name), name) |
| 247 | assert.isDefined(owner) |
| 248 | yield* cluster.kill(owner!) |
| 249 | const afterKill = ticks.length |
| 250 | yield* cluster.waitUntil( |
| 251 | "The cron did not resume after its singleton owner died", |
| 252 | Effect.sync(() => ticks.length >= afterKill + 3), |
| 253 | "12 seconds" |
| 254 | ) |
| 255 | yield* cluster.waitForStableAssignments() |
| 256 | |
| 257 | assertScheduledFromPrevious(everySecond, ticks) |
| 258 | assert.strictEqual(new Set(ticks.map((tick) => tick.scheduled)).size, ticks.length) |
| 259 | assert.isTrue(ticks.slice(afterKill).every((tick) => tick.runner !== addressString(owner!.address))) |
| 260 | })) |
| 261 | |
| 262 | it.live(`${backend}: assigns cron singletons and executions to their shard groups`, () => |
| 263 | Effect.gen(function*() { |
| 264 | const defaultName = `default-group-${backend}` |
| 265 | const specialName = `special-group-${backend}` |
| 266 | const defaultTicks: Array<Tick> = [] |
| 267 | const specialTicks: Array<Tick> = [] |
| 268 | const defaultCron = ClusterCron.make({ |
| 269 | name: defaultName, |
| 270 | cron: everySecond, |
| 271 | execute: recordTick(defaultTicks) |
| 272 | }) |
| 273 | const specialCron = ClusterCron.make({ |
| 274 | name: specialName, |
| 275 | cron: everySecond, |
| 276 | shardGroup: "special", |
| 277 | execute: recordTick(specialTicks) |
| 278 | }) |
| 279 | const cluster = yield* make({ |
| 280 | backend, |
| 281 | config: { |
| 282 | availableShardGroups: ["default", "special"], |
| 283 | shardsPerGroup: 12 |
| 284 | }, |
| 285 | entities: Layer.merge(defaultCron, specialCron) |
| 286 | }) |
| 287 | const [defaultRunner] = yield* cluster.start(1, { assignedShardGroups: ["default"] }) |
| 288 | const [specialRunner] = yield* cluster.start(1, { assignedShardGroups: ["special"] }) |
| 289 | yield* cluster.waitForStableAssignments() |
| 290 | yield* cluster.waitUntil( |
| 291 | "The shard-group cron jobs did not execute", |
| 292 | Effect.sync(() => defaultTicks.length >= 2 && specialTicks.length >= 2) |
| 293 | ) |
| 294 | |
| 295 | assert.strictEqual(yield* cluster.ownerOfEntity(cronProbe(defaultName), defaultName), defaultRunner) |
| 296 | assert.strictEqual( |
| 297 | yield* cluster.ownerOfEntity(cronProbe(specialName, "special"), specialName), |
| 298 | specialRunner |
| 299 | ) |
| 300 | assert.isTrue(defaultTicks.every((tick) => tick.runner === addressString(defaultRunner.address))) |
| 301 | assert.isTrue(specialTicks.every((tick) => tick.runner === addressString(specialRunner.address))) |
| 302 | })) |
| 303 | } |
| 304 | }) |