The best way to control Claude Code, Codex and more from any computer or phone

main
ts 809 lines 32.7 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import { inMemoryState } from "alchemy/State";
5 import * as Cause from "effect/Cause";
6 import type * as Duration from "effect/Duration";
7 import * as Effect from "effect/Effect";
8 import * as HttpClient from "effect/http/HttpClient";
9 import * as HttpClientRequest from "effect/http/HttpClientRequest";
10 import * as Schedule from "effect/Schedule";
11 import type { ScheduledRegressionResult } from "./fixtures/alarm-callback/legacy.ts";
12 import type {
13 AlarmObservationResult,
14 BatchResult,
15 BoundedSnapshot,
16 ExplicitRollbackResult,
17 FailedBatchResult,
18 RegistrationResult,
19 RollbackResult,
20 SiblingTransactionResult,
21 Snapshot,
22 } from "./fixtures/alarm-callback/object.ts";
23 import makeStack from "./fixtures/alarm-callback/stack.ts";
24
25 const requestJson = <T>(
26 url: string,
27 method: "GET" | "POST",
28 retryDelay: Duration.Input = "2 seconds",
29 ) =>
30 Effect.gen(function* () {
31 const client = HttpClient.mapRequest(
32 yield* HttpClient.HttpClient,
33 HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }),
34 );
35 const fresh = new URL(url);
36 fresh.searchParams.set("cb", yield* Effect.sync(() => String(Date.now())));
37 const response = yield* method === "POST" ? client.post(fresh.href) : client.get(fresh.href);
38 const body = yield* response.text;
39 if (response.status !== 200) {
40 const message = `${method} ${url}: HTTP ${response.status}: ${body}`;
41 if (
42 response.status === 404 ||
43 (response.status >= 500 && (method === "GET" || body.includes("<title>Script not found |")))
44 ) {
45 return yield* Effect.fail(
46 Object.assign(new Test.WorkerNotReady({ status: response.status }), {
47 message,
48 body,
49 url,
50 }),
51 );
52 }
53 return yield* Effect.fail(new Error(message));
54 }
55 return yield* Effect.try({
56 try: () => JSON.parse(body) as T,
57 catch: () => new Error(`${method} ${url}: Invalid JSON: ${body}`),
58 });
59 }).pipe(
60 Effect.timeout("5 seconds"),
61 Effect.retry({
62 while: (error) => error instanceof Test.WorkerNotReady,
63 schedule: Schedule.spaced(retryDelay),
64 times: 8,
65 }),
66 );
67
68 const json = <T>(url: string, method: "GET" | "POST" = "GET") =>
69 Effect.gen(function* () {
70 if (method === "POST") {
71 const snapshot = new URL(url);
72 snapshot.pathname = snapshot.pathname.replace(
73 /\/[^/]+$/,
74 snapshot.pathname.endsWith("/legacy") || snapshot.pathname.endsWith("/scheduled-regression")
75 ? "/legacy"
76 : "/snapshot",
77 );
78 snapshot.search = "";
79 yield* requestJson(snapshot.href, "GET");
80 }
81 return yield* requestJson<T>(url, method);
82 });
83
84 const poll = <T>(
85 url: string,
86 until: (value: T) => boolean,
87 interval: "2 seconds" | "4 seconds" = "2 seconds",
88 ) =>
89 json<T>(url).pipe(
90 Effect.repeat({ schedule: Schedule.spaced(interval), until, times: 10 }),
91 Effect.flatMap((snapshot) =>
92 until(snapshot)
93 ? Effect.succeed(snapshot)
94 : Effect.fail(new Error(`Alarm polling exhausted for ${url}: ${JSON.stringify(snapshot)}`)),
95 ),
96 );
97
98 const deliveries = (snapshot: Snapshot) =>
99 snapshot.deliveries.map(({ callback, value }) => `${callback}:${value}`).sort();
100
101 const drained = (count: number) => (snapshot: Snapshot) =>
102 snapshot.deliveries.length === count && snapshot.alarm === null;
103
104 const assertRollback = (snapshot: Snapshot) => {
105 expect(snapshot.application).toBeNull();
106 expect(snapshot.cleanupWrite).toBeNull();
107 expect(snapshot.rows).toEqual([]);
108 };
109
110 describe.concurrent.each([
111 { dev: true, stage: `${Test.defaultStage()}-alarm-callback-local` },
112 { dev: false, stage: `${Test.defaultStage()}-alarm-callback-live` },
113 ])(
114 "makeCallback (dev: $dev)",
115 ({ dev, stage }) => {
116 const state = dev ? inMemoryState() : Cloudflare.state();
117 const Stack = makeStack(state);
118 const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
119 providers: Cloudflare.providers(),
120 state,
121 dev,
122 stage,
123 });
124
125 const stack = beforeAll(
126 Effect.gen(function* () {
127 yield* destroy(Stack);
128 const output = yield* deploy(Stack);
129 yield* Effect.all([
130 requestJson<Snapshot>(`${output.url}/readiness/snapshot`, "GET", "4 seconds"),
131 requestJson(`${output.url}/readiness/legacy`, "GET", "4 seconds"),
132 ]).pipe(
133 Effect.retry({
134 while: Cause.isTimeoutError,
135 schedule: Schedule.spaced("1 second"),
136 times: 3,
137 }),
138 );
139 expect(output.url.startsWith("http://localhost:")).toBe(dev);
140 return output;
141 }),
142 { timeout: 120_000 },
143 );
144 afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack), { timeout: 30_000 });
145
146 test(
147 "Worker callback registration stays unsupported after a Durable Object call",
148 Effect.gen(function* () {
149 const { url } = yield* stack;
150 const result = yield* json<RegistrationResult & { runtimeType: string }>(
151 `${url}/worker-registration/worker-registration`,
152 "POST",
153 );
154 expect(result.snapshot.boots).toBeGreaterThan(0);
155 expect(result.runtimeType).toBe("Cloudflare.Worker");
156 expect(result.failure).toEqual({
157 tag: "CallbackError",
158 callback: "unsupported-worker",
159 message: `Durable callbacks are not supported by ${result.runtimeType}`,
160 });
161 expect(result.snapshot.pendingJobs).toEqual([]);
162 expect(result.snapshot.alarm).toBeNull();
163 }),
164 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
165 );
166
167 test(
168 "late DO registration defects without disabling initialized callbacks",
169 Effect.gen(function* () {
170 const { url } = yield* stack;
171 const base = `${url}/late-registration`;
172 const rejected = yield* json<RegistrationResult>(`${base}/late-registration`, "POST");
173 expect(rejected.failure?.tag).toBe("CallbackError");
174 expect(rejected.failure?.callback).toBe("late");
175 expect(rejected.failure?.message).toContain("instance initialization");
176 expect(rejected.snapshot.pendingJobs).toEqual([]);
177 expect(rejected.snapshot.deliveries).toEqual([]);
178 expect(rejected.snapshot.alarm).toBeNull();
179 const scheduled = yield* json<Snapshot>(`${base}/timing`, "POST");
180 expect(scheduled.id).toBe(rejected.snapshot.id);
181 expect(scheduled.alarm).not.toBeNull();
182 const delivered = yield* poll<Snapshot>(`${base}/snapshot`, drained(5));
183 expect(delivered.id).toBe(rejected.snapshot.id);
184 expect(deliveries(delivered)).toEqual([
185 "archive:checkpoint",
186 "archive:date",
187 "archive:duration",
188 "archive:latest",
189 "secondary:other-callback",
190 ]);
191 expect(delivered.pendingJobs).toEqual([]);
192 }),
193 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
194 );
195
196 test(
197 "batches native alarm bookkeeping across nested callbacks, legacy mutations, and finalizers",
198 Effect.gen(function* () {
199 const { url } = yield* stack;
200 const base = `${url}/bookkeeping`;
201 const result = yield* json<BatchResult>(`${base}/bookkeeping`, "POST");
202 yield* Effect.logInfo("Native alarm bookkeeping", result.counts);
203 expect(result.counts).toEqual({
204 schemaChecks: 1,
205 reconciliations: 1,
206 setAlarm: 1,
207 deleteAlarm: 0,
208 });
209 expect(result.snapshot.pendingJobs).toHaveLength(3);
210 expect(result.snapshot.alarm).not.toBeNull();
211 const delivered = yield* poll<Snapshot>(`${base}/snapshot`, drained(3));
212 expect(deliveries(delivered)).toEqual([
213 "archive:batched",
214 "archive:finalizer",
215 "secondary:nested",
216 ]);
217 }),
218 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
219 );
220
221 test(
222 "batches native alarm deletion for repeated cancellation",
223 Effect.gen(function* () {
224 const { url } = yield* stack;
225 const result = yield* json<BatchResult>(
226 `${url}/cancel-bookkeeping/cancel-bookkeeping`,
227 "POST",
228 );
229 yield* Effect.logInfo("Native cancellation bookkeeping", result.counts);
230 expect(result.counts).toEqual({
231 schemaChecks: 1,
232 reconciliations: 1,
233 setAlarm: 0,
234 deleteAlarm: 1,
235 });
236 expect(result.snapshot.pendingJobs).toEqual([]);
237 expect(result.snapshot.alarm).toBeNull();
238 }),
239 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
240 );
241
242 for (const explicit of [true, false]) {
243 test(
244 `${explicit ? "explicit rollback" : "native reconciliation failure"} discards deferred bookkeeping and retries schema initialization`,
245 Effect.gen(function* () {
246 const { url } = yield* stack;
247 const operation = explicit ? "bookkeeping-rollback" : "bookkeeping-failure";
248 const base = `${url}/${operation}`;
249 const result = yield* json<FailedBatchResult>(`${base}/${operation}`, "POST");
250 if (explicit) expect(result.failure).toBeNull();
251 else expect(result.failure).toContain("no such table: alchemy_alarm_callbacks");
252 expect(result.rolledBack.counts).toEqual({
253 schemaChecks: 1,
254 reconciliations: explicit ? 0 : 1,
255 setAlarm: 0,
256 deleteAlarm: 0,
257 });
258 assertRollback(result.rolledBack.snapshot);
259 expect(result.rolledBack.snapshot.pendingJobs).toEqual([]);
260 expect(result.rolledBack.snapshot.alarm).toBeNull();
261 expect(result.recovered.counts).toEqual({
262 schemaChecks: 1,
263 reconciliations: 1,
264 setAlarm: 1,
265 deleteAlarm: 0,
266 });
267 expect(result.recovered.snapshot.pendingJobs).toHaveLength(1);
268 expect(result.recovered.snapshot.alarm).not.toBeNull();
269 const delivered = yield* poll<Snapshot>(`${base}/snapshot`, drained(1));
270 expect(deliveries(delivered)).toEqual(["archive:recovered"]);
271 assertRollback(delivered);
272 }),
273 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
274 );
275 }
276
277 test(
278 "preserves getAlarm observations and explicit native alarm write ordering in transactions",
279 Effect.gen(function* () {
280 const { url } = yield* stack;
281 const { at, observations, committed } = yield* json<AlarmObservationResult>(
282 `${url}/alarm-observations/alarm-observations`,
283 "POST",
284 );
285 expect(observations).toEqual([at, null, at + 1_000, at + 2_000, null, null]);
286 expect(committed).toBe(at + 3_000);
287 }),
288 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
289 );
290
291 test(
292 "native alarms deliver typed payloads, overwrite by callback and ID, and cancel",
293 Effect.gen(function* () {
294 const { url } = yield* stack;
295 const initial = yield* json<Snapshot>(`${url}/timing/timing`, "POST");
296 expect(initial.alarm).not.toBeNull();
297 const result = yield* poll<Snapshot>(`${url}/timing/snapshot`, drained(5));
298 expect(deliveries(result)).toEqual([
299 "archive:checkpoint",
300 "archive:date",
301 "archive:duration",
302 "archive:latest",
303 "secondary:other-callback",
304 ]);
305 expect(result.alarm).toBeNull();
306 }),
307 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
308 );
309
310 test(
311 "commits application SQL, KV, and a scheduled callback in one transaction",
312 Effect.gen(function* () {
313 const { url } = yield* stack;
314 const initial = yield* json<{ marker: string; legacyOverload: string; snapshot: Snapshot }>(
315 `${url}/atomic/atomic`,
316 "POST",
317 );
318 expect(initial.marker).toBe("committed");
319 expect(initial.legacyOverload).toBe("callback");
320 expect(initial.snapshot.application).toBe("committed");
321 expect(initial.snapshot.rows).toEqual([{ value: "committed" }]);
322 expect(initial.snapshot.alarm).not.toBeNull();
323 const result = yield* poll<Snapshot>(`${url}/atomic/snapshot`, drained(1));
324 expect(result.deliveries).toEqual([
325 {
326 callback: "archive",
327 value: "committed",
328 application: "committed",
329 boots: result.boots,
330 },
331 ]);
332 expect(result.alarm).toBeNull();
333 }),
334 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
335 );
336
337 test(
338 "delivers a callback registered during transactional initialization",
339 Effect.gen(function* () {
340 const { url } = yield* stack;
341 const base = `${url}/transactional-registration`;
342 const initial = yield* json<Snapshot>(`${base}/transactional-registration`, "POST");
343 expect(initial.pendingJobs).toHaveLength(1);
344 expect(initial.pendingJobs[0]?.callback).toBe("transactional-init");
345 expect(initial.alarm).not.toBeNull();
346 const result = yield* poll<Snapshot>(`${base}/snapshot`, drained(1));
347 expect(result.id).toBe(initial.id);
348 expect(result.application).toBe("registered-in-transaction");
349 expect(result.deliveries).toEqual([
350 {
351 callback: "transactional-init",
352 value: "registered-in-transaction",
353 application: "registered-in-transaction",
354 boots: result.boots,
355 },
356 ]);
357 expect(result.pendingJobs).toEqual([]);
358 expect(result.alarm).toBeNull();
359 }),
360 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
361 );
362
363 test(
364 "rejects a pre-existing sibling fiber write during another fiber's transaction",
365 Effect.gen(function* () {
366 const { url } = yield* stack;
367 const result = yield* json<SiblingTransactionResult>(
368 `${url}/sibling-transaction/sibling-transaction`,
369 "POST",
370 );
371 expect(result.failure).toBe("sibling-rollback");
372 expect(result.siblingAcknowledged).toBe(false);
373 expect(result.siblingFailure?.tag).toBe("DurableObjectStorageError");
374 expect(result.siblingValue).toBeNull();
375 assertRollback(result.snapshot);
376 expect(result.snapshot.pendingJobs).toEqual([]);
377 expect(result.snapshot.alarm).toBeNull();
378 }),
379 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
380 );
381
382 test(
383 "explicit rollback rejects later storage writes and callback scheduling while repeated rollback succeeds",
384 Effect.gen(function* () {
385 const { url } = yield* stack;
386 const base = `${url}/rollback-explicit`;
387 const result = yield* json<ExplicitRollbackResult>(`${base}/rollback-explicit`, "POST");
388 expect(result.repeatedRollbackSucceeded).toBe(true);
389 expect(result.operations.map(({ operation }) => operation)).toEqual([
390 "put",
391 "sql",
392 "schedule",
393 ]);
394 for (const operation of result.operations) {
395 expect(operation.acknowledged).toBe(false);
396 expect(operation.failure?.tag).toBe("DurableObjectStorageError");
397 if (operation.operation !== "schedule") {
398 expect(operation.failure?.wrappedBy).toBeNull();
399 }
400 }
401 assertRollback(result.snapshot);
402 expect(result.snapshot.deliveries).toEqual([]);
403 expect(result.snapshot.pendingJobs).toEqual([]);
404 expect(result.snapshot.alarm).toBeNull();
405 const persisted = yield* json<Snapshot>(`${base}/snapshot`);
406 assertRollback(persisted);
407 expect(persisted.deliveries).toEqual([]);
408 expect(persisted.pendingJobs).toEqual([]);
409 expect(persisted.alarm).toBeNull();
410 }),
411 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
412 );
413
414 for (const [kind, failure] of [
415 ["typed", "typed-value"],
416 ["defect", "defect"],
417 ["interrupt", "interrupted"],
418 ] as const) {
419 test(
420 `${kind} failure rolls back SQL, KV, scheduling, cancellation, and the native wake`,
421 Effect.gen(function* () {
422 const { url } = yield* stack;
423 const base = `${url}/rollback-${kind}`;
424 const result = yield* json<RollbackResult>(`${base}/rollback-${kind}`, "POST");
425 expect(result.failure).toBe(failure);
426 expect(result.cleanupWaited).toBe(true);
427 expect(result.alarmBefore).not.toBeNull();
428 expect(result.alarmAfter).toBe(result.alarmBefore);
429 assertRollback(result.snapshot);
430 const after = yield* poll<Snapshot>(`${base}/snapshot`, drained(2));
431 assertRollback(after);
432 expect(deliveries(after)).toEqual(["archive:checkpoint", "archive:kept"]);
433 expect(after.alarm).toBeNull();
434 }),
435 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
436 );
437 }
438
439 test(
440 "persists a recovery wake before a fallible handler and acknowledges only success",
441 Effect.gen(function* () {
442 const { url } = yield* stack;
443 yield* json(`${url}/retry/retry`, "POST");
444 const result = yield* poll<Snapshot>(`${url}/retry/snapshot`, drained(1));
445 expect(deliveries(result)).toEqual(["retry:retried"]);
446 expect(result.attempts).toHaveLength(2);
447 for (const attempt of result.attempts) {
448 expect(attempt.recovery).not.toBeNull();
449 expect(attempt.recovery!).toBeGreaterThan(attempt.now);
450 }
451 expect(result.alarm).toBeNull();
452 }),
453 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
454 );
455
456 for (const mode of ["failure", "self-rearm"] as const) {
457 test(
458 `parks ${mode} callbacks across reconstruction and supports explicit recovery`,
459 Effect.gen(function* () {
460 const { url } = yield* stack;
461 const base = `${url}/bounded-${mode}`;
462 const initial = yield* json<BoundedSnapshot>(`${base}/bounded?mode=${mode}`, "POST");
463 expect(initial.status?.attempts).toBe(0);
464 const parked = yield* poll<BoundedSnapshot>(
465 `${base}/bounded?mode=${mode}`,
466 (value) => value.status?.parked === true,
467 );
468 expect(parked.status?.attempts).toBe(2);
469 expect(parked.snapshot.alarm).toBe(parked.status?.retryAt);
470 expect(parked.status!.retryAt! - Date.now()).toBeGreaterThan(3_500_000);
471 if (mode === "failure")
472 expect(parked.status?.scheduledAt).toBe(initial.status?.scheduledAt);
473 expect((yield* json<{ aborted: boolean }>(`${base}/abort`, "POST")).aborted).toBe(true);
474 const restored = yield* json<BoundedSnapshot>(`${base}/bounded?mode=${mode}`);
475 expect(restored.snapshot.boots).toBeGreaterThan(parked.snapshot.boots);
476 expect(restored.status).toEqual(parked.status);
477 expect(restored.snapshot.alarm).toBe(parked.snapshot.alarm);
478 const resumed = yield* json<BoundedSnapshot>(
479 `${base}/bounded-resume?mode=${mode}`,
480 "POST",
481 );
482 expect(resumed.status?.attempts).toBe(0);
483 expect(resumed.status?.parked).toBe(false);
484 expect(resumed.status?.retryAt).toBeUndefined();
485 const cancelled = yield* json<BoundedSnapshot>(
486 `${base}/bounded-cancel?mode=${mode}`,
487 "POST",
488 );
489 expect(cancelled.status).toBeNull();
490 expect(cancelled.snapshot.alarm).toBeNull();
491 }),
492 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
493 );
494 }
495
496 test(
497 "acknowledging a callback does not delete its same-ID replacement",
498 Effect.gen(function* () {
499 const { url } = yield* stack;
500 yield* json(`${url}/replacement/replace`, "POST");
501 const result = yield* poll<Snapshot>(`${url}/replacement/snapshot`, drained(2));
502 expect(deliveries(result)).toEqual(["archive:replace-first", "archive:replace-second"]);
503 expect(result.alarm).toBeNull();
504 }),
505 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
506 );
507
508 test(
509 "reconstructs instance-local registrations after abort without mixing object IDs",
510 Effect.gen(function* () {
511 const { url } = yield* stack;
512 const [first, second] = yield* Effect.all(
513 [
514 json<Snapshot>(`${url}/reset-first/reset?value=first`, "POST"),
515 json<Snapshot>(`${url}/reset-second/reset?value=second`, "POST"),
516 ],
517 { concurrency: "unbounded" },
518 );
519 expect(first.id).not.toBe(second.id);
520 // Pending jobs must survive a delayed abort.
521 yield* Effect.sleep("5 seconds");
522 const aborted = yield* json<{ aborted: boolean }>(`${url}/reset-first/abort`, "POST");
523 expect(aborted.aborted).toBe(true);
524 const reconstructed = yield* json<Snapshot>(`${url}/reset-first/snapshot`);
525 expect(reconstructed.boots).toBeGreaterThan(first.boots);
526 expect(reconstructed.pendingJobs).toEqual(first.pendingJobs);
527 expect(reconstructed.deliveries).toEqual([]);
528 yield* Effect.all(
529 [
530 json<Snapshot>(`${url}/reset-first/release-pending`, "POST"),
531 json<Snapshot>(`${url}/reset-second/release-pending`, "POST"),
532 ],
533 { concurrency: "unbounded" },
534 );
535 const [afterFirst, afterSecond] = yield* Effect.all(
536 [
537 poll<Snapshot>(`${url}/reset-first/snapshot`, drained(1)),
538 poll<Snapshot>(`${url}/reset-second/snapshot`, drained(1)),
539 ],
540 { concurrency: "unbounded" },
541 );
542 expect(afterFirst.id).toBe(first.id);
543 expect(afterFirst.boots).toBeGreaterThan(first.boots);
544 expect(afterFirst.deliveries[0]?.boots).toBe(afterFirst.boots);
545 expect(deliveries(afterFirst)).toEqual(["archive:first"]);
546 expect(deliveries(afterSecond)).toEqual(["archive:second"]);
547 expect(afterFirst.alarm).toBeNull();
548 expect(afterSecond.alarm).toBeNull();
549 }),
550 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
551 );
552
553 test(
554 "recovers a callback after the handler aborts with native alarm retries enabled",
555 Effect.gen(function* () {
556 const { url } = yield* stack;
557 yield* json(`${url}/recovery/recovery`, "POST");
558 const result = yield* poll<Snapshot>(`${url}/recovery/snapshot`, drained(1), "4 seconds");
559 yield* Effect.logInfo("Alarm crash recovery snapshot", result);
560 expect(deliveries(result)).toEqual(["crash:recovered"]);
561 expect(result.pendingJobs).toEqual([]);
562 expect(result.attempts).toHaveLength(2);
563 expect(result.attempts[1]!.boots).toBeGreaterThan(result.attempts[0]!.boots);
564 for (const attempt of result.attempts) {
565 expect(attempt.recovery).not.toBeNull();
566 expect(attempt.recovery!).toBeGreaterThan(attempt.now);
567 }
568 expect(result.alarm).toBeNull();
569 }),
570 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
571 );
572
573 test(
574 "retains an aborted callback with native retries disabled and recovers after an explicit wake",
575 Effect.gen(function* () {
576 const { url } = yield* stack;
577 const base = `${url}/recovery-no-retry`;
578 yield* json(`${base}/recovery-no-retry`, "POST");
579 const retained = yield* poll<Snapshot>(
580 `${base}/snapshot`,
581 (snapshot) => snapshot.attempts.length > 0,
582 );
583 expect(retained.attempts[0]!.recovery).not.toBeNull();
584 if (retained.deliveries.length === 0) {
585 expect(retained.pendingJobs).toHaveLength(1);
586 expect(retained.pendingJobs[0]!.callback).toBe("crash");
587 expect(retained.pendingJobs[0]!.id).toBe("recovery");
588 yield* json(`${base}/wake`, "POST");
589 }
590 const recovered = yield* poll<Snapshot>(`${base}/snapshot`, drained(1));
591 expect(deliveries(recovered)).toEqual(["crash:recovered"]);
592 expect(recovered.pendingJobs).toEqual([]);
593 expect(recovered.attempts).toHaveLength(2);
594 expect(recovered.attempts[1]!.boots).toBeGreaterThan(recovered.attempts[0]!.boots);
595 }),
596 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
597 );
598
599 test(
600 "retains an unknown callback until its registration is restored",
601 Effect.gen(function* () {
602 const { url } = yield* stack;
603 const initial = yield* json<Snapshot>(`${url}/unknown/optional`, "POST");
604 expect(initial.alarm).not.toBeNull();
605 yield* Effect.sleep("5 seconds");
606 expect((yield* json<{ aborted: boolean }>(`${url}/unknown/abort`, "POST")).aborted).toBe(
607 true,
608 );
609 const reconstructed = yield* json<Snapshot>(`${url}/unknown/snapshot`);
610 expect(reconstructed.boots).toBeGreaterThan(initial.boots);
611 expect(reconstructed.pendingJobs).toEqual(initial.pendingJobs);
612 expect(reconstructed.deliveries).toEqual([]);
613 const released = yield* json<Snapshot>(`${url}/unknown/release-pending`, "POST");
614 const pending = yield* poll<Snapshot>(
615 `${url}/unknown/snapshot`,
616 (value) => value.alarm !== null && value.alarm > released.alarm!,
617 );
618 expect(pending.boots).toBeGreaterThan(initial.boots);
619 expect(pending.deliveries).toEqual([]);
620 expect(pending.alarm).not.toBeNull();
621 expect(pending.alarm!).toBeGreaterThan(released.alarm!);
622 expect(pending.pendingJobs).toEqual(
623 released.pendingJobs.map((job) => ({ ...job, run_at: pending.alarm })),
624 );
625 yield* json(`${url}/unknown/enable-optional`, "POST");
626 expect((yield* json<{ aborted: boolean }>(`${url}/unknown/abort`, "POST")).aborted).toBe(
627 true,
628 );
629 const restored = yield* poll<Snapshot>(`${url}/unknown/snapshot`, drained(1), "4 seconds");
630 expect(deliveries(restored)).toEqual(["optional:retained"]);
631 expect(restored.boots).toBeGreaterThan(pending.boots);
632 expect(restored.alarm).toBeNull();
633 }),
634 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
635 );
636
637 test(
638 "drains a backlog over native alarm wakes without losing or duplicating jobs",
639 Effect.gen(function* () {
640 const { url } = yield* stack;
641 yield* json(`${url}/batch/batch`, "POST");
642 const result = yield* poll<Snapshot>(`${url}/batch/snapshot`, drained(105));
643 expect(deliveries(result)).toEqual(
644 Array.from({ length: 105 }, (_, i) => `archive:batch-${i}`).sort(),
645 );
646 expect(result.alarm).toBeNull();
647 }),
648 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
649 );
650
651 test(
652 "legacy scheduling and its native alarm handler coexist with registered callbacks",
653 Effect.gen(function* () {
654 const { url } = yield* stack;
655 yield* json(`${url}/legacy/legacy`, "POST");
656 const result = yield* poll<{
657 registered: string | null;
658 legacy: { id: string; payload: { value: string } }[];
659 pending: unknown[];
660 alarm: number | null;
661 }>(
662 `${url}/legacy/legacy`,
663 (value) => value.registered !== null && value.legacy.length === 1 && value.alarm === null,
664 );
665 expect(result.registered).toBe("registered");
666 expect(result.legacy).toHaveLength(1);
667 expect(result.legacy[0]?.id).toBe("shared-id");
668 expect(result.legacy[0]?.payload).toEqual({ value: "legacy" });
669 expect(result.pending).toEqual([]);
670 expect(result.alarm).toBeNull();
671 }),
672 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
673 );
674
675 for (const count of [100, 1_000]) {
676 for (const kind of ["oneshot", "repeat", "mixed"] as const) {
677 test(
678 `scheduled events process all ${count} ${kind} rows in order on native SQL`,
679 Effect.gen(function* () {
680 const { url } = yield* stack;
681 const result = yield* json<ScheduledRegressionResult>(
682 `${url}/scheduled-${kind}-${count}/scheduled-regression?scenario=${kind}&count=${count}`,
683 "POST",
684 );
685 expect(result.failure).toBeNull();
686 expect(result.before.rows).toHaveLength(count);
687 expect(result.fired).toEqual(
688 Array.from({ length: count }, (_, i) => {
689 const id = `event-${String(i).padStart(4, "0")}`;
690 return {
691 id,
692 runAt: result.now - count + i,
693 repeatMs:
694 kind === "repeat" || (kind === "mixed" && i % 2 === 1) ? 86_400_000 : null,
695 payload: { id },
696 };
697 }),
698 );
699 const repeating = result.fired.filter((event) => event.repeatMs !== null);
700 expect(result.after.rows).toEqual(
701 repeating.map((event) => ({
702 id: event.id,
703 run_at: result.now + 86_400_000,
704 repeat_ms: 86_400_000,
705 payload: JSON.stringify(event.payload),
706 })),
707 );
708 expect(result.after.alarm).toBe(repeating.length > 0 ? result.now + 86_400_000 : null);
709 }),
710 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
711 );
712 }
713 }
714
715 test(
716 "scheduled events preserve the due snapshot, fractional repeats, future rows and next alarm",
717 Effect.gen(function* () {
718 const { url } = yield* stack;
719 const result = yield* json<ScheduledRegressionResult>(
720 `${url}/scheduled-snapshot/scheduled-regression?scenario=snapshot`,
721 "POST",
722 );
723 expect(result.failure).toBeNull();
724 expect(result.fired.map((event) => event.id)).toEqual([
725 "one-shot",
726 "negative",
727 "fractional",
728 "zero",
729 ]);
730 expect(result.fired.map((event) => event.runAt)).toEqual([
731 result.now - 5,
732 result.now - 3,
733 result.now - 2,
734 result.now,
735 ]);
736 expect(result.fired.map((event) => event.payload)).toEqual(
737 result.fired.map((event) => ({ id: event.id })),
738 );
739 expect(result.after.rows.map(({ id, run_at }) => [id, run_at])).toEqual([
740 ["fractional", result.now + 0.25],
741 ["future", result.now + 10],
742 ["negative", result.now - 3],
743 ["zero", result.now],
744 ]);
745 expect(result.after.alarm).toBe(result.now + 1);
746 }),
747 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
748 );
749
750 for (const scenario of ["empty", "future"] as const) {
751 test(
752 `scheduled events leave ${scenario} rows unchanged and reconcile the native alarm`,
753 Effect.gen(function* () {
754 const { url } = yield* stack;
755 const result = yield* json<ScheduledRegressionResult>(
756 `${url}/scheduled-${scenario}/scheduled-regression?scenario=${scenario}`,
757 "POST",
758 );
759 expect(result.failure).toBeNull();
760 expect(result.fired).toEqual([]);
761 expect(result.after.rows).toEqual(result.before.rows);
762 expect(result.after.alarm).toBe(scenario === "empty" ? null : result.now + 60_000);
763 }),
764 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
765 );
766 }
767
768 test(
769 "scheduled events preserve JavaScript addition through native SQLite numeric affinity",
770 Effect.gen(function* () {
771 const { url } = yield* stack;
772 const result = yield* json<ScheduledRegressionResult>(
773 `${url}/scheduled-arithmetic/scheduled-regression?scenario=arithmetic`,
774 "POST",
775 );
776 expect(result.failure).toBeNull();
777 expect(result.fired).toHaveLength(9);
778 expect(result.after.rows).toHaveLength(9);
779 expect(result.actual).toEqual(result.expected);
780 expect(result.after.alarm).toBe(result.now + 1);
781 }),
782 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
783 );
784
785 for (const scenario of ["malformed", "sql-failure"] as const) {
786 test(
787 `scheduled events roll back native SQL, KV and alarm state after ${scenario}`,
788 Effect.gen(function* () {
789 const { url } = yield* stack;
790 const result = yield* json<ScheduledRegressionResult>(
791 `${url}/scheduled-${scenario}/scheduled-regression?scenario=${scenario}`,
792 "POST",
793 );
794 expect(result.failure).not.toBeNull();
795 if (scenario === "sql-failure") {
796 expect(result.failure).toContain("no such table: alchemy_alarm_callbacks");
797 }
798 expect(result.fired).toEqual([]);
799 expect(result.before.rows).toHaveLength(3);
800 expect(result.before.marker).toBeNull();
801 expect(result.before.callbackTablePresent).toBe(true);
802 expect(result.after).toEqual(result.before);
803 }),
804 { tags: [...(dev ? ["local"] : ["live"])], timeout: 90_000 },
805 );
806 }
807 },
808 { tags: ["provider:cloudflare", "provider:cloudflare:worker"] },
809 );