The best way to control Claude Code, Codex and more from any computer or phone

main
ts 532 lines 19.1 KB
Raw
1 import type * as cf from "@cloudflare/workers-types";
2 import {
3 CallbackError,
4 type Callback,
5 type CallbackOptions,
6 type CallbackScheduleOptions,
7 type CallbackFactory,
8 } from "alchemy/Callback";
9 import { RuntimeContext } from "alchemy/RuntimeContext";
10 import * as Cause from "effect/Cause";
11 import * as Clock from "effect/Clock";
12 import * as Context from "effect/Context";
13 import * as Duration from "effect/Duration";
14 import * as Effect from "effect/Effect";
15 import * as Exit from "effect/Exit";
16 import * as Logger from "effect/Logger";
17 import * as Schema from "effect/Schema";
18 import * as Scope from "effect/Scope";
19 import * as Tracer from "effect/Tracer";
20 import { ensureAlarmTables, reconcileDurableObjectAlarm } from "./DurableObjectAlarmStorage.ts";
21 import { DurableObjectState, fromDurableObjectState } from "./DurableObjectState.ts";
22 import { fromDurableObjectStorage } from "./DurableObjectStorage.ts";
23 import { ActiveStorageTransactions } from "./DurableObjectTransactionContext.ts";
24
25 type InvocationServices = RuntimeContext | DurableObjectState | Scope.Scope;
26 interface RetryPolicy {
27 readonly delay: number;
28 readonly maxAttempts: number;
29 readonly parkedDelay: number;
30 }
31 const defaultRetryPolicy: RetryPolicy = {
32 delay: 30_000,
33 maxAttempts: 8,
34 parkedDelay: 3_600_000,
35 };
36
37 const recoveryDelay = (policy: RetryPolicy, attempts: number, parked: boolean) =>
38 parked ? policy.parkedDelay : Math.min(policy.parkedDelay, policy.delay * 2 ** (attempts - 1));
39
40 interface RegisteredCallback {
41 readonly retry: RetryPolicy;
42 readonly handler: (payload: unknown) => Effect.Effect<unknown, unknown, InvocationServices>;
43 }
44 interface CallbackRegistry {
45 readonly callbacks: Map<string, RegisteredCallback>;
46 open: boolean;
47 }
48 type AlarmRow = {
49 callback: string;
50 id: string;
51 version: string;
52 run_at: number;
53 scheduled_at: number;
54 retry_at: number | null;
55 attempts: number;
56 parked: number;
57 progress: number;
58 payload: string;
59 };
60 type AlarmRef = Pick<AlarmRow, "callback" | "id" | "version">;
61 interface AlarmPass {
62 readonly storage: cf.DurableObjectStorage;
63 readonly row: AlarmRow;
64 readonly justParked: boolean;
65 readonly parking: AlarmRef[];
66 active: boolean;
67 }
68 const CurrentAlarmPass = Context.Reference<AlarmPass | undefined>(
69 "alchemy/Cloudflare/CurrentAlarmPass",
70 { defaultValue: () => undefined },
71 );
72 const readAlarm = (storage: cf.DurableObjectStorage, callback: string, id: string) =>
73 Effect.sync(
74 () =>
75 storage.sql
76 .exec<AlarmRow>(
77 "SELECT * FROM alchemy_alarm_callbacks WHERE callback = ? AND id = ?",
78 callback,
79 id,
80 )
81 .toArray()[0],
82 );
83
84 const activePass = (storage: cf.DurableObjectStorage, callback: string) =>
85 Effect.gen(function* () {
86 const current = yield* CurrentAlarmPass;
87 const pass = current?.storage === storage ? current : undefined;
88 if (pass !== undefined && !pass.active) {
89 return yield* new CallbackError({
90 callback,
91 message: "The alarm callback invocation has already finished",
92 });
93 }
94 return pass;
95 });
96
97 const reportParking = (pass: AlarmPass) =>
98 Effect.gen(function* () {
99 for (const ref of pass.parking) {
100 const row = yield* readAlarm(pass.storage, ref.callback, ref.id);
101 // A replacement, progress or rollback can remove tentative parking. Only
102 // report committed work, without exposing its identifiers or payload.
103 if (row?.version === ref.version && row.parked === 1) {
104 yield* Effect.logWarning("Durable Object alarm callback parked", {
105 attempts: row.attempts,
106 retryAt: row.retry_at,
107 }).pipe(Effect.exit);
108 }
109 }
110 });
111 const registries = new WeakMap<cf.DurableObjectState, CallbackRegistry>();
112
113 /**
114 * @internal Run background work handed to `waitUntil` apart from the callback
115 * invocation that started it. The invocation ends first, so its schedules
116 * would be rejected; here they start a fresh budget, as external schedules
117 * do. Only for work whose schedules report their own progress.
118 */
119 export const outsideAlarmPass = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
120 Effect.provideService(effect, CurrentAlarmPass, undefined);
121
122 /** @internal */
123 export const initializeAlarmCallbacks = (state: cf.DurableObjectState) => {
124 const registry: CallbackRegistry = { callbacks: new Map(), open: true };
125 registries.set(state, registry);
126 return () => {
127 registry.open = false;
128 };
129 };
130
131 /** @internal */
132 export const makeDurableObjectCallbackFactory = (raw: cf.DurableObjectState): CallbackFactory => {
133 const state = fromDurableObjectState(raw);
134 return (name, handler, options) => makeAlarmCallback(state, name, handler, options);
135 };
136
137 const makeAlarmCallback = <Payload, E, R>(
138 state: DurableObjectState["Service"],
139 name: string,
140 handler: (payload: Payload) => Effect.Effect<unknown, E, R>,
141 options?: CallbackOptions,
142 ): Effect.Effect<Callback<Payload>, never, RuntimeContext | Exclude<R, Scope.Scope>> =>
143 Effect.gen(function* () {
144 const context = (yield* Effect.context<Exclude<R, Scope.Scope>>()).pipe(
145 Context.omit(
146 ActiveStorageTransactions,
147 CurrentAlarmPass,
148 DurableObjectState,
149 RuntimeContext,
150 Scope.Scope,
151 Tracer.ParentSpan,
152 Tracer.Tracer,
153 Logger.CurrentLoggers,
154 ),
155 );
156 const registry = registries.get(state.raw);
157 if (!registry?.open || !name || registry.callbacks.has(name)) {
158 return yield* Effect.die(
159 new CallbackError({
160 callback: name,
161 message: !registry?.open
162 ? "Alarm callbacks must be registered during Durable Object instance initialization"
163 : "Alarm callback names must be non-empty and unique within an instance",
164 }),
165 );
166 }
167 const retry = yield* Effect.try({
168 try: () => {
169 const delay = Math.ceil(
170 Duration.toMillis(options?.retry?.delay ?? defaultRetryPolicy.delay),
171 );
172 const maxAttempts = options?.retry?.maxAttempts ?? defaultRetryPolicy.maxAttempts;
173 const parkedDelay = Math.ceil(
174 Duration.toMillis(
175 options?.retry?.parkedDelay ?? Math.max(defaultRetryPolicy.parkedDelay, delay),
176 ),
177 );
178 if (
179 !Number.isSafeInteger(delay) ||
180 delay <= 0 ||
181 !Number.isSafeInteger(maxAttempts) ||
182 maxAttempts < 1 ||
183 !Number.isSafeInteger(parkedDelay) ||
184 parkedDelay < 3_600_000 ||
185 parkedDelay < delay
186 ) {
187 throw new Error(
188 "Retry delay must be positive, maxAttempts a positive safe integer, and parkedDelay at least one hour and the retry delay",
189 );
190 }
191 return { delay: Math.max(1_000, delay), maxAttempts, parkedDelay };
192 },
193 catch: (cause) =>
194 new CallbackError({
195 callback: name,
196 message: "Invalid alarm retry policy",
197 cause,
198 }),
199 }).pipe(Effect.orDie);
200 yield* Effect.sync(() =>
201 registry.callbacks.set(name, {
202 retry,
203 handler: (payload) =>
204 Effect.gen(function* () {
205 const invocation = yield* Effect.context<InvocationServices>();
206 return yield* handler(payload as Payload).pipe(
207 Effect.provide(Context.merge(invocation, context) as Context.Context<R>),
208 );
209 }),
210 }),
211 );
212
213 const raw = state.raw.storage;
214 const transaction = <A, R>(effect: Effect.Effect<A, CallbackError, R>) =>
215 state.storage.transaction(effect).pipe(
216 Effect.mapError((cause) =>
217 cause._tag === "CallbackError"
218 ? cause
219 : new CallbackError({
220 callback: name,
221 message: "Callback storage transaction failed",
222 cause,
223 }),
224 ),
225 );
226 return {
227 schedule: Effect.fn(function* (id: string, schedule: CallbackScheduleOptions<Payload>) {
228 const now = yield* Clock.currentTimeMillis;
229 const { at, payload } = yield* Effect.try({
230 try: () => {
231 if (!id) throw new Error("Alarm IDs must be non-empty");
232 if ((schedule.at === undefined) === (schedule.after === undefined)) {
233 throw new Error("Specify exactly one of at or after");
234 }
235 const delay =
236 schedule.after === undefined ? undefined : Duration.toMillis(schedule.after);
237 const at =
238 schedule.at === undefined
239 ? now + delay!
240 : schedule.at instanceof Date
241 ? schedule.at.getTime()
242 : schedule.at;
243 if (
244 !Number.isFinite(at) ||
245 at <= 0 ||
246 (delay !== undefined && (!Number.isFinite(delay) || delay < 0))
247 ) {
248 throw new Error("Alarm time must be finite and delay must be non-negative");
249 }
250 if (!Schema.is(Schema.Json)(schedule.payload)) {
251 throw new Error("Alarm payload must be a JSON value");
252 }
253 if (
254 schedule.progress !== undefined &&
255 (!Number.isSafeInteger(schedule.progress) || schedule.progress < 0)
256 ) {
257 throw new Error("Alarm progress must be a nonnegative safe integer");
258 }
259 const payload = JSON.stringify(schedule.payload);
260 if (payload === undefined) throw new Error("Alarm payload must be JSON-serializable");
261 return { at, payload };
262 },
263 catch: (cause) =>
264 new CallbackError({
265 callback: name,
266 message: "Invalid alarm schedule",
267 cause,
268 }),
269 });
270 yield* transaction(
271 Effect.gen(function* () {
272 const pass = yield* activePass(raw, name);
273 yield* ensureAlarmTables(raw);
274 const existing = yield* readAlarm(raw, name, id);
275 const source = pass?.row;
276 // Another callback's cursor counts in its own units: it never hides
277 // this schedule, but a cursor that isn't past it isn't progress.
278 const progress = Math.max(
279 existing?.progress ?? -1,
280 source?.callback === name ? source.progress : -1,
281 );
282 if (
283 schedule.progress !== undefined &&
284 (schedule.progress < progress ||
285 (source === undefined && schedule.progress === progress))
286 )
287 return;
288 const progressed =
289 schedule.progress !== undefined &&
290 schedule.progress > Math.max(progress, source?.progress ?? -1);
291 const sourceProgressed =
292 source?.callback === name &&
293 source.id === id &&
294 (existing?.progress ?? -1) > source.progress;
295 const attempts =
296 source === undefined || progressed || sourceProgressed
297 ? 0
298 : Math.max(source.attempts, existing?.attempts ?? 0);
299 const parked =
300 attempts > 0 &&
301 (source?.parked === 1 || existing?.parked === 1 || attempts >= retry.maxAttempts);
302 const retryAt =
303 attempts === 0
304 ? null
305 : (yield* Clock.currentTimeMillis) + recoveryDelay(retry, attempts, parked);
306 const version = yield* Effect.sync(() => crypto.randomUUID());
307 yield* Effect.sync(() =>
308 raw.sql.exec(
309 `INSERT INTO alchemy_alarm_callbacks
310 (callback, id, version, run_at, payload, scheduled_at, retry_at, attempts, parked, progress)
311 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
312 ON CONFLICT (callback, id) DO UPDATE SET
313 version = excluded.version, run_at = excluded.run_at, payload = excluded.payload,
314 scheduled_at = excluded.scheduled_at, retry_at = excluded.retry_at,
315 attempts = excluded.attempts, parked = excluded.parked, progress = excluded.progress`,
316 name,
317 id,
318 version,
319 Math.max(at, retryAt ?? at),
320 payload,
321 at,
322 retryAt,
323 attempts,
324 parked ? 1 : 0,
325 Math.max(progress, schedule.progress ?? -1),
326 ),
327 );
328 if (
329 pass !== undefined &&
330 parked &&
331 (pass.justParked || (source?.parked !== 1 && existing?.parked !== 1))
332 ) {
333 const ref = { callback: name, id, version };
334 pass.parking.push(ref);
335 }
336 yield* reconcileDurableObjectAlarm(raw);
337 }),
338 );
339 }),
340 cancel: Effect.fn(function* (id: string) {
341 yield* transaction(
342 Effect.gen(function* () {
343 yield* activePass(raw, name);
344 yield* ensureAlarmTables(raw);
345 yield* Effect.sync(() =>
346 raw.sql.exec(
347 "DELETE FROM alchemy_alarm_callbacks WHERE callback = ? AND id = ?",
348 name,
349 id,
350 ),
351 );
352 yield* reconcileDurableObjectAlarm(raw);
353 }),
354 );
355 }),
356 getStatus: Effect.fn(function* (id: string) {
357 return yield* transaction(
358 Effect.gen(function* () {
359 yield* ensureAlarmTables(raw);
360 const row = yield* readAlarm(raw, name, id);
361 return row === undefined
362 ? undefined
363 : {
364 scheduledAt: row.scheduled_at,
365 retryAt: row.retry_at ?? undefined,
366 attempts: row.attempts,
367 parked: row.parked === 1,
368 progress: row.progress < 0 ? undefined : row.progress,
369 };
370 }),
371 );
372 }),
373 };
374 });
375
376 /** @internal */
377 export const dispatchAlarmCallbacks = (state: cf.DurableObjectState, hasLegacyHandler: boolean) =>
378 Effect.gen(function* () {
379 const registry = registries.get(state);
380 if (!registry) return;
381 const raw = state.storage;
382 if (registry.callbacks.size === 0) {
383 const hasJobs = yield* Effect.sync(
384 () =>
385 raw.sql
386 ?.exec(
387 "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('alchemy_alarm_callbacks', 'alchemy_alarm_schema', 'alchemy_scheduled_events')",
388 )
389 .toArray().length > 0,
390 );
391 if (!hasJobs) return;
392 }
393 const storage = fromDurableObjectStorage(raw);
394 yield* ensureAlarmTables(raw);
395 if (!hasLegacyHandler) {
396 const legacy = yield* Effect.sync(() =>
397 raw.sql.exec("SELECT id FROM alchemy_scheduled_events LIMIT 1").toArray(),
398 );
399 if (legacy.length > 0) {
400 return yield* new CallbackError({
401 callback: "scheduleEvent",
402 message: "Pending legacy events require an alarm handler calling processScheduledEvents",
403 });
404 }
405 }
406 const now = yield* Clock.currentTimeMillis;
407 const due = yield* Effect.sync(() =>
408 raw.sql
409 .exec<AlarmRow>(
410 `SELECT * FROM alchemy_alarm_callbacks
411 WHERE run_at <= ? ORDER BY run_at, callback, id LIMIT 100`,
412 now,
413 )
414 .toArray(),
415 );
416 for (const job of due) {
417 const callback = registry.callbacks.get(job.callback);
418 const retry = callback?.retry ?? defaultRetryPolicy;
419 const claimed = yield* storage.transaction(
420 Effect.gen(function* () {
421 const now = yield* Clock.currentTimeMillis;
422 const attempts =
423 job.parked === 1 ? job.attempts : Math.min(retry.maxAttempts, job.attempts + 1);
424 const parked = job.parked === 1 || attempts >= retry.maxAttempts;
425 const retryAt = now + recoveryDelay(retry, attempts, parked);
426 const claimed = yield* Effect.sync(
427 () =>
428 raw.sql.exec(
429 `UPDATE alchemy_alarm_callbacks
430 SET run_at = MAX(scheduled_at, ?), retry_at = ?, attempts = ?, parked = ?
431 WHERE callback = ? AND id = ? AND version = ?`,
432 retryAt,
433 retryAt,
434 attempts,
435 parked ? 1 : 0,
436 job.callback,
437 job.id,
438 job.version,
439 ).rowsWritten > 0,
440 );
441 yield* reconcileDurableObjectAlarm(raw);
442 return claimed
443 ? {
444 ...job,
445 run_at: Math.max(job.scheduled_at, retryAt),
446 retry_at: retryAt,
447 attempts,
448 parked: parked ? 1 : 0,
449 }
450 : undefined;
451 }),
452 );
453 if (!claimed) continue;
454 yield* storage.sync();
455 const pass: AlarmPass = {
456 storage: raw,
457 row: claimed,
458 justParked: job.parked === 0 && claimed.parked === 1,
459 parking: [],
460 active: true,
461 };
462 if (pass.justParked) pass.parking.push(claimed);
463 const result = yield* Effect.gen(function* () {
464 if (!callback) {
465 return yield* new CallbackError({
466 callback: job.callback,
467 message: "No handler is registered for a pending alarm",
468 });
469 }
470 const payload = yield* Effect.try({
471 try: () => JSON.parse(job.payload),
472 catch: (cause) =>
473 new CallbackError({
474 callback: job.callback,
475 message: "Invalid persisted alarm payload",
476 cause,
477 }),
478 });
479 yield* callback.handler(payload);
480 }).pipe(
481 Effect.provideService(CurrentAlarmPass, pass),
482 Effect.scoped,
483 Effect.ensuring(
484 Effect.sync(() => {
485 pass.active = false;
486 }),
487 ),
488 Effect.exit,
489 );
490 if (Exit.isFailure(result)) {
491 if (Cause.hasInterrupts(result.cause)) return yield* Effect.failCause(result.cause);
492 yield* storage.transaction(
493 Effect.gen(function* () {
494 // The pre-handler wake survives crashes. A completed failure starts
495 // its backoff here, even when the handler outlasted that recovery wake.
496 const retryAt =
497 (yield* Clock.currentTimeMillis) +
498 recoveryDelay(retry, claimed.attempts, claimed.parked === 1);
499 yield* Effect.sync(() =>
500 raw.sql.exec(
501 `UPDATE alchemy_alarm_callbacks SET run_at = MAX(scheduled_at, ?), retry_at = ?
502 WHERE callback = ? AND id = ? AND version = ?`,
503 retryAt,
504 retryAt,
505 job.callback,
506 job.id,
507 job.version,
508 ),
509 );
510 yield* reconcileDurableObjectAlarm(raw);
511 }),
512 );
513 yield* Effect.logError("Durable Object alarm callback failed", result.cause);
514 } else {
515 yield* storage.transaction(
516 Effect.gen(function* () {
517 yield* Effect.sync(() =>
518 raw.sql.exec(
519 "DELETE FROM alchemy_alarm_callbacks WHERE callback = ? AND id = ? AND version = ?",
520 job.callback,
521 job.id,
522 job.version,
523 ),
524 );
525 yield* reconcileDurableObjectAlarm(raw);
526 }),
527 );
528 }
529 yield* reportParking(pass);
530 }
531 yield* storage.transaction(reconcileDurableObjectAlarm(raw));
532 });