The best way to control Claude Code, Codex and more from any computer or phone

main
ts 2,356 lines 103 KB
Raw
1 import type * as Aggregate from "@alchemy.run/domain/Aggregate";
2 import { AlreadyCancelled, AlreadyExecuted, NotScheduled } from "@alchemy.run/domain/Aggregate";
3 import * as Backfill from "@alchemy.run/domain/Backfill";
4 import { type Routed, targetId } from "@alchemy.run/domain/CatchUp";
5 import { isAfter, mergeCheckpoints } from "@alchemy.run/domain/InMemory";
6 import {
7 type Accepted,
8 type AggregateKit,
9 captureContext,
10 type Delivery,
11 emptySnapshot,
12 emptyStreamMeta,
13 type HostedModel,
14 hostedModels,
15 type ModelHost,
16 Platform,
17 type Json,
18 type PolicyKit,
19 type ProjectionKit,
20 type Push,
21 type SendResult,
22 type StoredEvent,
23 type StreamEntry,
24 type StreamKit,
25 type StreamMeta,
26 type StreamWrite,
27 Subscription,
28 type ViewEntry,
29 type ViewKit,
30 type ViewSnapshot,
31 } from "@alchemy.run/domain/Platform";
32 import { makeCallback } from "alchemy/Callback";
33 import type { RuntimeContext } from "alchemy/RuntimeContext";
34 import { StorageName } from "alchemy/Workflows";
35 import * as Cause from "effect/Cause";
36 import * as Clock from "effect/Clock";
37 import * as Context from "effect/Context";
38 import * as Data from "effect/Data";
39 import * as Duration from "effect/Duration";
40 import * as Effect from "effect/Effect";
41 import * as Headers from "effect/http/Headers";
42 import { HttpServerRequest } from "effect/http/HttpServerRequest";
43 import * as HttpServerResponse from "effect/http/HttpServerResponse";
44 import * as Layer from "effect/Layer";
45 import * as Option from "effect/Option";
46 import * as Schedule from "effect/Schedule";
47 import * as Schema from "effect/Schema";
48 import type * as Scope from "effect/Scope";
49 import * as Semaphore from "effect/Semaphore";
50 import * as Stream from "effect/Stream";
51 import { WorkflowEngine } from "effect/workflow";
52 import { outsideAlarmPass } from "../Workers/AlarmCallback.ts";
53 import {
54 DurableObject as CloudflareDurableObject,
55 type DurableObjectJurisdiction,
56 } from "../Workers/DurableObject.ts";
57 import { DurableObjectState } from "../Workers/DurableObjectState.ts";
58 import { makeRpcStub } from "../Workers/Rpc.ts";
59 import { upgrade, type WebSocket } from "../Workers/WebSocket.ts";
60 import { Worker as CloudflareWorker, type Worker } from "../Workers/Worker.ts";
61 import { WorkerEnvironment } from "../Workers/WorkerRuntime.ts";
62 import { layerDurableObjectEngine } from "../Workflows/DurableObjectEngine.ts";
63 import { sleep, task, Workflow, type WorkflowImpl } from "../Workflows/Workflow.ts";
64 import { Backfill as BackfillRun } from "./Backfill.ts";
65 import { DeployVersion } from "./Version.ts";
66
67 const pad = (n: number) => n.toString().padStart(12, "0");
68
69 const DefectTag = "alchemy/Domain/Defect";
70
71 /**
72 * Workers RPC replaces thrown errors with an opaque "internal error". Return
73 * failures as data instead, and rethrow them as defects on the calling side.
74 */
75 const reportDefects = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
76 effect.pipe(
77 Effect.catchCause((cause) =>
78 Effect.succeed({ [DefectTag]: Cause.pretty(cause) } as unknown as A),
79 ),
80 );
81
82 /**
83 * How long to keep retrying a Durable Object call that failed in transit:
84 * an instance being reset reports an opaque "internal error", and for a while
85 * after a deploy some instances still run the previous script (whose classes
86 * "do not implement" our methods). Backs off from 100ms to 2s, about a minute
87 * in total, so a call survives a deploy's rollout.
88 */
89 const TRANSIT_RETRY = {
90 // Whichever delay is shorter: exponential at first, then every 2 seconds.
91 schedule: Schedule.min([Schedule.exponential("100 millis"), Schedule.spaced("2 seconds")]),
92 times: 35,
93 } as const;
94
95 /**
96 * Call a Durable Object. A call that fails in transit is retried: every Domain
97 * object method is idempotent (sends dedupe by command id, deliveries by
98 * checkpoint, schedules replace, reads are reads). A defect the object
99 * reported, or a call that keeps failing, is a defect for the caller.
100 */
101 const unwrapDefects = <A>(call: Effect.Effect<unknown, unknown>): Effect.Effect<A> =>
102 call.pipe(
103 Effect.catchDefect((defect) => Effect.fail(defect)),
104 Effect.retry(TRANSIT_RETRY),
105 Effect.orDie,
106 Effect.flatMap((value) =>
107 value !== null && typeof value === "object" && DefectTag in value
108 ? Effect.die(new Error(String((value as Record<string, unknown>)[DefectTag])))
109 : Effect.succeed(value as A),
110 ),
111 );
112
113 /**
114 * Call a Durable Object once, for the backfill protocol: it retries for itself,
115 * backing off when Cloudflare refuses the account's load. Retrying here too (as
116 * `unwrapDefects` does) would hold each level of a run for a minute, and keep
117 * the account overloaded with calls every 2 seconds from every waiting caller.
118 */
119 const callOnce = <A>(call: Effect.Effect<unknown, unknown>): Effect.Effect<A> =>
120 call.pipe(
121 Effect.orDie,
122 Effect.flatMap((value) =>
123 value !== null && typeof value === "object" && DefectTag in value
124 ? Effect.die(new Error(String((value as Record<string, unknown>)[DefectTag])))
125 : Effect.succeed(value as A),
126 ),
127 );
128
129 /** A Durable Object stub: every method is an RPC returning an Effect. */
130 interface Stub {
131 readonly [method: string]: (...args: ReadonlyArray<unknown>) => Effect.Effect<unknown, unknown>;
132 }
133
134 interface Namespace {
135 readonly getByName: (name: string) => Stub & {
136 readonly fetch: (
137 request: HttpServerRequest,
138 ) => Effect.Effect<HttpServerResponse.HttpServerResponse, unknown>;
139 };
140 readonly jurisdiction: (jurisdiction: DurableObjectJurisdiction) => Namespace;
141 }
142
143 /** What an instance's methods and constructor run with. */
144 type InstanceServices = DurableObjectState | RuntimeContext | Scope.Scope;
145
146 /** The usual outer (construction) / inner (instance) pair. */
147 type ObjectImpl = Effect.Effect<
148 Effect.Effect<Record<string, unknown>, never, InstanceServices>,
149 never,
150 DurableObjectState
151 >;
152
153 /**
154 * Declare a Durable Object class on the host Worker from inside a factory.
155 * The inline form of `Cloudflare.DurableObject` is typed for single-stage
156 * implementations, so it is re-typed here for the two-stage form it runs.
157 */
158 const declare = CloudflareDurableObject as unknown as (
159 name: string,
160 impl: ObjectImpl,
161 ) => Effect.Effect<Namespace, never, Worker>;
162
163 /**
164 * The Durable Object class of a view or projection generation. A rebuilt
165 * consumer is a new class (`DepositsViewV1`): it exists only in the code that
166 * rebuilds it, so none of its objects can start on older code during a
167 * rollout, and the previous generation's are left as they were (a shadow
168 * rebuild).
169 */
170 const consumerClass = (kind: "view" | "projection", name: string, generation: number) =>
171 `${name}${kind === "view" ? "View" : "Projection"}${generation > 0 ? `V${generation}` : ""}`;
172
173 /**
174 * The class of a consumer generation's run: its root, relays and scanners. A
175 * class of its own, new with the deploy like the keys' class, so none of them
176 * can start on older code; and apart from the keys, so a burst of keys that
177 * runs an isolate out of memory doesn't reset the run that drives them.
178 */
179 const runClass = (keys: string) => `${keys}Run`;
180
181 /** A consumer's generation from its target id: `view:Name#2` is 2. */
182 const generationOfTarget = (target: string) => Number(/#(\d+)$/.exec(target)?.[1] ?? 0);
183
184 /**
185 * Run `effect` with every other event on this instance held back. Durable
186 * Objects run each event in its own I/O context, so cross-event in-memory
187 * coordination (semaphores, refs, deferreds) is unsafe; the platform's gate
188 * is not.
189 */
190 const exclusively = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
191 Effect.gen(function* () {
192 const state = yield* DurableObjectState;
193 const exit = yield* state.blockConcurrencyWhile(() => Effect.exit(effect));
194 return yield* exit;
195 });
196
197 /** What `scheduled:<commandId>` holds once that command is cancelled; never a job token. */
198 const CANCELLED = "cancelled";
199
200 /** How long a queue drain may hold its lease before another drain may take over. */
201 const LEASE_MS = 60_000;
202
203 /**
204 * A durable, ordered work queue in the instance's storage. Rows are written
205 * in the same transaction as the state change that produced them, processed
206 * right after the commit (`kick`), and again by a durable callback if the
207 * instance dies first. One drain runs at a time (a storage lease), rows are
208 * processed in order, and a row is deleted only after its work succeeds.
209 */
210 const queue = <Row>(name: string, process: (row: Row) => Effect.Effect<void>) =>
211 Effect.gen(function* () {
212 const state = yield* DurableObjectState;
213 const prefix = `${name}:`;
214 const lease = `${name}-lease`;
215 const acquire = exclusively(
216 Effect.gen(function* () {
217 const held = yield* state.storage.get<number>(lease);
218 if (held !== undefined && held > Date.now()) return false;
219 yield* state.storage.put(lease, Date.now() + LEASE_MS);
220 return true;
221 }),
222 );
223 // oxlint-disable-next-line effecttsgo/unsafe-effect-type-assertion -- the Durable Object provides its state and RuntimeContext on every event
224 const drain: Effect.Effect<void> = Effect.gen(function* () {
225 if (!(yield* acquire)) return;
226 yield* Effect.gen(function* () {
227 const rows = yield* state.storage.list<Row>({ prefix });
228 for (const [key, row] of rows) {
229 yield* process(row);
230 yield* state.storage.delete(key);
231 }
232 }).pipe(Effect.ensuring(state.storage.delete(lease).pipe(Effect.ignore)));
233 // Rows staged while this drain ran.
234 const more = yield* state.storage.list({ prefix, limit: 1 });
235 if (more.size > 0) yield* drain;
236 }) as Effect.Effect<void>;
237 const callback = yield* makeCallback(`domain:${name}`, () => drain);
238 return {
239 /** Stage rows; call inside the storage transaction that commits their cause. */
240 stage: (rows: ReadonlyArray<Row>) =>
241 Effect.gen(function* () {
242 if (rows.length === 0) return;
243 const last = ((yield* state.storage.get<number>(`${name}-seq`)) ?? 0) + rows.length;
244 const entries: Record<string, Row> = {};
245 rows.forEach((row, index) => {
246 entries[`${prefix}${pad(last - rows.length + index + 1)}`] = row;
247 });
248 yield* state.storage.put(entries);
249 yield* state.storage.put(`${name}-seq`, last);
250 // The backstop: delivered even if this instance dies before `kick` runs.
251 // Its rows are progress, so a callback staging them doesn't hand it
252 // its retry budget.
253 yield* callback.schedule("drain", { after: "10 seconds", payload: {}, progress: last });
254 }),
255 /** Process staged rows now, in the background, keeping the instance alive until done. */
256 kick: state.waitUntil(outsideAlarmPass(drain.pipe(Effect.ignoreCause))),
257 /** Process staged rows now, and wait (unless another drain holds the lease). */
258 flush: drain,
259 };
260 });
261
262 // ---------------------------------------------------------------- live reads
263
264 /** What a live read's WebSocket carries across hibernation. */
265 const Attachment = Schema.Struct({ key: Schema.String, subscription: Subscription });
266 type Attachment = typeof Attachment.Type;
267 const decodeAttachment = Schema.decodeUnknownOption(Schema.fromJsonString(Attachment));
268
269 const SUBSCRIPTION_HEADER = "x-domain-subscription";
270
271 /** Send a push to its socket; close it when the read ended or could not be judged. */
272 const deliverPush = <R>(socket: WebSocket, key: string, push: Effect.Effect<Push, never, R>) =>
273 push.pipe(
274 Effect.flatMap((pushed) =>
275 Effect.gen(function* () {
276 // Each frame carries the cursor a reconnecting client resumes from.
277 const cursor = pushed.subscription.cursor;
278 for (const message of pushed.messages) {
279 yield* socket.send(JSON.stringify({ ...(message as object), cursor }));
280 }
281 if (pushed.done) return yield* socket.close(1000, "");
282 yield* Effect.sync(() =>
283 socket.serializeAttachment({
284 key,
285 subscription: pushed.subscription,
286 } satisfies Attachment),
287 );
288 }),
289 ),
290 Effect.catchCause(() => socket.close(1011, "Domain could not push to this read")),
291 );
292
293 /**
294 * Accept a live read's WebSocket (the current request, forwarded by
295 * {@link connectTo}) with the hibernation API, and send what it starts with.
296 * Runs exclusively, so no commit's push interleaves with the first one.
297 */
298 const accept = (
299 initial: (
300 key: string,
301 subscription: Subscription,
302 ) => Effect.Effect<Push, never, DurableObjectState | RuntimeContext>,
303 ) =>
304 Effect.gen(function* () {
305 const request = yield* HttpServerRequest;
306 const header = request.headers[SUBSCRIPTION_HEADER];
307 if (header === undefined || request.headers.upgrade?.toLowerCase() !== "websocket") {
308 return HttpServerResponse.empty({ status: 426, headers: { Upgrade: "websocket" } });
309 }
310 const attachment = decodeAttachment(header);
311 if (Option.isNone(attachment)) {
312 return HttpServerResponse.text("Invalid live read", { status: 400 });
313 }
314 const { key, subscription } = attachment.value;
315 return yield* exclusively(
316 Effect.gen(function* () {
317 const [response, socket] = yield* upgrade();
318 if (subscription.refused !== undefined) {
319 yield* socket.send(
320 JSON.stringify({
321 _tag: "Exit",
322 requestId: subscription.requestId,
323 exit: subscription.refused,
324 }),
325 );
326 yield* socket.close(1000, "");
327 return response;
328 }
329 yield* deliverPush(socket, key, initial(key, subscription));
330 return response;
331 }),
332 );
333 });
334
335 /** Push to every live read of this instance after a commit. Runs inside the commit's exclusive section. */
336 const broadcast = (
337 push: (attachment: Attachment) => Effect.Effect<Push, never, DurableObjectState | RuntimeContext>,
338 ) =>
339 Effect.gen(function* () {
340 const state = yield* DurableObjectState;
341 for (const socket of yield* state.getWebSockets()) {
342 const attachment = socket.deserializeAttachment<Attachment>();
343 if (attachment === null) continue;
344 yield* deliverPush(socket, attachment.key, push(attachment));
345 }
346 });
347
348 /** Live reads only receive; a client message or close needs no work beyond ending the socket. */
349 const socketHandlers = {
350 webSocketMessage: () => Effect.void,
351 webSocketClose: (socket: WebSocket) => socket.close(1000, "").pipe(Effect.ignoreCause),
352 webSocketError: (socket: WebSocket) => socket.close(1011, "").pipe(Effect.ignoreCause),
353 };
354
355 /**
356 * Forward a WebSocket upgrade (the current request) to the instance that owns
357 * `key`, carrying the subscription. The Worker sets the header, overwriting
358 * any a client sent, so the object can trust it.
359 */
360 const connectTo = (namespace: Namespace) => (key: string, subscription: Subscription) =>
361 Effect.gen(function* () {
362 const request = yield* HttpServerRequest;
363 const forwarded = request.modify({
364 headers: Headers.set(
365 request.headers,
366 SUBSCRIPTION_HEADER,
367 JSON.stringify({ key, subscription } satisfies Attachment),
368 ),
369 });
370 // Like every other call, ride out a deploy's rollout: an instance still
371 // running the previous script cannot upgrade, so retry until one does.
372 const response = yield* namespace
373 .getByName(key)
374 .fetch(forwarded)
375 .pipe(
376 Effect.filterOrFail(
377 (response) => response.status === 101,
378 (response) => new Error(`Live read upgrade answered ${response.status}`),
379 ),
380 Effect.retry(TRANSIT_RETRY),
381 Effect.orDie,
382 );
383 // A fetched Response's headers are immutable; re-wrap the socket so
384 // middleware (CORS, cookies) can still add headers to the upgrade.
385 const body = response.body;
386 if (body._tag === "Raw" && body.body instanceof Response) {
387 const webSocket = (body.body as unknown as { readonly webSocket: unknown }).webSocket;
388 if (webSocket) {
389 return HttpServerResponse.raw(
390 new Response(null, { status: 101, webSocket } as ResponseInit),
391 { status: 101 },
392 );
393 }
394 }
395 return response;
396 });
397
398 // ---------------------------------------------------------------- backfill
399
400 /** Delete every stored key under these prefixes, a batch at a time. */
401 const clearPrefixes = (prefixes: ReadonlyArray<string>) =>
402 Effect.gen(function* () {
403 const state = yield* DurableObjectState;
404 for (const prefix of prefixes) {
405 while (true) {
406 const keys = [...(yield* state.storage.list({ prefix, limit: 128 })).keys()];
407 if (keys.length === 0) break;
408 yield* state.storage.delete(keys);
409 }
410 }
411 });
412
413 /**
414 * What the backfill protocol runs on in one Durable Object: its storage (an
415 * exclusive section is one transaction, with every other event held back),
416 * and its jobs as durable callbacks. Each step counts as progress, so a long
417 * backfill never exhausts a callback's retry budget.
418 */
419 /** How many due backfill jobs one tick runs, and for how long, before it hands over to the next. */
420 const TICK_RUNS = 64;
421 const TICK_MILLIS = 5_000;
422
423 const backfillRuntime = (
424 run: (id: string) => Effect.Effect<void, never, DurableObjectState | RuntimeContext>,
425 ) =>
426 Effect.gen(function* () {
427 const state = yield* DurableObjectState;
428 const context = yield* captureContext<DurableObjectState | RuntimeContext>();
429 const provide = <A, E>(
430 effect: Effect.Effect<A, E, DurableObjectState | RuntimeContext>,
431 ): Effect.Effect<A, E> => Effect.provide(effect, context);
432 const storage: Backfill.Storage = {
433 get: <A>(key: string) => provide(state.storage.get<A>(key)),
434 put: (entries) =>
435 provide(
436 Object.keys(entries).length === 0
437 ? Effect.void
438 : state.storage.put(entries as Record<string, unknown>),
439 ),
440 delete: (keys) =>
441 provide(keys.length === 0 ? Effect.void : Effect.asVoid(state.storage.delete([...keys]))),
442 list: <A>(options: {
443 readonly prefix: string;
444 readonly startAfter?: string;
445 readonly limit?: number;
446 }) =>
447 provide(
448 state.storage
449 .list<A>({
450 prefix: options.prefix,
451 ...(options.startAfter === undefined ? {} : { startAfter: options.startAfter }),
452 ...(options.limit === undefined ? {} : { limit: options.limit }),
453 })
454 .pipe(Effect.map((rows) => [...rows.entries()])),
455 ),
456 exclusive: (effect) =>
457 state.storage.transaction(effect).pipe(exclusively, Effect.orDie, provide),
458 };
459 // Jobs are rows here (`bf:job:<id>` holds when it's due and a token), run by one
460 // alarm callback, the tick. A tick runs every due job, and those its jobs schedule
461 // at once, until none is left or its budget is spent: scheduling a job writes one
462 // row, where a callback of its own writes about ten. The tick's callback is
463 // removed only once it returns, so a crash in between runs the tick again; a
464 // job's row is removed only if it wasn't scheduled again while it ran.
465 const jobKey = (id: string) => `bf:job:${id}`;
466 let ticking = false;
467 let dirty = false;
468 /** The job running now, and the token of its row: that row doesn't count as already scheduled. */
469 let current: { readonly id: string; readonly token: string } | undefined;
470 const now = Clock.currentTimeMillis;
471 // One at a time: the callback is one row, so a later schedule would replace a sooner one.
472 const tickLock = yield* Semaphore.make(1);
473 /** Make sure a tick fires by `due`. */
474 const tickBy = (due: number) =>
475 Effect.gen(function* () {
476 const pending = yield* state.storage.get<number>("bf:tick");
477 if (pending !== undefined && pending <= due) return;
478 const at = yield* now;
479 // A callback ignores a schedule whose progress isn't past the last one's: count
480 // them, rather than use the clock (each event has its own, and they disagree).
481 const progress = ((yield* state.storage.get<number>("bf:tick-progress")) ?? 0) + 1;
482 yield* callback.schedule("tick", {
483 after: `${Math.max(0, due - at)} millis`,
484 payload: {},
485 progress,
486 });
487 yield* state.storage.put({ "bf:tick": due, "bf:tick-progress": progress });
488 }).pipe(Effect.orDie, tickLock.withPermits(1));
489 const tick: Effect.Effect<void, never, DurableObjectState | RuntimeContext> = Effect.gen(
490 function* () {
491 yield* state.storage.delete("bf:tick");
492 ticking = true;
493 const started = yield* now;
494 let runs = 0;
495 while (true) {
496 dirty = false;
497 const at = yield* now;
498 const rows = [
499 ...(yield* state.storage.list<{ due: number; token: string }>({
500 prefix: "bf:job:",
501 })).entries(),
502 ];
503 const due = rows.filter(([, job]) => job.due <= at);
504 const spent = runs >= TICK_RUNS || at - started >= TICK_MILLIS;
505 if ((due.length === 0 && !dirty) || spent) {
506 ticking = false;
507 const next = Math.min(...rows.map(([, job]) => job.due));
508 if (Number.isFinite(next)) yield* tickBy(spent ? Math.min(next, at) : next);
509 return;
510 }
511 for (const [key, job] of due) {
512 runs += 1;
513 current = { id: key.slice("bf:job:".length), token: job.token };
514 yield* run(current.id);
515 current = undefined;
516 const after = yield* state.storage.get<{ due: number; token: string }>(key);
517 if (after?.token === job.token) yield* state.storage.delete(key);
518 }
519 }
520 },
521 ).pipe(
522 Effect.ensuring(
523 Effect.sync(() => {
524 ticking = false;
525 current = undefined;
526 }),
527 ),
528 );
529 const callback = yield* makeCallback("domain:backfill", (_: unknown) => tick);
530 const jobs: Backfill.Jobs = {
531 schedule: (id, afterMillis) =>
532 provide(
533 Effect.gen(function* () {
534 const due = (yield* now) + afterMillis;
535 const stored = yield* state.storage.get<{ due: number; token: string }>(jobKey(id));
536 // The running job's own row is done once it returns; one scheduled while it
537 // ran (by a message that arrived meanwhile) still stands.
538 const existing =
539 stored !== undefined && current?.id === id && stored.token === current.token
540 ? undefined
541 : stored;
542 // The earliest request wins: already due sooner, that run sees this cause too.
543 if (existing !== undefined && existing.due <= due) return;
544 const token = yield* Effect.sync(() => crypto.randomUUID());
545 yield* state.storage.put(jobKey(id), {
546 due: existing !== undefined ? Math.min(existing.due, due) : due,
547 token,
548 });
549 // A running tick picks it up.
550 if (ticking) {
551 dirty = true;
552 return;
553 }
554 yield* tickBy(due);
555 }).pipe(Effect.orDie),
556 ),
557 };
558 return { storage, jobs, provide };
559 });
560
561 /** A directory node: one Durable Object per path in the tree. */
562 /** What a backfill needs to know of a consumer, in the deployed model. */
563 type PlanOf = (target: string) => Backfill.Plan | undefined;
564
565 /** Every plan the model has for these targets, as an announcement carries them. */
566 const plansOf = (targets: ReadonlyArray<string>, planOf: PlanOf) => {
567 const plans: Record<string, Backfill.Plan> = {};
568 for (const target of targets) {
569 const plan = planOf(target);
570 if (plan !== undefined) plans[target] = plan;
571 }
572 return plans;
573 };
574
575 const directoryObject = (net: Backfill.Network) =>
576 Effect.gen(function* () {
577 return Effect.gen(function* () {
578 let node: Backfill.Node | undefined;
579 const runtime = yield* backfillRuntime((id) => node!.run(id));
580 node = Backfill.makeNode(runtime.storage, runtime.jobs, net, Backfill.defaultLimits);
581 const role = node;
582 return {
583 register: (path: string, ref: Backfill.SourceRef) =>
584 reportDefects(role.register(path, ref)),
585 child: (path: string, child: string) => reportDefects(role.child(path, child)),
586 list: (path: string, after: string | undefined) => reportDefects(role.list(path, after)),
587 };
588 });
589 });
590
591 /**
592 * The object of a consumer's class that scans one source instance for it: one
593 * per instance and consumer generation, new with the deploy.
594 */
595 const scannerName = (ref: Backfill.SourceRef) =>
596 `~scan/${ref.source}/${ref.id}${ref.generation ? `#${ref.generation}` : ""}`;
597
598 /** A consumer object's scanner methods (an object named by `scannerName`). */
599 const scannerRpc = (scanner: () => Backfill.Scanner) => ({
600 bfScannerScan: (target: string, ref: Backfill.SourceRef, waitMillis: number) =>
601 reportDefects(scanner().scan(target, ref, waitMillis)),
602 bfScannerReady: (target: string, waitMillis: number) =>
603 reportDefects(scanner().ready(target, waitMillis)),
604 bfScannerKick: (target: string, ref: Backfill.SourceRef) =>
605 reportDefects(scanner().kick(target, ref)),
606 });
607
608 /**
609 * How long the root of a relay run works in one alarm before it starts another
610 * (an alarm may run for 15 minutes): keys still reading history are called again.
611 */
612 const RELAY_ROOT_WAIT_MS = 5 * 60_000;
613
614 /** A step of the deploy's Workflow couldn't reach a consumer's class: this invocation runs older code. */
615 class Unreachable extends Data.TaggedError("Unreachable")<{ readonly message: string }> {}
616
617 /**
618 * A Workflow step that reaches a consumer's class fails (rather than dies) when
619 * the invocation runs code without it, and is tried a few times. A run started
620 * on an edge that doesn't have the deployed code yet stays on that code: it
621 * ends with this error, and the deploy starts it again (see `Backfill.ts`).
622 */
623 const reachable = <A>(effect: Effect.Effect<A>) =>
624 effect.pipe(
625 Effect.catchCause((cause) => Effect.fail(new Unreachable({ message: Cause.pretty(cause) }))),
626 );
627 const REACH_RETRIES = { limit: 3, delay: "1 second", backoff: "constant" } as const;
628
629 /** The object of a consumer's class at the root of its run: it drives the run's passes. */
630 const RUN_ROOT = "~run";
631
632 /** The object of a consumer's class for one relay of its run. */
633 const relayName = (target: string, name: string) => `~relay/${target}/${name}`;
634
635 /**
636 * A consumer object's run methods. Every part of a consumer's run (its root,
637 * relays and scanners) is an object of the consumer's own class: a class the
638 * deploy adds exists only in the deployed code, so none of them can start on
639 * older code during a rollout. The root (`RUN_ROOT`) drives the run's passes
640 * from its alarm (see `Backfill.makeRun`), so whoever starts it (the deploy's
641 * Workflow, or the first call after a local redeploy) only asks how far it
642 * has got.
643 */
644 const runRpc = (relay: Backfill.Relay, run: () => Backfill.Run) => ({
645 bfRelayRun: (
646 name: string,
647 relayRun: Backfill.RelayRun,
648 items: ReadonlyArray<Backfill.RelayItem>,
649 ) => reportDefects(relay.run(name, relayRun, items)),
650 bfRunBegin: (target: string, plan: Backfill.Plan) => reportDefects(run().begin(target, plan)),
651 bfRunPhase: (target: string) => reportDefects(run().phase(target)),
652 bfRunProgress: (target: string) => reportDefects(run().progress(target)),
653 });
654
655 /** How long before a key that couldn't list itself in the directory tries again. */
656 const REGISTER_RETRY_MS = 2_000;
657
658 /** The id an instance was listed under, saved so jobs that fire later find it. */
659 const ID_KEY = "bf:id";
660
661 /**
662 * List an instance in the directory before its first commit, so every
663 * instance with an event can be reached when a consumer is added later.
664 */
665 const ensureRegistered = (net: Backfill.Network, ref: Backfill.SourceRef) =>
666 Effect.gen(function* () {
667 const state = yield* DurableObjectState;
668 if (yield* state.storage.get<boolean>("registered")) return;
669 yield* net.node.register(Backfill.leafOf(ref), ref);
670 yield* state.storage.put({ registered: true, [ID_KEY]: ref.id });
671 });
672
673 const aggregateObject = (
674 kit: AggregateKit,
675 net: Backfill.Network,
676 model: () => ReadonlyArray<string>,
677 planOf: PlanOf,
678 ) =>
679 Effect.gen(function* () {
680 const state = yield* DurableObjectState;
681 return Effect.gen(function* () {
682 const outbox = yield* queue<Delivery>("outbox", kit.deliver);
683 const currentMeta = state.storage
684 .get<{ state: Json; version: number }>("meta")
685 .pipe(Effect.map((meta) => meta ?? { state: null, version: 0 }));
686 const eventsAfter = (after: number) =>
687 state.storage
688 .list<StoredEvent>({ prefix: "event:", start: `event:${pad(after + 1)}` })
689 .pipe(Effect.map((rows) => [...rows.values()]));
690
691 // This instance as a backfill source: consumers added later pull its events.
692 let source: Backfill.Source | undefined;
693 const runtime = yield* backfillRuntime((id) =>
694 id === "bf-flush"
695 ? Effect.flatMap(state.storage.get<string>(ID_KEY), (stored) =>
696 stored === undefined ? Effect.void : sourceFor(stored).flush,
697 )
698 : Effect.void,
699 );
700 const sourceFor = (id: string): Backfill.Source =>
701 (source ??= Backfill.makeSource(runtime.storage, runtime.jobs, net, {
702 ref: { source: kit.name, id },
703 singleSource: (target) => planOf(target)?.singleSource === true,
704 model,
705 log: (after, limit) =>
706 runtime.provide(
707 state.storage
708 .list<StoredEvent>({ prefix: "event:", start: `event:${pad(after + 1)}`, limit })
709 .pipe(Effect.map((rows) => [...rows.values()])),
710 ),
711 version: runtime.provide(Effect.map(currentMeta, (meta) => meta.version)),
712 routed: runtime.provide(state.storage.get<Routed>("routed")),
713 markLive: (target) =>
714 runtime.provide(
715 Effect.gen(function* () {
716 const { version } = yield* currentMeta;
717 const routed = yield* state.storage.get<Routed>("routed");
718 yield* state.storage.put("routed", { ...routed, [target]: version });
719 }),
720 ),
721 }));
722
723 // Callers run it `exclusively`.
724 const commit = (id: string, command: Json, meta: Aggregate.SendMeta) =>
725 Effect.gen(function* () {
726 const duplicate = yield* state.storage.get<SendResult>(`receipt:${meta.commandId}`);
727 if (duplicate) return { sent: duplicate, kick: false };
728 const current = yield* currentMeta;
729 const result = yield* kit.handle({
730 id,
731 state: current.state,
732 version: current.version,
733 command,
734 commandId: meta.commandId,
735 context: meta.context,
736 now: yield* Effect.sync(() => Date.now()),
737 correlationId: meta.correlationId,
738 trace: meta.trace,
739 link: meta.link,
740 });
741 if (result._tag === "Rejected") return { sent: result as SendResult, kick: false };
742 const sent: SendResult = {
743 _tag: "Accepted",
744 receipt: {
745 stream: `${kit.name}/${id}`,
746 version: result.version,
747 events: result.events.map((e) => e.event),
748 reply: result.reply,
749 },
750 };
751 const routed = yield* state.storage.transaction(
752 Effect.gen(function* () {
753 const events: Record<string, StoredEvent> = {};
754 for (const event of result.events) events[`event:${pad(event.envelope.seq)}`] = event;
755 yield* state.storage.put(events);
756 yield* state.storage.put("meta", {
757 state: result.state,
758 version: result.version,
759 });
760 yield* state.storage.put(`receipt:${meta.commandId}`, sent);
761 // Live to caught-up consumers; through the backfill to lagging ones.
762 const live = yield* Backfill.routeCommit(
763 sourceFor(id),
764 kit.targets,
765 yield* state.storage.get<Routed>("routed"),
766 current.version,
767 result.version,
768 result.deliveries,
769 );
770 yield* state.storage.put("routed", live.routed);
771 yield* outbox.stage(live.send);
772 return live.send;
773 }),
774 );
775 const committed = { state: result.state, version: result.version };
776 yield* broadcast(({ key, subscription }) =>
777 Effect.gen(function* () {
778 // A subscriber behind this commit's first event reads the gap from storage.
779 const first = result.events[0]?.envelope.seq ?? Infinity;
780 const events =
781 subscription.mode === "events" && subscription.cursor < first - 1
782 ? yield* eventsAfter(subscription.cursor)
783 : result.events;
784 return yield* kit.push(key, subscription, committed, events);
785 }),
786 );
787 return { sent, kick: routed.length > 0 };
788 });
789 const send = (id: string, command: Json, meta: Aggregate.SendMeta) =>
790 Effect.gen(function* () {
791 yield* ensureRegistered(net, { source: kit.name, id });
792 const { sent, kick } = yield* exclusively(commit(id, command, meta));
793 if (kick) yield* outbox.kick;
794 return sent;
795 });
796 const executed = (commandId: string) =>
797 state.storage
798 .get<SendResult>(`receipt:${commandId}`)
799 .pipe(Effect.map((receipt) => receipt !== undefined));
800 // Scheduled commands: one durable callback job per command id, run here
801 // when due. `scheduled:<commandId>` holds the token of the job that may
802 // run, or `CANCELLED` once it is cancelled, so a job cancelled or
803 // replaced after the alarm claimed it does nothing. It is cleared after
804 // the commit (the receipt takes over), so a retried job still runs.
805 const timers = yield* makeCallback(
806 "domain:schedule",
807 (job: {
808 readonly id: string;
809 readonly command: Json;
810 readonly meta: Aggregate.SendMeta;
811 readonly token?: string;
812 }) =>
813 Effect.gen(function* () {
814 const key = `scheduled:${job.meta.commandId}`;
815 yield* ensureRegistered(net, { source: kit.name, id: job.id });
816 const kick = yield* exclusively(
817 Effect.gen(function* () {
818 // A job scheduled before tokens existed has none, and runs.
819 if (
820 job.token !== undefined &&
821 (yield* state.storage.get<string>(key)) !== job.token
822 )
823 return false;
824 const { kick } = yield* commit(job.id, job.command, job.meta);
825 yield* state.storage.delete(key);
826 return kick;
827 }),
828 );
829 if (kick) yield* outbox.kick;
830 }),
831 );
832 return {
833 send: (id: string, command: Json, meta: Aggregate.SendMeta) =>
834 reportDefects(send(id, command, meta)),
835 schedule: (id: string, command: Json, meta: Aggregate.SendMeta, due: Aggregate.Due) =>
836 Effect.gen(function* () {
837 if (yield* executed(meta.commandId)) return "AlreadyExecuted" as const;
838 const token = yield* Effect.sync(() => crypto.randomUUID());
839 const at =
840 "at" in due ? due.at : (yield* Effect.sync(() => Date.now())) + due.afterMillis;
841 yield* state.storage.put(`scheduled:${meta.commandId}`, token);
842 yield* timers.schedule(meta.commandId, {
843 at,
844 payload: { id, command, meta, token },
845 });
846 return { at };
847 }).pipe(exclusively, reportDefects),
848 unschedule: (commandId: string) =>
849 Effect.gen(function* () {
850 if (yield* executed(commandId)) return "AlreadyExecuted" as const;
851 const key = `scheduled:${commandId}`;
852 const token = yield* state.storage.get<string>(key);
853 if (token === CANCELLED) return "AlreadyCancelled" as const;
854 // A job scheduled before tokens existed has a job but no token.
855 if (token === undefined && (yield* timers.getStatus(commandId)) === undefined)
856 return "NotScheduled" as const;
857 yield* state.storage.put(key, CANCELLED);
858 yield* timers.cancel(commandId);
859 return "Cancelled" as const;
860 }).pipe(exclusively, reportDefects),
861 state: () => reportDefects(currentMeta),
862 after: (after: number) => reportDefects(eventsAfter(after)),
863 // The backfill protocol, as a source.
864 // Any code answers these: a run reads its log, and routes it with its own code.
865 bfLog: (id: string, after: number, limit: number) =>
866 reportDefects(sourceFor(id).log(after, limit)),
867 bfCursor: (id: string, target: string) => reportDefects(sourceFor(id).cursor(target)),
868 bfRead: (id: string, target: string, key: string, after: number, limit: number) =>
869 reportDefects(sourceFor(id).read(target, key, after, limit)),
870 bfHandoff: (id: string, target: string, key: string, through: number) =>
871 reportDefects(sourceFor(id).handoff(target, key, through)),
872 // A live read of this instance: accept its WebSocket, then push on every commit.
873 fetch: accept((id, subscription) =>
874 Effect.gen(function* () {
875 const current = yield* currentMeta;
876 const events =
877 subscription.mode === "events" ? yield* eventsAfter(subscription.cursor) : [];
878 return yield* kit.push(id, subscription, current, events);
879 }),
880 ),
881 ...socketHandlers,
882 };
883 });
884 });
885
886 const viewObject = (
887 kit: ViewKit,
888 net: Backfill.Network,
889 model: () => ReadonlyArray<string>,
890 planOf: PlanOf,
891 ) =>
892 Effect.gen(function* () {
893 const state = yield* DurableObjectState;
894 return Effect.gen(function* () {
895 const outbox = yield* queue<Delivery>("outbox", kit.deliver);
896 const stored = state.storage
897 .get<ViewSnapshot>("snapshot")
898 .pipe(Effect.map((s) => s ?? emptySnapshot));
899 // Built by an earlier generation of the view: it reads as empty, and starts over on write.
900 const stale = (snapshot: ViewSnapshot) => (snapshot.generation ?? 0) < kit.generation;
901 const read = stored.pipe(Effect.map((s) => (stale(s) ? emptySnapshot : s)));
902 const reset = Effect.gen(function* () {
903 if (!stale(yield* stored)) return;
904 yield* clearPrefixes(["entry:", "in:"]);
905 yield* state.storage.delete(["snapshot", "routed", "in-seq"]);
906 });
907 const entriesAfter = (after: number) =>
908 stored.pipe(
909 Effect.flatMap((snapshot) =>
910 stale(snapshot)
911 ? Effect.succeed([])
912 : state.storage
913 .list<ViewEntry>({ prefix: "entry:", start: `entry:${pad(after + 1)}` })
914 .pipe(Effect.map((rows) => [...rows.values()])),
915 ),
916 );
917 const target = kit.target;
918 // Pushes and sends an apply leaves for after its transaction commits.
919 let after: Array<Effect.Effect<void, never, DurableObjectState | RuntimeContext>> = [];
920
921 /**
922 * Apply deliveries to the key and route what it emits. Callers hold the
923 * exclusive section and its transaction.
924 */
925 const apply = (key: string, deliveries: ReadonlyArray<Delivery>) =>
926 Effect.gen(function* () {
927 yield* reset;
928 const before = yield* read;
929 const result = yield* kit.apply(key, before, deliveries);
930 if (!result.changed) return;
931 const rows: Record<string, ViewEntry> = {};
932 for (const entry of result.entries) rows[`entry:${pad(entry.seq)}`] = entry;
933 // The key's immutable input log: every delivery it applied, in order.
934 const accepted = deliveries.filter((d) => (before.checkpoints[d.source] ?? 0) < d.seq);
935 const inputs = (yield* state.storage.get<number>("in-seq")) ?? 0;
936 const log: Record<string, Delivery> = {};
937 accepted.forEach((delivery, index) => {
938 log[`in:${pad(inputs + index + 1)}`] = delivery;
939 });
940 if (accepted.length > 0) yield* state.storage.put(log);
941 yield* state.storage.put("in-seq", inputs + accepted.length);
942 yield* state.storage.put("snapshot", result.snapshot);
943 if (result.entries.length > 0) yield* state.storage.put(rows);
944 const live = yield* Backfill.routeCommit(
945 sourceFor(key),
946 kit.targets,
947 yield* state.storage.get<Routed>("routed"),
948 before.seq,
949 result.snapshot.seq,
950 result.downstream,
951 );
952 yield* state.storage.put("routed", live.routed);
953 yield* outbox.stage(live.send);
954 after.push(
955 broadcast(({ key, subscription }) =>
956 Effect.gen(function* () {
957 // A subscriber behind this commit's first entry reads the gap from storage.
958 const first = result.entries[0]?.seq ?? Infinity;
959 const entries =
960 subscription.mode === "events" && subscription.cursor < first - 1
961 ? yield* entriesAfter(subscription.cursor)
962 : result.entries;
963 return yield* kit.push(key, subscription, result.snapshot, entries);
964 }),
965 ),
966 );
967 if (live.send.length > 0) after.push(outbox.kick);
968 }).pipe(kit.observe(key, deliveries));
969 const flushAfter = Effect.suspend(() => {
970 const pending = after;
971 after = [];
972 return Effect.forEach(pending, (effect) => effect, { discard: true });
973 });
974
975 // This key as a backfill consumer (it buffers and merges history) and source.
976 let consumer: Backfill.Consumer | undefined;
977 let source: Backfill.Source | undefined;
978 const runtime = yield* backfillRuntime(
979 (id): Effect.Effect<void, never, DurableObjectState | RuntimeContext> =>
980 Effect.flatMap(state.storage.get<string>(ID_KEY), (key) => {
981 if (id.startsWith("bf-scan:")) return scannerFor().run(id);
982 if (id.startsWith("drive:")) return runFor().run(id);
983 if (key === undefined) return Effect.void;
984 if (id === "bf-flush") return sourceFor(key).flush;
985 if (id.startsWith("bf-merge:"))
986 // Listed before its first entry, as on the live path, but here in the key's
987 // own job: its sources' messages don't wait on the directory.
988 return ensureRegistered(net, { source: kit.name, id: key }).pipe(
989 Effect.andThen(consumerFor(key).merge(id.slice("bf-merge:".length))),
990 Effect.catchCause(() => runtime.jobs.schedule(id, REGISTER_RETRY_MS)),
991 );
992 return Effect.void;
993 }),
994 );
995 const consumerFor = (key: string): Backfill.Consumer =>
996 (consumer ??= Backfill.makeConsumer(runtime.storage, runtime.jobs, net, {
997 key,
998 target: () => target,
999 singleSource: () => kit.singleSource,
1000 sources: () => planOf(target)?.sources ?? [],
1001 history: (ref, rows) => kit.history(target, ref.source, ref.id, rows),
1002 apply: (deliveries) => runtime.provide(Effect.orDie(apply(key, deliveries))),
1003 after: runtime.provide(flushAfter),
1004 }));
1005 // An object of this class named for a source instance scans it (see `scannerName`).
1006 let scanner: Backfill.Scanner | undefined;
1007 const scannerFor = (): Backfill.Scanner =>
1008 (scanner ??= Backfill.makeScanner(runtime.storage, runtime.jobs, net, {
1009 route: (target, ref, rows) => kit.history(target, ref.source, ref.id, rows),
1010 }));
1011 // An object of this class named `RUN_ROOT` drives the run; others relay it.
1012 let run: Backfill.Run | undefined;
1013 const runFor = (): Backfill.Run =>
1014 (run ??= Backfill.makeRun(runtime.storage, runtime.jobs, net, {
1015 waitMillis: RELAY_ROOT_WAIT_MS,
1016 againMillis: 1_000,
1017 }));
1018 const relay = Backfill.makeRelay(net);
1019 const sourceFor = (key: string): Backfill.Source =>
1020 (source ??= Backfill.makeSource(runtime.storage, runtime.jobs, net, {
1021 ref: {
1022 source: kit.name,
1023 id: key,
1024 ...(kit.generation > 0 ? { generation: kit.generation } : {}),
1025 },
1026 singleSource: (target) => planOf(target)?.singleSource === true,
1027 model,
1028 log: (after, limit) =>
1029 runtime.provide(
1030 state.storage
1031 .list<ViewEntry>({ prefix: "entry:", start: `entry:${pad(after + 1)}`, limit })
1032 .pipe(Effect.map((rows) => [...rows.values()])),
1033 ),
1034 version: runtime.provide(Effect.map(read, (snapshot) => snapshot.seq)),
1035 routed: runtime.provide(state.storage.get<Routed>("routed")),
1036 markLive: (target) =>
1037 runtime.provide(
1038 Effect.gen(function* () {
1039 const { seq } = yield* read;
1040 const routed = yield* state.storage.get<Routed>("routed");
1041 yield* state.storage.put("routed", { ...routed, [target]: seq });
1042 }),
1043 ),
1044 }));
1045
1046 /**
1047 * A key of a view keyed by its one aggregate reads its history from that
1048 * instance (if it hasn't yet) before it serves it.
1049 */
1050 const filled = (key: string) =>
1051 Effect.suspend(() => {
1052 const catchUp = kit.catchUp;
1053 if (catchUp === undefined) return Effect.void;
1054 return Effect.gen(function* () {
1055 yield* state.storage.put(ID_KEY, key);
1056 yield* consumerFor(key)
1057 .start(target, { source: catchUp.source, id: key }, 20_000)
1058 .pipe(Effect.repeat({ until: (started) => started === "settled", times: 5 }));
1059 });
1060 });
1061 return {
1062 receive: (key: string, deliveries: ReadonlyArray<Delivery>) =>
1063 reportDefects(
1064 Effect.gen(function* () {
1065 yield* ensureRegistered(net, { source: kit.name, id: key });
1066 // While a backfill runs, the key buffers deliveries and merges them in order.
1067 if (!(yield* consumerFor(key).admit(target, deliveries))) return;
1068 yield* exclusively(state.storage.transaction(apply(key, deliveries)));
1069 yield* flushAfter;
1070 }),
1071 ),
1072 read: () => reportDefects(read),
1073 after: (after: number) => reportDefects(entriesAfter(after)),
1074 // The backfill protocol, as a source, a consumer and a scanner.
1075 ...scannerRpc(() => scannerFor()),
1076 ...runRpc(relay, runFor),
1077 // Read as a source, a key keyed by its aggregate has its own history
1078 // first: a view or projection catching up never sees it still filling.
1079 bfLog: (key: string, after: number, limit: number) =>
1080 reportDefects(Effect.andThen(filled(key), sourceFor(key).log(after, limit))),
1081 bfCursor: (key: string, target: string) =>
1082 reportDefects(Effect.andThen(filled(key), sourceFor(key).cursor(target))),
1083 bfRead: (key: string, target: string, reader: string, after: number, limit: number) =>
1084 reportDefects(
1085 Effect.andThen(filled(key), sourceFor(key).read(target, reader, after, limit)),
1086 ),
1087 bfHandoff: (key: string, target: string, consumerKey: string, through: number) =>
1088 reportDefects(
1089 Effect.andThen(filled(key), sourceFor(key).handoff(target, consumerKey, through)),
1090 ),
1091 bfStart: (key: string, target: string, ref: Backfill.SourceRef, waitMillis?: number) =>
1092 reportDefects(
1093 Effect.gen(function* () {
1094 yield* state.storage.put(ID_KEY, key);
1095 return yield* consumerFor(key).start(target, ref, waitMillis);
1096 }),
1097 ),
1098 // The key lists itself in its merge job, before its first entry.
1099 bfPush: (key: string, target: string, deliveries: ReadonlyArray<Delivery>) =>
1100 reportDefects(
1101 Effect.gen(function* () {
1102 yield* state.storage.put(ID_KEY, key);
1103 yield* consumerFor(key).push(target, deliveries);
1104 }),
1105 ),
1106 bfConsumerReady: (key: string, target: string, waitMillis?: number) =>
1107 reportDefects(
1108 Effect.gen(function* () {
1109 yield* state.storage.put(ID_KEY, key);
1110 return yield* consumerFor(key).ready(target, waitMillis);
1111 }),
1112 ),
1113 // A live read of this key: accept its WebSocket, then push on every commit.
1114 fetch: accept((key, subscription) =>
1115 Effect.gen(function* () {
1116 const snapshot = yield* read;
1117 const entries =
1118 subscription.mode === "events" ? yield* entriesAfter(subscription.cursor) : [];
1119 return yield* kit.push(key, subscription, snapshot, entries);
1120 }),
1121 ),
1122 ...socketHandlers,
1123 };
1124 });
1125 });
1126
1127 const policyObject = (kit: PolicyKit) =>
1128 Effect.gen(function* () {
1129 const state = yield* DurableObjectState;
1130 return Effect.gen(function* () {
1131 const inbox = yield* queue<Delivery>("inbox", kit.run);
1132 return {
1133 receive: (delivery: Delivery) =>
1134 reportDefects(
1135 Effect.gen(function* () {
1136 const id = `${delivery.source}:${delivery.seq}`;
1137 const accepted = yield* exclusively(
1138 state.storage.transaction(
1139 Effect.gen(function* () {
1140 if (yield* state.storage.get(`seen:${id}`)) return false;
1141 yield* state.storage.put(`seen:${id}`, true);
1142 yield* inbox.stage([delivery]);
1143 return true;
1144 }),
1145 ),
1146 );
1147 if (accepted) yield* inbox.kick;
1148 }),
1149 ),
1150 };
1151 });
1152 });
1153
1154 /** The stored progress of one projection key. */
1155 interface ProjectionMeta {
1156 readonly key: string | null;
1157 /** Inputs accepted into the log. */
1158 readonly accepted: number;
1159 /** Inputs handled. */
1160 readonly processed: number;
1161 /** The last accepted `seq` per source: duplicates are dropped on arrival. */
1162 readonly intake: Readonly<Record<string, number>>;
1163 /** The last handled `seq` per source. */
1164 readonly checkpoints: Readonly<Record<string, number>>;
1165 /** The projection's generation the log was built by: an earlier one starts over. */
1166 readonly generation?: number;
1167 }
1168
1169 const emptyProjectionMeta: ProjectionMeta = {
1170 key: null,
1171 accepted: 0,
1172 processed: 0,
1173 intake: {},
1174 checkpoints: {},
1175 };
1176
1177 /** How long a failed or interrupted run waits before it is retried. */
1178 const PROJECTION_RETRY_MS = 10_000;
1179
1180 const projectionObject = (kit: ProjectionKit, net: Backfill.Network, planOf: PlanOf) =>
1181 Effect.gen(function* () {
1182 const state = yield* DurableObjectState;
1183 return Effect.gen(function* () {
1184 // A log built by an earlier generation of the projection reads as empty.
1185 const readMeta = state.storage
1186 .get<ProjectionMeta>("meta")
1187 .pipe(
1188 Effect.map((meta) =>
1189 meta === undefined || (meta.generation ?? 0) < kit.generation
1190 ? { ...emptyProjectionMeta, generation: kit.generation }
1191 : meta,
1192 ),
1193 );
1194 /** Start a log built by an earlier generation over. */
1195 const reset = Effect.gen(function* () {
1196 const meta = yield* state.storage.get<ProjectionMeta>("meta");
1197 if (meta === undefined || (meta.generation ?? 0) >= kit.generation) return;
1198 yield* clearPrefixes(["in:"]);
1199 yield* state.storage.delete("meta");
1200 });
1201 const size = kit.options.batch?.size ?? 1;
1202 const waitMillis = kit.options.batch?.waitMillis ?? 0;
1203 const lease = "drain-lease";
1204 const acquire = exclusively(
1205 Effect.gen(function* () {
1206 const held = yield* state.storage.get<number>(lease);
1207 if (held !== undefined && held > Date.now()) return false;
1208 yield* state.storage.put(lease, Date.now() + LEASE_MS);
1209 return true;
1210 }),
1211 );
1212 // Re-run the drain later: when a short batch has waited long enough, or after a failure.
1213 let wake: (at: number) => Effect.Effect<void> = () => Effect.void;
1214 // Each input's backstop advances this cursor: once an input arrives, a
1215 // drain waiting for it from inside its own callback waits the batch
1216 // wait, not that callback's retry backoff.
1217 const advance = Effect.gen(function* () {
1218 const progress = ((yield* state.storage.get<number>("drain-progress")) ?? 0) + 1;
1219 yield* state.storage.put("drain-progress", progress);
1220 return progress;
1221 });
1222 // oxlint-disable-next-line effecttsgo/unsafe-effect-type-assertion -- the Durable Object provides its state and RuntimeContext on every event
1223 const drain: Effect.Effect<void> = Effect.gen(function* () {
1224 if (!(yield* acquire)) return;
1225 const exit = yield* Effect.exit(
1226 Effect.gen(function* () {
1227 while (true) {
1228 const meta = yield* readMeta;
1229 if (meta.key === null || meta.processed >= meta.accepted) return;
1230 const rows = yield* state.storage.list<Accepted>({
1231 prefix: "in:",
1232 start: `in:${pad(meta.processed + 1)}`,
1233 limit: size,
1234 });
1235 const run = [...rows.values()];
1236 if (run.length === 0) return;
1237 if (run.length < size && waitMillis > 0 && Date.now() - run[0]!.at < waitMillis) {
1238 return yield* wake(run[0]!.at + waitMillis);
1239 }
1240 yield* kit.run(meta.key, run);
1241 yield* exclusively(
1242 Effect.gen(function* () {
1243 const current = yield* readMeta;
1244 const handled: Record<string, number> = {};
1245 for (const { delivery } of run) {
1246 handled[delivery.source] = Math.max(
1247 handled[delivery.source] ?? 0,
1248 delivery.seq,
1249 );
1250 }
1251 yield* state.storage.put("meta", {
1252 ...current,
1253 processed: run[run.length - 1]!.position,
1254 checkpoints: mergeCheckpoints(current.checkpoints, handled),
1255 } satisfies ProjectionMeta);
1256 }),
1257 );
1258 }
1259 }),
1260 );
1261 yield* state.storage.delete(lease).pipe(Effect.ignore);
1262 if (exit._tag === "Failure") {
1263 yield* Effect.logError(`projection ${kit.name} failed; retrying`, exit.cause);
1264 yield* wake(Date.now() + PROJECTION_RETRY_MS);
1265 }
1266 }) as Effect.Effect<void>;
1267 const callback = yield* makeCallback("domain:projection", () => drain);
1268 wake = (at) =>
1269 // oxlint-disable-next-line effecttsgo/unsafe-effect-type-assertion -- the Durable Object provides RuntimeContext on every event
1270 callback.schedule("drain", { at, payload: {} }).pipe(Effect.ignore) as Effect.Effect<void>;
1271 /**
1272 * Accept a delivery into the key's log, once. Callers hold the exclusive
1273 * section and its transaction; `true` when it was new.
1274 */
1275 const intake = (key: string, delivery: Delivery) =>
1276 Effect.gen(function* () {
1277 const sent = delivery.target.generation ?? 0;
1278 // For an earlier generation: sent by an instance still on older code.
1279 if (sent < kit.generation) return false;
1280 // Sent by newer code: fail, so the sender retries once this instance runs it.
1281 if (sent > kit.generation) {
1282 return yield* Effect.die(
1283 new Error(`${kit.name} received generation ${sent} but runs ${kit.generation}`),
1284 );
1285 }
1286 yield* reset;
1287 const meta = yield* readMeta;
1288 if ((meta.intake[delivery.source] ?? 0) >= delivery.seq) return false;
1289 const position = meta.accepted + 1;
1290 yield* state.storage.put(`in:${pad(position)}`, {
1291 position,
1292 delivery,
1293 at: Date.now(),
1294 } satisfies Accepted);
1295 yield* state.storage.put("meta", {
1296 ...meta,
1297 key,
1298 accepted: position,
1299 intake: { ...meta.intake, [delivery.source]: delivery.seq },
1300 } satisfies ProjectionMeta);
1301 // The backstop: handled even if this instance dies before the drain runs.
1302 yield* callback.schedule("drain", {
1303 after: `${Math.max(waitMillis, PROJECTION_RETRY_MS)} millis`,
1304 payload: {},
1305 progress: yield* advance,
1306 });
1307 return true;
1308 });
1309 const target = targetId({
1310 kind: "projection",
1311 name: kit.name,
1312 ...(kit.generation > 0 ? { generation: kit.generation } : {}),
1313 });
1314 // This key as a backfill consumer: it buffers and merges history in order.
1315 let consumer: Backfill.Consumer | undefined;
1316 const runtime = yield* backfillRuntime((id) =>
1317 id.startsWith("bf-scan:")
1318 ? scannerFor().run(id)
1319 : id.startsWith("drive:")
1320 ? runFor().run(id)
1321 : Effect.flatMap(state.storage.get<string>(ID_KEY), (key) =>
1322 key !== undefined && id.startsWith("bf-merge:")
1323 ? consumerFor(key).merge(id.slice("bf-merge:".length))
1324 : Effect.void,
1325 ),
1326 );
1327 // An object of this class named for a source instance scans it (see `scannerName`).
1328 let scanner: Backfill.Scanner | undefined;
1329 const scannerFor = (): Backfill.Scanner =>
1330 (scanner ??= Backfill.makeScanner(runtime.storage, runtime.jobs, net, {
1331 route: (target, ref, rows) => kit.history(target, ref.source, ref.id, rows),
1332 }));
1333 // An object of this class named `RUN_ROOT` drives the run; others relay it.
1334 let run: Backfill.Run | undefined;
1335 const runFor = (): Backfill.Run =>
1336 (run ??= Backfill.makeRun(runtime.storage, runtime.jobs, net, {
1337 waitMillis: RELAY_ROOT_WAIT_MS,
1338 againMillis: 1_000,
1339 }));
1340 const relay = Backfill.makeRelay(net);
1341 const consumerFor = (key: string): Backfill.Consumer =>
1342 (consumer ??= Backfill.makeConsumer(runtime.storage, runtime.jobs, net, {
1343 key,
1344 target: () => target,
1345 singleSource: () => kit.singleSource,
1346 sources: () => planOf(target)?.sources ?? [],
1347 history: (ref, rows) => kit.history(target, ref.source, ref.id, rows),
1348 apply: (deliveries) =>
1349 runtime.provide(
1350 Effect.orDie(Effect.forEach(deliveries, (d) => intake(key, d), { discard: true })),
1351 ),
1352 after: runtime.provide(state.waitUntil(outsideAlarmPass(drain.pipe(Effect.ignoreCause)))),
1353 }));
1354 const keep = (key: string) => state.storage.put(ID_KEY, key);
1355 return {
1356 receive: (key: string, delivery: Delivery) =>
1357 reportDefects(
1358 Effect.gen(function* () {
1359 yield* keep(key);
1360 if (!(yield* consumerFor(key).admit(target, [delivery]))) return;
1361 const accepted = yield* exclusively(state.storage.transaction(intake(key, delivery)));
1362 if (accepted)
1363 yield* state.waitUntil(outsideAlarmPass(drain.pipe(Effect.ignoreCause)));
1364 }),
1365 ),
1366 checkpoints: () => reportDefects(readMeta.pipe(Effect.map((meta) => meta.checkpoints))),
1367 // The backfill protocol, as a consumer and a scanner.
1368 ...scannerRpc(() => scannerFor()),
1369 ...runRpc(relay, runFor),
1370 bfPush: (key: string, target: string, deliveries: ReadonlyArray<Delivery>) =>
1371 reportDefects(keep(key).pipe(Effect.andThen(consumerFor(key).push(target, deliveries)))),
1372 bfConsumerReady: (key: string, target: string, waitMillis?: number) =>
1373 reportDefects(keep(key).pipe(Effect.andThen(consumerFor(key).ready(target, waitMillis)))),
1374 bfStart: (key: string, target: string, ref: Backfill.SourceRef, waitMillis?: number) =>
1375 reportDefects(
1376 keep(key).pipe(Effect.andThen(consumerFor(key).start(target, ref, waitMillis))),
1377 ),
1378 };
1379 });
1380 });
1381
1382 const streamObject = (kit: StreamKit) =>
1383 Effect.gen(function* () {
1384 const state = yield* DurableObjectState;
1385 return Effect.gen(function* () {
1386 const readMeta = state.storage
1387 .get<StreamMeta>("meta")
1388 .pipe(Effect.map((meta) => meta ?? emptyStreamMeta));
1389 const entriesAfter = (after: number) =>
1390 state.storage
1391 .list<StreamEntry>({ prefix: "entry:", start: `entry:${pad(after + 1)}` })
1392 .pipe(Effect.map((rows) => [...rows.values()]));
1393 const latest = (meta: StreamMeta) =>
1394 meta.seq === 0
1395 ? Effect.undefined
1396 : state.storage.get<StreamEntry>(`entry:${pad(meta.seq)}`);
1397 return {
1398 append: (key: string, write: StreamWrite) =>
1399 Effect.gen(function* () {
1400 const meta = yield* readMeta;
1401 if (!isAfter(write.origin, meta.origin)) return;
1402 const added: Array<StreamEntry> = write.events.map((event, index) => {
1403 const seq = meta.seq + index + 1;
1404 return {
1405 seq,
1406 event,
1407 envelope: {
1408 id: key,
1409 stream: `${kit.name}/${key}`,
1410 seq,
1411 at: write.at,
1412 commandId: write.commandId,
1413 },
1414 };
1415 });
1416 const next: StreamMeta = {
1417 seq: meta.seq + added.length,
1418 origin: write.origin,
1419 checkpoints: mergeCheckpoints(meta.checkpoints, {
1420 [write.source]: write.sourceSeq,
1421 }),
1422 };
1423 yield* state.storage.transaction(
1424 Effect.gen(function* () {
1425 const rows: Record<string, StreamEntry> = {};
1426 for (const entry of added) rows[`entry:${pad(entry.seq)}`] = entry;
1427 if (added.length > 0) yield* state.storage.put(rows);
1428 yield* state.storage.put("meta", next);
1429 }),
1430 );
1431 if (added.length === 0) return;
1432 yield* broadcast(({ key, subscription }) =>
1433 Effect.gen(function* () {
1434 // A subscriber behind this append's first entry reads the gap from storage.
1435 const entries =
1436 subscription.mode === "events" && subscription.cursor < added[0]!.seq - 1
1437 ? yield* entriesAfter(subscription.cursor)
1438 : added;
1439 return yield* kit.push(key, subscription, next, entries, added[added.length - 1]);
1440 }),
1441 );
1442 }).pipe(exclusively, reportDefects),
1443 advance: (checkpoints: Readonly<Record<string, number>>) =>
1444 Effect.gen(function* () {
1445 const meta = yield* readMeta;
1446 yield* state.storage.put("meta", {
1447 ...meta,
1448 checkpoints: mergeCheckpoints(meta.checkpoints, checkpoints),
1449 } satisfies StreamMeta);
1450 }).pipe(exclusively, reportDefects),
1451 read: () => reportDefects(readMeta),
1452 after: (after: number) => reportDefects(entriesAfter(after)),
1453 // A live read of this key: accept its WebSocket, then push on every append.
1454 fetch: accept((key, subscription) =>
1455 Effect.gen(function* () {
1456 const meta = yield* readMeta;
1457 const entries =
1458 subscription.mode === "events" ? yield* entriesAfter(subscription.cursor) : [];
1459 return yield* kit.push(key, subscription, meta, entries, yield* latest(meta));
1460 }),
1461 ),
1462 ...socketHandlers,
1463 };
1464 });
1465 });
1466
1467 /** How often a subscription re-reads the Durable Object that owns a key. */
1468 const POLL = Schedule.spaced("300 millis");
1469
1470 /** Entries after a cursor, then new ones, by polling the object that owns them. */
1471 const pollAfter = <A>(
1472 from: number,
1473 read: (cursor: number) => Effect.Effect<ReadonlyArray<A>>,
1474 seqOf: (entry: A) => number,
1475 ): Stream.Stream<A> =>
1476 Stream.suspend(() => {
1477 let first = true;
1478 return Stream.paginate(from, (cursor: number) => {
1479 const poll = read(cursor);
1480 const next = first ? poll : Effect.delay(poll, "300 millis");
1481 first = false;
1482 return Effect.map(
1483 next,
1484 (fresh) =>
1485 [
1486 fresh,
1487 Option.some(fresh.length > 0 ? seqOf(fresh[fresh.length - 1]!) : cursor),
1488 ] as const,
1489 );
1490 });
1491 });
1492
1493 /** The aggregates, views, policies, projections and streams, each on its own Durable Object class. */
1494 /** A model's Durable Object class name, like `Billing.AccountAggregate`. */
1495 const className = (modelName: string, name: string) =>
1496 `${modelName}.${name}`.replace(/[^A-Za-z0-9_.]/g, "_");
1497
1498 /**
1499 * Tell the Worker deploy about a class a model hosts and the name it had
1500 * before Domain version 0, so the deploy carries its data over. See
1501 * `Version.ts`.
1502 */
1503 const record = (modelName: string, legacyName: string, legacyData: boolean, name = legacyName) =>
1504 Effect.gen(function* () {
1505 const worker = yield* CloudflareWorker;
1506 const current = className(modelName, name);
1507 yield* worker.bind`${current}.DomainVersion`({
1508 domainClasses: [{ model: modelName, className: current, legacyName, legacyData }],
1509 });
1510 });
1511
1512 /** The models each Worker hosts, by name. */
1513 const claims = new WeakMap<object, ReturnType<typeof hostedModels>>();
1514 const claimsOf = (host: object) => {
1515 let claim = claims.get(host);
1516 if (claim === undefined) claims.set(host, (claim = hostedModels({ release: false })));
1517 return claim;
1518 };
1519
1520 const durableObjectPlatform = (
1521 jurisdiction: DurableObjectJurisdiction | undefined,
1522 /** How long a deploy waits for a backfill, in seconds. */
1523 backfillTimeoutSeconds: number | undefined,
1524 /** The Domain version to lay models out with: `undefined` for the layout from before versions. */
1525 version: number | undefined,
1526 legacySagas: Map<string, string>,
1527 ): Layer.Layer<Platform, never, Worker> =>
1528 Layer.effect(
1529 Platform,
1530 Effect.gen(function* () {
1531 /** A Durable Object namespace from the Worker's bindings, in the platform's jurisdiction. */
1532 const fromBinding = (native: any): Namespace => {
1533 const wrap = (namespace: any): Namespace => ({
1534 getByName: (name: string) => makeRpcStub(namespace.getByName(name)),
1535 jurisdiction: (inside: DurableObjectJurisdiction) => wrap(namespace.jurisdiction(inside)),
1536 });
1537 return wrap(jurisdiction === undefined ? native : native.jurisdiction(jurisdiction));
1538 };
1539 // Every kind of object is declared through here, so none is left
1540 // outside the jurisdiction.
1541 const declareIn = (name: string, impl: ObjectImpl) =>
1542 Effect.map(declare(name, impl), (namespace) =>
1543 jurisdiction === undefined ? namespace : namespace.jurisdiction(jurisdiction),
1544 );
1545 // Factories run while the Domain's Layer is built, inside the Worker's
1546 // init phase; declaring a Durable Object needs that context.
1547 const init = yield* captureContext<Worker>();
1548 // The Worker's bindings, at runtime: see `bound`.
1549 const env = Option.getOrUndefined(yield* Effect.serviceOption(WorkerEnvironment));
1550 const inInit = <A, E>(effect: Effect.Effect<A, E, Worker>): Effect.Effect<A, E> =>
1551 Effect.provide(effect, init);
1552
1553 // Model names are unique per Worker, not per platform build: two
1554 // different models named alike would declare the same classes. The
1555 // same model hosted twice declares them again, which is harmless.
1556 const claim = claimsOf(yield* CloudflareWorker);
1557 const hosts = new Map<string, ModelHost>();
1558 // Everything of a model is its own: its objects, named after it
1559 // (`Billing.AccountAggregate`), and its backfill's directory, runs and
1560 // Workflow. Models that share a definition never share its data.
1561 const hostFor = ({ modelName, legacyData }: HostedModel) =>
1562 Effect.gen(function* () {
1563 const named = (name: string) =>
1564 version === undefined ? name : className(modelName, name);
1565 const object = (name: string, impl: ObjectImpl) =>
1566 version === undefined
1567 ? declareIn(name, impl)
1568 : Effect.andThen(record(modelName, name, legacyData), declareIn(named(name), impl));
1569 // Every consumer in the model this isolate runs, set once the model is built.
1570 let model: ReadonlyArray<string> = [];
1571 let planOf: PlanOf = () => undefined;
1572 const consumerPlans: PlanOf = (target) => planOf(target);
1573 const current = () => model;
1574 // The objects the backfill protocol reaches: directory nodes, sources by name,
1575 // consumers by kind and name.
1576 let directory: Namespace | undefined;
1577 const sources = new Map<string, Namespace>();
1578 const consumers = new Map<string, Namespace>();
1579 const nodeStub = (path: string) => directory!.getByName(`node:${path}`);
1580 /**
1581 * A class by its binding name, from the Worker's bindings. An isolate that
1582 * hosts only directory nodes never builds the model, so its maps are empty:
1583 * it still reaches every aggregate, view and projection this way.
1584 */
1585 const bound = (binding: string): Namespace | undefined => {
1586 const namespace = env?.[binding];
1587 return namespace !== undefined && typeof namespace.getByName === "function"
1588 ? fromBinding(namespace)
1589 : undefined;
1590 };
1591 /** The generation of each view in this code, as a source: its keys' class. */
1592 const viewGenerations = new Map<string, number>();
1593 /** A source's class: an aggregate's, or a view generation's (a rebuilt view's keys are its class's). */
1594 const sourceNamespace = (ref: Backfill.SourceRef) => {
1595 const view = named(
1596 consumerClass(
1597 "view",
1598 ref.source,
1599 ref.generation ?? viewGenerations.get(ref.source) ?? 0,
1600 ),
1601 );
1602 return (
1603 sources.get(named(`${ref.source}Aggregate`)) ??
1604 bound(named(`${ref.source}Aggregate`)) ??
1605 sources.get(view) ??
1606 bound(view)
1607 );
1608 };
1609 const sourceObject = (namespace: Namespace, ref: Backfill.SourceRef) =>
1610 namespace.getByName(ref.id);
1611 /** A consumer's keys (`key`), or its run's root, relays and scanners (`run`): see `runClass`. */
1612 const consumerStub = (target: string, key: string, role: "key" | "run" = "key") => {
1613 const { kind, name } = Backfill.consumerOf(target);
1614 const keys = named(consumerClass(kind, name, generationOfTarget(target)));
1615 const cls = role === "run" ? runClass(keys) : keys;
1616 return (consumers.get(cls) ?? bound(cls))?.getByName(key);
1617 };
1618 const consumerOf = (target: string, key: string, role: "key" | "run" = "key") => {
1619 const stub = consumerStub(target, key, role);
1620 return stub === undefined
1621 ? Effect.die(new Error(`backfill: ${target} isn't in the code this object runs yet`))
1622 : Effect.succeed(stub);
1623 };
1624 /** Runs this isolate has seen complete: a key asks once per isolate, not on every first delivery. */
1625 const complete = new Set<string>();
1626 const net: Backfill.Network = {
1627 node: {
1628 register: (path, ref) => unwrapDefects(nodeStub(path).register(path, ref)),
1629 child: (path, child) => unwrapDefects(nodeStub(path).child(path, child)),
1630 list: (path, after) => callOnce(nodeStub(path).list(path, after)),
1631 },
1632 source: {
1633 // A source no longer in the model has nothing to give.
1634 read: (ref, target, key, after, limit) => {
1635 const namespace = sourceNamespace(ref);
1636 return namespace
1637 ? callOnce<Backfill.Read>(
1638 sourceObject(namespace, ref).bfRead(ref.id, target, key, after, limit),
1639 )
1640 : Effect.succeed({ from: after, rows: [], handoff: "live" as const });
1641 },
1642 cursor: (ref, target) => {
1643 const namespace = sourceNamespace(ref);
1644 return namespace
1645 ? callOnce<number>(sourceObject(namespace, ref).bfCursor(ref.id, target))
1646 : Effect.succeed(0);
1647 },
1648 log: (ref, after, limit) => {
1649 const namespace = sourceNamespace(ref);
1650 return namespace
1651 ? callOnce(sourceObject(namespace, ref).bfLog(ref.id, after, limit))
1652 : Effect.succeed([]);
1653 },
1654 handoff: (ref, target, key, through) => {
1655 const namespace = sourceNamespace(ref);
1656 return namespace
1657 ? callOnce(sourceObject(namespace, ref).bfHandoff(ref.id, target, key, through))
1658 : Effect.succeed("live" as const);
1659 },
1660 },
1661 // A consumer this isolate can't reach runs code older than the announcement
1662 // (its class is new): fail the call, and the caller asks again once it can.
1663 consumer: {
1664 push: (target, key, deliveries) =>
1665 Effect.flatMap(consumerOf(target, key), (stub) =>
1666 callOnce(stub.bfPush(key, target, deliveries)),
1667 ),
1668 ready: (target, key, waitMillis) =>
1669 Effect.flatMap(consumerOf(target, key), (stub) =>
1670 callOnce(stub.bfConsumerReady(key, target, waitMillis)),
1671 ),
1672 start: (target, key, ref, waitMillis) =>
1673 Effect.flatMap(consumerOf(target, key), (stub) =>
1674 callOnce(stub.bfStart(key, target, ref, waitMillis)),
1675 ),
1676 },
1677 // A source instance's scanner is an object of the consumer's own class: it has its routing.
1678 scanner: {
1679 scan: (target, ref, waitMillis) =>
1680 Effect.flatMap(consumerOf(target, scannerName(ref), "run"), (stub) =>
1681 callOnce(stub.bfScannerScan(target, ref, waitMillis)),
1682 ),
1683 ready: (target, ref, waitMillis) =>
1684 Effect.flatMap(consumerOf(target, scannerName(ref), "run"), (stub) =>
1685 callOnce(stub.bfScannerReady(target, waitMillis)),
1686 ),
1687 kick: (target, ref) =>
1688 Effect.flatMap(consumerOf(target, scannerName(ref), "run"), (stub) =>
1689 callOnce(stub.bfScannerKick(target, ref)),
1690 ),
1691 },
1692 relay: {
1693 run: (name, run, items) =>
1694 Effect.flatMap(consumerOf(run.target, relayName(run.target, name), "run"), (stub) =>
1695 callOnce<Backfill.RelayResult>(stub.bfRelayRun(name, run, items)),
1696 ),
1697 },
1698 run: {
1699 begin: (target, plan) =>
1700 Effect.flatMap(consumerOf(target, RUN_ROOT, "run"), (stub) =>
1701 callOnce<Backfill.Begin>(stub.bfRunBegin(target, plan)),
1702 ),
1703 phase: (target) =>
1704 complete.has(target)
1705 ? Effect.succeed("complete" as const)
1706 : Effect.flatMap(consumerOf(target, RUN_ROOT, "run"), (stub) =>
1707 callOnce<Backfill.Phase>(stub.bfRunPhase(target)),
1708 ).pipe(
1709 Effect.tap((phase) =>
1710 Effect.sync(() => {
1711 if (phase === "complete") complete.add(target);
1712 }),
1713 ),
1714 ),
1715 progress: (target) =>
1716 Effect.flatMap(consumerOf(target, RUN_ROOT, "run"), (stub) =>
1717 callOnce<Backfill.Progress>(stub.bfRunProgress(target)),
1718 ),
1719 },
1720 };
1721 directory = yield* inInit(object("DomainDirectory", directoryObject(net)));
1722
1723 // A deploy's backfill: start each new consumer's run, then wait until every one is
1724 // complete. Reconciliation starts it and waits for it.
1725 const backfill = yield* inInit(
1726 Workflow(
1727 named("DomainBackfill"),
1728 Effect.succeed(
1729 (params: {
1730 readonly targets: ReadonlyArray<string>;
1731 readonly plans?: Readonly<Record<string, Backfill.Plan>>;
1732 }) =>
1733 Effect.gen(function* () {
1734 const plans = { ...plansOf(params.targets, planOf), ...params.plans };
1735 const targets = params.targets.filter((target) => plans[target] !== undefined);
1736 // This isolate doesn't build the model: reach each run's root by its binding.
1737 const workerEnv = yield* WorkerEnvironment;
1738 const root = (target: string) =>
1739 Effect.suspend(() => {
1740 const { kind, name } = Backfill.consumerOf(target);
1741 const namespace =
1742 workerEnv[
1743 runClass(named(consumerClass(kind, name, generationOfTarget(target))))
1744 ];
1745 return namespace === undefined
1746 ? Effect.die(
1747 new Error(`backfill: ${target} isn't in the code this runs yet`),
1748 )
1749 : Effect.succeed(fromBinding(namespace).getByName(RUN_ROOT));
1750 });
1751 for (const target of targets) {
1752 const begun = yield* task(
1753 `begin-${target}`,
1754 reachable(
1755 Effect.flatMap(root(target), (stub) =>
1756 unwrapDefects<Backfill.Begin>(stub.bfRunBegin(target, plans[target]!)),
1757 ),
1758 ),
1759 { retries: REACH_RETRIES },
1760 );
1761 // Started reading other sources: waiting on it would report a history
1762 // that misses the new ones. The deploy fails, saying to bump the version.
1763 if (begun.status === "changed")
1764 return yield* Effect.die(
1765 new Backfill.PlanChanged({
1766 message: Backfill.planChanged(target, begun.started, plans[target]!),
1767 }),
1768 );
1769 }
1770 // Each check's output is the runs' progress: the deploy reads the latest one
1771 // from the instance's steps, and reports it while it waits.
1772 for (let check = 0; ; check++) {
1773 const progress = yield* task(
1774 `check-${check}`,
1775 reachable(
1776 Effect.forEach(targets, (target) =>
1777 Effect.flatMap(root(target), (stub) =>
1778 unwrapDefects<Backfill.Progress>(stub.bfRunProgress(target)),
1779 ),
1780 ),
1781 ),
1782 { retries: REACH_RETRIES },
1783 );
1784 if (progress.every((run) => run.phase === "complete"))
1785 return { complete: true, progress };
1786 yield* sleep(
1787 `wait-${check}`,
1788 // Soon at first, then less often: each check is a step (an instance has 10,000).
1789 `${Math.min(5_000, Math.round(500 * 1.25 ** check))} millis`,
1790 );
1791 }
1792 }),
1793 ) as unknown as Effect.Effect<WorkflowImpl<any, any, unknown>>,
1794 ),
1795 );
1796
1797 // The first Domain call in each isolate starts the runs of the model's consumers
1798 // (each starts once): locally, where no Workflow runs, that starts a catch-up.
1799 let news: Effect.Effect<void> | undefined;
1800 const announce = Effect.suspend(() => {
1801 const pending = news;
1802 if (pending === undefined) return Effect.void;
1803 news = undefined;
1804 return pending.pipe(
1805 Effect.timeout("5 seconds"),
1806 Effect.catchCause(() =>
1807 Effect.sync(() => {
1808 news = pending;
1809 }),
1810 ),
1811 );
1812 });
1813
1814 const host: ModelHost = {
1815 aggregate: (aggregate, kit) =>
1816 inInit(
1817 Effect.gen(function* () {
1818 const namespace = yield* object(
1819 `${aggregate.aggregateName}Aggregate`,
1820 aggregateObject(kit, net, current, consumerPlans),
1821 );
1822 sources.set(named(`${aggregate.aggregateName}Aggregate`), namespace);
1823 return {
1824 send: (id, command, meta) =>
1825 announce.pipe(
1826 Effect.andThen(
1827 unwrapDefects<SendResult>(
1828 namespace.getByName(id).send(id, command, meta),
1829 ),
1830 ),
1831 ),
1832 // The object answers with a tag (RPC loses typed failures); fail with it here.
1833 schedule: (id, command, meta, due) =>
1834 unwrapDefects<{ readonly at: number } | "AlreadyExecuted">(
1835 namespace.getByName(id).schedule(id, command, meta, due),
1836 ).pipe(
1837 Effect.flatMap((outcome) =>
1838 outcome === "AlreadyExecuted"
1839 ? Effect.fail(
1840 new AlreadyExecuted({
1841 aggregate: aggregate.aggregateName,
1842 id,
1843 commandId: meta.commandId,
1844 }),
1845 )
1846 : Effect.succeed(outcome.at),
1847 ),
1848 ),
1849 unschedule: (id, commandId) =>
1850 unwrapDefects<
1851 "Cancelled" | "AlreadyCancelled" | "AlreadyExecuted" | "NotScheduled"
1852 >(namespace.getByName(id).unschedule(commandId)).pipe(
1853 Effect.flatMap(
1854 (
1855 outcome,
1856 ): Effect.Effect<
1857 void,
1858 AlreadyExecuted | AlreadyCancelled | NotScheduled
1859 > => {
1860 const fields = { aggregate: aggregate.aggregateName, id, commandId };
1861 if (outcome === "AlreadyExecuted")
1862 return Effect.fail(new AlreadyExecuted(fields));
1863 if (outcome === "AlreadyCancelled")
1864 return Effect.fail(new AlreadyCancelled(fields));
1865 if (outcome === "NotScheduled")
1866 return Effect.fail(new NotScheduled(fields));
1867 return Effect.void;
1868 },
1869 ),
1870 ),
1871 state: (id) =>
1872 announce.pipe(
1873 Effect.andThen(
1874 unwrapDefects<{ state: Json; version: number }>(
1875 namespace.getByName(id).state(),
1876 ),
1877 ),
1878 ),
1879 // Streamed responses poll; WebSocket reads (`connect`) are pushed on commit.
1880 changes: (id) =>
1881 Stream.fromEffectSchedule(
1882 unwrapDefects<{ state: Json; version: number }>(
1883 namespace.getByName(id).state(),
1884 ),
1885 POLL,
1886 ).pipe(Stream.changesWith((a, b) => a.version === b.version)),
1887 tail: (id, from) =>
1888 pollAfter(
1889 from,
1890 (cursor) =>
1891 unwrapDefects<ReadonlyArray<StoredEvent>>(
1892 namespace.getByName(id).after(cursor),
1893 ),
1894 (event) => event.envelope.seq,
1895 ),
1896 connect: connectTo(namespace),
1897 };
1898 }),
1899 ),
1900 view: (view, kit) =>
1901 inInit(
1902 Effect.gen(function* () {
1903 // A view rebuilt in place keeps its class: see `Kernel.inPlace`.
1904 const cls = consumerClass("view", view.viewName, kit.classGeneration);
1905 const namespace = yield* object(
1906 cls,
1907 viewObject(kit, net, current, consumerPlans),
1908 );
1909 sources.set(named(cls), namespace);
1910 viewGenerations.set(view.viewName, kit.classGeneration);
1911 consumers.set(named(cls), namespace);
1912 consumers.set(
1913 runClass(named(cls)),
1914 yield* object(runClass(cls), viewObject(kit, net, current, consumerPlans)),
1915 );
1916 const target = kit.target;
1917 // A lazy view's key reads its history from its one source the first
1918 // time it is read; each isolate asks a key once.
1919 const caughtUp = new Set<string>();
1920 const ready = (key: string): Effect.Effect<void> =>
1921 Effect.suspend(() => {
1922 const catchUp = kit.catchUp;
1923 if (catchUp === undefined || caughtUp.has(key)) return Effect.void;
1924 return net.consumer
1925 .start(target, key, { source: catchUp.source, id: key }, 20_000)
1926 .pipe(
1927 Effect.repeat({ until: (started) => started === "settled", times: 5 }),
1928 Effect.tap((started) =>
1929 Effect.sync(() => {
1930 if (started === "settled") caughtUp.add(key);
1931 }),
1932 ),
1933 Effect.asVoid,
1934 );
1935 });
1936 /**
1937 * The key computed here, with this code, from its source's events.
1938 * A key's object still on the code from before a deploy (until it is
1939 * idle) can't fill itself for a version it doesn't know: until it
1940 * moves, its reads are answered this way, and nothing is stored.
1941 */
1942 const computed = (key: string) =>
1943 Effect.gen(function* () {
1944 const source = kit.catchUp!.source;
1945 const deliveries: Array<Delivery> = [];
1946 for (let after = 0; ;) {
1947 const rows = yield* net.source.log({ source, id: key }, after, 1_000);
1948 for (const row of yield* kit.history(target, source, key, rows))
1949 deliveries.push(...row.deliveries);
1950 if (rows.length < 1_000) break;
1951 after += rows.length;
1952 }
1953 return yield* kit.apply(key, emptySnapshot, deliveries);
1954 });
1955 /** Read from the key's object, or compute it here while that object can't fill itself. */
1956 const orComputed = <A>(
1957 fromObject: Effect.Effect<A>,
1958 fromHere: (result: Effect.Success<ReturnType<typeof computed>>) => A,
1959 key: string,
1960 ): Effect.Effect<A> =>
1961 ready(key).pipe(
1962 Effect.andThen(fromObject),
1963 Effect.catchCause((cause) =>
1964 kit.catchUp === undefined
1965 ? Effect.failCause(cause)
1966 : Effect.map(computed(key), fromHere),
1967 ),
1968 );
1969 const read = (key: string) =>
1970 announce.pipe(
1971 Effect.andThen(
1972 orComputed(
1973 unwrapDefects<ViewSnapshot>(namespace.getByName(key).read()),
1974 (result) => result.snapshot,
1975 key,
1976 ),
1977 ),
1978 );
1979 const after = (key: string, cursor: number) =>
1980 orComputed(
1981 unwrapDefects<ReadonlyArray<ViewEntry>>(
1982 namespace.getByName(key).after(cursor),
1983 ),
1984 (result) => result.entries.filter((entry) => entry.seq > cursor),
1985 key,
1986 );
1987 return {
1988 receive: (key, deliveries) =>
1989 unwrapDefects(namespace.getByName(key).receive(key, deliveries)),
1990 read,
1991 // Streamed responses poll; WebSocket reads (`connect`) are pushed on commit.
1992 changes: (key) =>
1993 Stream.fromEffectSchedule(read(key), POLL).pipe(
1994 Stream.changesWith((a, b) => a.version === b.version),
1995 ),
1996 list: (key) => after(key, 0),
1997 connect: (key, subscription) =>
1998 ready(key).pipe(Effect.andThen(connectTo(namespace)(key, subscription))),
1999 tail: (key, from) =>
2000 Stream.suspend(() => {
2001 let first = true;
2002 return Stream.paginate(from, (cursor: number) => {
2003 const poll = after(key, cursor);
2004 const next = first ? poll : Effect.delay(poll, "300 millis");
2005 first = false;
2006 return Effect.map(
2007 next,
2008 (fresh) =>
2009 [
2010 fresh,
2011 Option.some(
2012 fresh.length > 0 ? fresh[fresh.length - 1]!.seq : cursor,
2013 ),
2014 ] as const,
2015 );
2016 });
2017 }),
2018 };
2019 }),
2020 ),
2021 projection: (projection, kit) =>
2022 inInit(
2023 Effect.gen(function* () {
2024 const cls = consumerClass(
2025 "projection",
2026 projection.projectionName,
2027 kit.generation,
2028 );
2029 const namespace = yield* object(cls, projectionObject(kit, net, consumerPlans));
2030 consumers.set(named(cls), namespace);
2031 consumers.set(
2032 runClass(named(cls)),
2033 yield* object(runClass(cls), projectionObject(kit, net, consumerPlans)),
2034 );
2035 return {
2036 receive: (delivery) =>
2037 unwrapDefects(
2038 namespace.getByName(delivery.key).receive(delivery.key, delivery),
2039 ),
2040 checkpoints: (key) =>
2041 unwrapDefects<Readonly<Record<string, number>>>(
2042 namespace.getByName(key).checkpoints(),
2043 ),
2044 };
2045 }),
2046 ),
2047 stream: (stream, kit) =>
2048 inInit(
2049 Effect.gen(function* () {
2050 const namespace = yield* object(`${stream.streamName}Stream`, streamObject(kit));
2051 const read = (key: string) =>
2052 unwrapDefects<StreamMeta>(namespace.getByName(key).read());
2053 const after = (key: string, cursor: number) =>
2054 unwrapDefects<ReadonlyArray<StreamEntry>>(
2055 namespace.getByName(key).after(cursor),
2056 );
2057 return {
2058 append: (key, write) =>
2059 unwrapDefects(namespace.getByName(key).append(key, write)),
2060 advance: (key, checkpoints) =>
2061 unwrapDefects(namespace.getByName(key).advance(checkpoints)),
2062 read,
2063 // Streamed responses poll; WebSocket reads (`connect`) are pushed on append.
2064 changes: (key) =>
2065 Stream.fromEffectSchedule(read(key), POLL).pipe(
2066 Stream.changesWith((a, b) => a.seq === b.seq),
2067 ),
2068 list: (key) => after(key, 0),
2069 tail: (key, from) =>
2070 pollAfter(
2071 from,
2072 (cursor) => after(key, cursor),
2073 (e) => e.seq,
2074 ),
2075 connect: connectTo(namespace),
2076 };
2077 }),
2078 ),
2079 history: (target) =>
2080 net.run
2081 .phase(target)
2082 .pipe(
2083 Effect.map((phase) => ({ complete: phase === "complete", through: undefined })),
2084 ),
2085 start: (targets, plansOfTarget, backfilled) =>
2086 Effect.gen(function* () {
2087 // At deploy time: reconciliation runs the backfill of new consumers and waits.
2088 if (!globalThis.__ALCHEMY_RUNTIME__ && backfilled.length > 0) {
2089 yield* inInit(
2090 Effect.gen(function* () {
2091 const worker = yield* CloudflareWorker;
2092 yield* BackfillRun(named("DomainBackfillRun"), {
2093 scriptName: worker.workerName as unknown as string,
2094 workflowName: backfill.workflowName as string,
2095 targets: [...backfilled],
2096 plans: plansOf(backfilled, plansOfTarget) as Record<
2097 string,
2098 { sources?: string[]; singleSource: boolean }
2099 >,
2100 ...(backfillTimeoutSeconds === undefined
2101 ? {}
2102 : { timeoutSeconds: backfillTimeoutSeconds }),
2103 }) as unknown as Effect.Effect<unknown>;
2104 }),
2105 );
2106 }
2107 }).pipe(
2108 Effect.andThen(
2109 Effect.sync(() => {
2110 model = targets;
2111 planOf = plansOfTarget;
2112 if (backfilled.length === 0) return;
2113 // Locally no Workflow runs: the first call starts each run (once).
2114 news = Effect.suspend(() => {
2115 const plans = plansOf(backfilled, planOf);
2116 return Effect.forEach(
2117 backfilled.filter((target) => plans[target] !== undefined),
2118 (target) => net.run.begin(target, plans[target]!),
2119 { discard: true },
2120 );
2121 });
2122 }),
2123 ),
2124 ),
2125 policy: (policy, kit) =>
2126 inInit(
2127 Effect.gen(function* () {
2128 const namespace = yield* object(`${policy.policyName}Policy`, policyObject(kit));
2129 return {
2130 receive: (delivery) =>
2131 unwrapDefects(namespace.getByName(delivery.key).receive(delivery)),
2132 };
2133 }),
2134 ),
2135 };
2136 return host;
2137 });
2138
2139 return Platform.of({
2140 model: (model) =>
2141 Effect.gen(function* () {
2142 yield* claim(model.modelName, model.id);
2143 // Each saga's runs live on a Durable Object class of the workflow
2144 // engine, named after the model too.
2145 for (const saga of model.sagas)
2146 if (version === undefined) legacySagas.set(`${model.modelName}.${saga}`, saga);
2147 else
2148 yield* inInit(
2149 record(
2150 model.modelName,
2151 `${saga.replace(/[^A-Za-z0-9_]/g, "_")}Workflow`,
2152 model.legacyData,
2153 `${saga}Workflow`,
2154 ),
2155 );
2156 const hosted = hosts.get(model.modelName);
2157 if (hosted !== undefined) return hosted;
2158 const host = yield* hostFor(model);
2159 hosts.set(model.modelName, host);
2160 return host;
2161 }),
2162 });
2163 }),
2164 );
2165
2166 /**
2167 * Host a domain model on Cloudflare Durable Objects. Every aggregate, view
2168 * and policy becomes a Durable Object class on the host Worker, declared
2169 * while the model's Layer is built, so there are no bindings to write. One
2170 * instance per aggregate id, per view key, and per policy and source.
2171 *
2172 * Commands to one instance run one at a time, and their events, state and
2173 * outbox commit together. Deliveries to views and policies are at least once
2174 * and in order per source, and survive restarts. Live reads over
2175 * `Domain.protocolWebSocket` are WebSockets to the object that owns the key,
2176 * pushed on every commit; the object hibernates in between.
2177 *
2178 * A view or projection added in a later deploy receives the history recorded
2179 * before it, and the deploy waits until it has. Every instance lists itself in
2180 * a directory of Durable Objects; a run of new objects reads it, fans out six
2181 * ways, and gives each key of the new consumer its sources' events, which the
2182 * key applies in timestamp order.
2183 *
2184 * ### Hosting on Cloudflare
2185 * **Example:** A Worker serving an Api
2186 * ```typescript
2187 * import * as Cloudflare from "@alchemy.run/cloudflare";
2188 * import * as DomainCloudflare from "@alchemy.run/cloudflare/Domain";
2189 * import * as Effect from "effect/Effect";
2190 * import * as Layer from "effect/Layer";
2191 * import { Bank, BankPolicies } from "./Bank.ts";
2192 * import { CustomerApi } from "./CustomerApi.ts";
2193 * import { SessionLive } from "./Session.ts";
2194 *
2195 * const BankLive = SessionLive.pipe(
2196 * Layer.provideMerge(Bank.layer(BankPolicies)),
2197 * Layer.provide(DomainCloudflare.layerDurableObjects()),
2198 * );
2199 *
2200 * export default class BankWorker extends Cloudflare.Worker<BankWorker>()(
2201 * "BankWorker",
2202 * { main: import.meta.url },
2203 * Effect.gen(function* () {
2204 * const api = yield* CustomerApi;
2205 * return { fetch: api.fetch };
2206 * }).pipe(Effect.provide(BankLive)),
2207 * ) {}
2208 * ```
2209 *
2210 * ### Restricting a model to a jurisdiction
2211 * **Example:** Every Durable Object of the model in the EU
2212 * ```typescript
2213 * import * as DomainCloudflare from "@alchemy.run/cloudflare/Domain";
2214 * import * as Layer from "effect/Layer";
2215 * import { Bank, BankPolicies } from "./Bank.ts";
2216 *
2217 * const BankLive = Bank.layer(BankPolicies).pipe(
2218 * Layer.provide(DomainCloudflare.layerDurableObjects({ jurisdiction: "eu" })),
2219 * );
2220 * ```
2221 *
2222 * ### Hosting several models in one Worker
2223 * Each model gets its own Durable Object classes, named after the model
2224 * (`Billing.AccountAggregate`), so models that list the same aggregate never
2225 * share its data. A deploy moves data a Worker deployed with an earlier Domain
2226 * layout to the current one; see `legacyData` on a model's `layer` for models
2227 * that shared a class before Domain versions existed.
2228 *
2229 * **Example:** Two models that both list `Account`
2230 * ```typescript
2231 * import * as DomainCloudflare from "@alchemy.run/cloudflare/Domain";
2232 * import * as Layer from "effect/Layer";
2233 * import { Audit, AuditHandlers } from "./Audit.ts";
2234 * import { Billing, BillingHandlers } from "./Billing.ts";
2235 *
2236 * const Platform = DomainCloudflare.layerDurableObjects();
2237 * const BillingLive = Billing.layer(BillingHandlers).pipe(Layer.provide(Platform));
2238 * const AuditLive = Audit.layer(AuditHandlers).pipe(Layer.provide(Platform));
2239 * ```
2240 *
2241 * ### Restricting a model to a jurisdiction
2242 * **Example:** Every Durable Object of the model in the EU
2243 * ```typescript
2244 * import * as DomainCloudflare from "@alchemy.run/cloudflare/Domain";
2245 * import * as Layer from "effect/Layer";
2246 * import { Bank, BankPolicies } from "./Bank.ts";
2247 *
2248 * const BankLive = Bank.layer(BankPolicies).pipe(
2249 * Layer.provide(DomainCloudflare.layerDurableObjects({ jurisdiction: "eu" })),
2250 * );
2251 * ```
2252 *
2253 * @layer
2254 * @provides Domain.Platform
2255 * @provides WorkflowEngine
2256 */
2257 export function layerDurableObjects({
2258 jurisdiction,
2259 backfillTimeout,
2260 }: DurableObjectsOptions = {}): Layer.Layer<
2261 Platform | WorkflowEngine.WorkflowEngine,
2262 never,
2263 Worker
2264 > {
2265 const timeoutSeconds =
2266 backfillTimeout === undefined ? undefined : Duration.toSeconds(backfillTimeout);
2267 // The same options return the same Layer, so models that each provide it
2268 // share one platform and one workflow engine.
2269 const key = `${jurisdiction ?? ""}|${timeoutSeconds ?? ""}`;
2270 let platform = platforms.get(key);
2271 if (platform === undefined)
2272 platforms.set(
2273 key,
2274 (platform = Layer.unwrap(
2275 Effect.gen(function* () {
2276 const version = yield* DeployVersion;
2277 const engine = layerDurableObjectEngine({ jurisdiction });
2278 const legacySagas = new Map<string, string>();
2279 return Layer.merge(
2280 durableObjectPlatform(jurisdiction, timeoutSeconds, version, legacySagas),
2281 version === undefined ? legacyEngine(legacySagas).pipe(Layer.provide(engine)) : engine,
2282 );
2283 }),
2284 )),
2285 );
2286 return platform;
2287 }
2288
2289 /**
2290 * Before Domain version 0, a saga's runs lived on a class named after the
2291 * saga alone. Store each under that name, given by `legacySagas`.
2292 */
2293 const legacyEngine = (
2294 legacySagas: ReadonlyMap<string, string>,
2295 ): Layer.Layer<WorkflowEngine.WorkflowEngine, never, WorkflowEngine.WorkflowEngine> =>
2296 Layer.effect(
2297 WorkflowEngine.WorkflowEngine,
2298 Effect.gen(function* () {
2299 const engine = yield* WorkflowEngine.WorkflowEngine;
2300 const legacy: WorkflowEngine.WorkflowEngine["Service"] = {
2301 ...engine,
2302 register: (workflow, execute) => {
2303 const storageName = Context.getOrUndefined(workflow.annotations, StorageName);
2304 const legacyName = storageName === undefined ? undefined : legacySagas.get(storageName);
2305 return engine.register(
2306 legacyName === undefined
2307 ? workflow
2308 : Object.create(workflow, {
2309 annotations: {
2310 value: Context.add(workflow.annotations, StorageName, legacyName),
2311 enumerable: true,
2312 },
2313 }),
2314 execute,
2315 );
2316 },
2317 };
2318 return legacy;
2319 }),
2320 );
2321
2322 /** Options for {@link layerDurableObjects}. */
2323 export interface DurableObjectsOptions {
2324 /**
2325 * Create and store every Durable Object of the model (aggregates, views,
2326 * policies, projections, streams and saga executions) only inside this
2327 * jurisdiction, e.g. `"eu"`.
2328 *
2329 * A Durable Object in a jurisdiction is a different object from the one
2330 * with the same name outside it. Setting this on a model that already has
2331 * data does not move that data: the model starts empty in the
2332 * jurisdiction, and the old objects are left as they were. Choose it before
2333 * the first deploy, or migrate deliberately.
2334 *
2335 * Like a D1 database, R2 bucket or Queue declared with a jurisdiction, it is
2336 * enforced when deployed and not emulated in `alchemy dev`: local workerd
2337 * does not implement jurisdictions, so the model runs unrestricted there.
2338 *
2339 * @default undefined (no jurisdiction restriction)
2340 */
2341 readonly jurisdiction?: DurableObjectJurisdiction;
2342 /**
2343 * How long a deploy waits for the backfill of the views and projections it
2344 * adds or rebuilds. Past it the deploy fails with `BackfillStillRunning`,
2345 * naming how far each one has got, while the backfill keeps running on
2346 * Cloudflare: the next deploy waits on it again, from where it is.
2347 *
2348 * @default "30 minutes"
2349 */
2350 readonly backfillTimeout?: Duration.Input;
2351 }
2352
2353 const platforms = new Map<
2354 string,
2355 Layer.Layer<Platform | WorkflowEngine.WorkflowEngine, never, Worker>
2356 >();