The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,643 lines 51.9 KB
Raw
1 // Alchemy modifications are licensed under Apache-2.0.
2 // This file includes third-party code; see /THIRD_PARTY_LICENSES.md.
3 // Alchemy modifications: uses Array<T> syntax for non-tuple array types to match the repository convention.
4 import { DurableObject } from "cloudflare:workers";
5 import type { WorkflowEntrypoint, WorkflowEvent, WorkflowStep } from "cloudflare:workers";
6 import type { RestartFromStep, WorkflowInstanceTerminateOptions } from "./binding.ts";
7 import { Context, REDACTED_STEP_OUTPUT } from "./context.ts";
8 import type { Event } from "./context.ts";
9 import {
10 INSTANCE_METADATA,
11 InstanceEvent,
12 InstanceStatus,
13 instanceStatusName,
14 InstanceTrigger,
15 toInstanceStatus,
16 } from "./instance.ts";
17 import type { InstanceMetadata, RawInstanceLog } from "./instance.ts";
18 import { computeHash } from "./lib/cache.ts";
19 import {
20 ABORT_REASONS,
21 createWorkflowError,
22 isAbortError,
23 PreservedNonRetryableError,
24 shouldPreserveNonRetryableError,
25 stepNotFoundError,
26 WorkflowFatalError,
27 } from "./lib/errors.ts";
28 import {
29 ENGINE_TIMEOUT,
30 GracePeriodSemaphore,
31 startGracePeriod,
32 } from "./lib/gracePeriodSemaphore.ts";
33 import {
34 readAndClearRestartFromStep,
35 resolveGroupKeysToWipe,
36 storeRestartFromStep,
37 wipeRestartState,
38 } from "./lib/restart.ts";
39 import {
40 clearRollbackRegistry,
41 disposeRollbackStub,
42 executeRollbacks,
43 registerRollbackFn,
44 ROLLBACK_CACHE_KEY_PREFIX,
45 } from "./lib/rollback.ts";
46 import type { RollbackRegistration, RollbackRegistryEntry } from "./lib/rollback.ts";
47 import { normalizeForStorage } from "./lib/serialization.ts";
48 import {
49 createReplayReadableStream,
50 getInvalidStoredStreamOutputError,
51 getStoredStreamOutputPreview,
52 getStreamOutputMetaKey,
53 StreamOutputState,
54 } from "./lib/streams.ts";
55 import type { StreamOutputMeta } from "./lib/streams.ts";
56 import { TimePriorityQueue } from "./lib/timePriorityQueue.ts";
57 import { MODIFIER_KEYS, WorkflowInstanceModifier } from "./modifier.ts";
58 import {
59 isTerminalEvent,
60 parseResolvedStepConfig,
61 WorkflowSubscriptionTarget,
62 } from "./subscription.ts";
63 import type {
64 WorkflowSubscriptionEvent,
65 WorkflowSubscriptionOptions,
66 WorkflowSubscriptionState,
67 } from "./subscription.ts";
68
69 interface Env {
70 ENGINE: DurableObjectNamespace<Engine>;
71 USER_WORKFLOW: WorkflowEntrypoint;
72 WORKFLOW_NAME?: string;
73 STEP_LIMIT?: string; // JSON-encoded number from miniflare binding
74 STEP_LIMITS?: Record<string, number>;
75 }
76
77 /**
78 * Set by workerd when the Engine runs a Workflow exposed on `ctx.exports`. A
79 * single Engine class then serves every Workflow of a Worker, so these replace
80 * the per-Workflow `USER_WORKFLOW` and `WORKFLOW_NAME` bindings (which are then
81 * absent), and the step limit is looked up by name in `STEP_LIMITS`.
82 */
83 interface EngineProps {
84 workflowClass?: WorkflowEntrypoint;
85 workflowName?: string;
86 }
87
88 export type DatabaseWorkflow = {
89 name: string;
90 id: string;
91 created_on: string;
92 modified_on: string;
93 script_name: string;
94 class_name: string | null;
95 triggered_on: string | null;
96 };
97
98 export type DatabaseVersion = {
99 id: string;
100 class_name: string;
101 created_on: string;
102 modified_on: string;
103 workflow_id: string;
104 mutable_pipeline_id: string;
105 };
106
107 export type DatabaseInstance = {
108 id: string;
109 created_on: string;
110 modified_on: string;
111 workflow_id: string;
112 version_id: string;
113 status: InstanceStatus;
114 started_on: string | null;
115 ended_on: string | null;
116 };
117
118 export type Log = {
119 event: InstanceEvent;
120 group: string | null;
121 target: string | null;
122 metadata: {
123 result: string;
124 payload: string;
125 error: { name: string; message: string };
126 };
127 };
128
129 export type EngineLogs = {
130 logs: Array<Log>;
131 };
132
133 const ENGINE_STATUS_KEY = "ENGINE_STATUS";
134 const WORKFLOW_OUTPUT_KEY = "WORKFLOW_OUTPUT";
135
136 const EVENT_MAP_PREFIX = "EVENT_MAP";
137
138 export const DEFAULT_STEP_LIMIT = 10_000;
139
140 const PAUSE_DATETIME = "PAUSE_DATETIME";
141
142 export type RollbackPhase = "replay" | "rollback";
143
144 /**
145 * JSON.stringify replacer that converts TypedArrays and ArrayBuffers to a
146 * human-readable description. Without this, JSON.stringify(Uint8Array) encodes
147 * each byte as a numeric key ({"0":1,"1":2,...}), producing a string ~10x larger
148 * than byteLength and causing SQLITE_TOOBIG for outputs above ~170 KB.
149 * The replacer is called recursively by JSON.stringify, so nested binary values
150 * inside objects or arrays are also handled.
151 */
152 function binaryReplacer(_key: string, value: unknown): unknown {
153 if (typeof value === "bigint") {
154 return `[BigInt(${value})]`;
155 }
156 if (value instanceof ArrayBuffer) {
157 return `[ArrayBuffer(${value.byteLength} bytes)]`;
158 }
159 if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
160 return `[${value.constructor.name}(${(value as ArrayBufferView).byteLength} bytes)]`;
161 }
162 return value;
163 }
164
165 function isStepSuccessEvent(event: InstanceEvent): boolean {
166 return event === InstanceEvent.STEP_SUCCESS || event === InstanceEvent.ROLLBACK_STEP_SUCCESS;
167 }
168
169 type EngineWorkflowSubscriptionEvent =
170 | WorkflowSubscriptionEvent
171 | (Pick<WorkflowSubscriptionEvent, "instanceId" | "eventId" | "timestamp"> & {
172 type: "internal";
173 });
174
175 async function readStepConfig(storage: DurableObjectStorage, groupKey: string | null) {
176 if (groupKey === null) {
177 return undefined;
178 }
179 return parseResolvedStepConfig(await storage.get(`${groupKey}-config`));
180 }
181
182 async function readStepCompletedOutput(storage: DurableObjectStorage, groupKey: string | null) {
183 if (groupKey === null) {
184 return undefined;
185 }
186
187 const valueKey = `${groupKey}-value`;
188 const streamMetaKey = getStreamOutputMetaKey(groupKey);
189 const configKey = `${groupKey}-config`;
190 const stored = await storage.get([valueKey, streamMetaKey, configKey]);
191 const config = parseResolvedStepConfig(stored.get(configKey));
192 if (config === undefined) {
193 return undefined;
194 }
195 if (config.sensitive === "output") {
196 return REDACTED_STEP_OUTPUT;
197 }
198
199 const streamMeta = stored.get(streamMetaKey) as StreamOutputMeta | undefined;
200 if (streamMeta?.state === StreamOutputState.Complete) {
201 const integrityError = getInvalidStoredStreamOutputError(storage, groupKey, streamMeta);
202 if (integrityError !== undefined) {
203 throw createWorkflowError(
204 "Step has completed but its stored stream output is corrupt or incomplete",
205 "instance.step_output_corrupt",
206 );
207 }
208 return createReplayReadableStream({
209 storage,
210 cacheKey: groupKey,
211 meta: streamMeta,
212 });
213 }
214
215 return (stored.get(valueKey) as { value: unknown } | undefined)?.value;
216 }
217
218 async function buildWorkflowSubscriptionEvent(
219 storage: DurableObjectStorage,
220 log: RawInstanceLog,
221 instanceId: string,
222 params: unknown,
223 ): Promise<EngineWorkflowSubscriptionEvent> {
224 const common = {
225 instanceId,
226 eventId: log.id,
227 timestamp: new Date(log.timestamp).valueOf(),
228 };
229 const parseMetadata = () =>
230 JSON.parse(log.metadata) as {
231 result?: unknown;
232 error: { name: string; message: string };
233 attempt: number;
234 retryDelayMs?: number;
235 durationMs: number;
236 event: string;
237 };
238 const stepEvent = (
239 createEvent: (stepName: string) => Extract<WorkflowSubscriptionEvent, { stepName: string }>,
240 ): EngineWorkflowSubscriptionEvent =>
241 log.target === null ? { ...common, type: "internal" } : createEvent(log.target);
242
243 switch (log.event) {
244 case InstanceEvent.WORKFLOW_QUEUED:
245 return { ...common, type: "workflow_queued" };
246 case InstanceEvent.WORKFLOW_START:
247 return { ...common, type: "workflow_started", params };
248 case InstanceEvent.WORKFLOW_RUNNING:
249 return { ...common, type: "workflow_running" };
250 case InstanceEvent.WORKFLOW_PAUSED:
251 return { ...common, type: "workflow_paused" };
252 case InstanceEvent.WORKFLOW_WAITING_FOR_PAUSE:
253 return { ...common, type: "workflow_waiting_for_pause" };
254 case InstanceEvent.WORKFLOW_WAITING:
255 return { ...common, type: "workflow_waiting" };
256 case InstanceEvent.WORKFLOW_SUCCESS: {
257 const storedOutput = await storage.get<{ value: unknown }>(WORKFLOW_OUTPUT_KEY);
258 return {
259 ...common,
260 type: "workflow_completed",
261 output: storedOutput === undefined ? parseMetadata().result : storedOutput.value,
262 };
263 }
264 case InstanceEvent.WORKFLOW_FAILURE:
265 return {
266 ...common,
267 type: "workflow_errored",
268 error: parseMetadata().error,
269 };
270 case InstanceEvent.WORKFLOW_TERMINATED:
271 return { ...common, type: "workflow_terminated" };
272 case InstanceEvent.STEP_START: {
273 const config = await readStepConfig(storage, log.groupKey);
274 return stepEvent((stepName) => ({
275 ...common,
276 type: "step_started",
277 stepName,
278 ...(config === undefined ? {} : { config }),
279 }));
280 }
281 case InstanceEvent.STEP_SUCCESS: {
282 const output = await readStepCompletedOutput(storage, log.groupKey);
283 return stepEvent((stepName) => ({
284 ...common,
285 type: "step_completed",
286 stepName,
287 ...(output === undefined ? {} : { output }),
288 }));
289 }
290 case InstanceEvent.STEP_FAILURE:
291 return stepEvent((stepName) => ({
292 ...common,
293 type: "step_errored",
294 stepName,
295 }));
296 case InstanceEvent.ATTEMPT_START:
297 return stepEvent((stepName) => ({
298 ...common,
299 type: "attempt_started",
300 stepName,
301 attempt: parseMetadata().attempt,
302 }));
303 case InstanceEvent.ATTEMPT_SUCCESS:
304 return stepEvent((stepName) => ({
305 ...common,
306 type: "attempt_completed",
307 stepName,
308 attempt: parseMetadata().attempt,
309 }));
310 case InstanceEvent.ATTEMPT_FAILURE: {
311 const metadata = parseMetadata();
312 return stepEvent((stepName) => ({
313 ...common,
314 type: "attempt_errored",
315 stepName,
316 attempt: metadata.attempt,
317 error: metadata.error,
318 ...(metadata.retryDelayMs === undefined ? {} : { retryDelayMs: metadata.retryDelayMs }),
319 }));
320 }
321 case InstanceEvent.SLEEP_START:
322 return stepEvent((stepName) => ({
323 ...common,
324 type: "sleep_started",
325 stepName,
326 durationMs: parseMetadata().durationMs,
327 }));
328 case InstanceEvent.SLEEP_COMPLETE:
329 return stepEvent((stepName) => ({
330 ...common,
331 type: "sleep_completed",
332 stepName,
333 }));
334 case InstanceEvent.WAIT_START:
335 return stepEvent((stepName) => ({
336 ...common,
337 type: "wait_started",
338 stepName,
339 eventType: parseMetadata().event,
340 }));
341 case InstanceEvent.WAIT_COMPLETE:
342 return stepEvent((stepName) => ({
343 ...common,
344 type: "wait_completed",
345 stepName,
346 }));
347 case InstanceEvent.WAIT_TIMED_OUT:
348 return stepEvent((stepName) => ({
349 ...common,
350 type: "wait_timed_out",
351 stepName,
352 }));
353 case InstanceEvent.ROLLBACK_START:
354 return { ...common, type: "rollback_started" };
355 case InstanceEvent.ROLLBACK_STEP_START: {
356 const config = await readStepConfig(storage, log.groupKey);
357 return stepEvent((stepName) => ({
358 ...common,
359 type: "rollback_step_started",
360 stepName,
361 ...(config === undefined ? {} : { config }),
362 }));
363 }
364 case InstanceEvent.ROLLBACK_STEP_SUCCESS:
365 return stepEvent((stepName) => ({
366 ...common,
367 type: "rollback_step_completed",
368 stepName,
369 }));
370 case InstanceEvent.ROLLBACK_STEP_FAILURE:
371 return stepEvent((stepName) => ({
372 ...common,
373 type: "rollback_step_errored",
374 stepName,
375 error: parseMetadata().error,
376 }));
377 case InstanceEvent.ROLLBACK_ATTEMPT_START:
378 return stepEvent((stepName) => ({
379 ...common,
380 type: "rollback_attempt_started",
381 stepName,
382 attempt: parseMetadata().attempt,
383 }));
384 case InstanceEvent.ROLLBACK_ATTEMPT_SUCCESS:
385 return stepEvent((stepName) => ({
386 ...common,
387 type: "rollback_attempt_completed",
388 stepName,
389 attempt: parseMetadata().attempt,
390 }));
391 case InstanceEvent.ROLLBACK_ATTEMPT_FAILURE: {
392 const metadata = parseMetadata();
393 return stepEvent((stepName) => ({
394 ...common,
395 type: "rollback_attempt_errored",
396 stepName,
397 attempt: metadata.attempt,
398 error: metadata.error,
399 ...(metadata.retryDelayMs === undefined ? {} : { retryDelayMs: metadata.retryDelayMs }),
400 }));
401 }
402 case InstanceEvent.ROLLBACK_COMPLETE:
403 return { ...common, type: "rollback_completed" };
404 case InstanceEvent.ROLLBACK_FAILED:
405 return { ...common, type: "rollback_errored" };
406 case InstanceEvent.__INTERNAL_PROD:
407 return { ...common, type: "internal" };
408 }
409 }
410
411 export class Engine extends DurableObject<Env, EngineProps> {
412 logs: Array<unknown> = [];
413
414 isRunning: boolean = false;
415 accountId: number | undefined;
416 instanceId: string | undefined;
417 workflowName: string | undefined;
418 timeoutHandler: GracePeriodSemaphore;
419 priorityQueue: TimePriorityQueue | undefined;
420 stepLimit: number;
421 engineAbortController: AbortController = new AbortController();
422 pauseController: AbortController = new AbortController();
423 rollbackPhase: RollbackPhase | undefined = undefined;
424 rollbackEligibleCacheKeys: Set<string> | undefined = undefined;
425
426 waiters: Map<
427 string,
428 Array<[cacheKey: string, resolve: (event: Event | PromiseLike<Event>) => void]>
429 > = new Map();
430 eventMap: Map<string, Array<Event>> = new Map();
431
432 // Not persisted: rollback fns are RPC stubs, dead across DO restarts.
433 rollbackRegistry: Map<string, RollbackRegistryEntry> = new Map();
434
435 subscribers = new Set<WorkflowSubscriptionState>();
436
437 constructor(state: DurableObjectState, env: Env) {
438 super(state, env);
439
440 const { workflowName } = this.ctx.props;
441 this.stepLimit =
442 (workflowName === undefined ? undefined : env.STEP_LIMITS?.[workflowName]) ??
443 (env.STEP_LIMIT ? JSON.parse(env.STEP_LIMIT) : DEFAULT_STEP_LIMIT);
444
445 void this.ctx.blockConcurrencyWhile(async () => {
446 this.ctx.storage.transactionSync(() => {
447 try {
448 this.ctx.storage.sql.exec(`
449 CREATE TABLE IF NOT EXISTS priority_queue (
450 id INTEGER PRIMARY KEY NOT NULL,
451 created_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
452 target_timestamp INTEGER NOT NULL,
453 action INTEGER NOT NULL, -- should only be 0 or 1 (1 for added, 0 for deleted),
454 entryType INTEGER NOT NULL,
455 hash TEXT NOT NULL,
456 CHECK (action IN (0, 1)), -- guararentee that action can only be 0 or 1
457 UNIQUE (action, entryType, hash)
458 );
459 CREATE TABLE IF NOT EXISTS states (
460 id INTEGER PRIMARY KEY NOT NULL,
461 timestamp TIMESTAMP DEFAULT (DATETIME('now','subsec')),
462 groupKey TEXT,
463 target TEXT,
464 metadata TEXT,
465 event INTEGER NOT NULL
466 );
467 CREATE TABLE IF NOT EXISTS streaming_step_chunks (
468 cache_key TEXT NOT NULL,
469 attempt INTEGER NOT NULL,
470 chunk_index INTEGER NOT NULL,
471 chunk BLOB NOT NULL,
472 PRIMARY KEY (cache_key, attempt, chunk_index)
473 ) WITHOUT ROWID
474 `);
475 } catch (e) {
476 console.error(e);
477 throw e;
478 }
479 });
480 });
481
482 this.timeoutHandler = new GracePeriodSemaphore(startGracePeriod, ENGINE_TIMEOUT);
483 }
484
485 writeLog(
486 event: InstanceEvent,
487 group: string | null,
488 target: string | null = null,
489 metadata: Record<string, unknown>,
490 ) {
491 this.ctx.storage.sql.exec(
492 "INSERT INTO states (event, groupKey, target, metadata) VALUES (?, ?, ?, ?)",
493 event,
494 group,
495 target,
496 JSON.stringify(metadata, binaryReplacer),
497 );
498
499 for (const subscriber of this.subscribers) {
500 const waiter = subscriber.waiter;
501 if (waiter !== undefined) {
502 subscriber.waiter = undefined;
503 waiter.resolve();
504 }
505 }
506
507 // Wake any waiters if this is a terminal step event
508 if (group) {
509 this.handleStepResultWaiter(group, event, metadata);
510 }
511 }
512
513 readEligibleRollbackStepsDesc(limit?: number): Array<{ cacheKey: string; target: string }> {
514 const rollbackTerminalGroups = new Set<string>();
515 const rollbackEligibleGroups = new Set<string>();
516 const stepStartsDesc: Array<{ groupKey: string; target: string | null }> = [];
517 const rows = [
518 ...this.ctx.storage.sql.exec<{
519 event: InstanceEvent;
520 groupKey: string;
521 target: string | null;
522 metadata: string;
523 }>(
524 "SELECT event, groupKey, target, metadata FROM states WHERE groupKey IS NOT NULL ORDER BY id DESC",
525 ),
526 ];
527
528 for (const row of rows) {
529 if (row.event === InstanceEvent.STEP_START) {
530 stepStartsDesc.push({ groupKey: row.groupKey, target: row.target });
531 }
532
533 if (
534 row.event === InstanceEvent.ROLLBACK_STEP_SUCCESS ||
535 row.event === InstanceEvent.ROLLBACK_STEP_FAILURE
536 ) {
537 rollbackTerminalGroups.add(
538 row.groupKey.startsWith(ROLLBACK_CACHE_KEY_PREFIX)
539 ? row.groupKey.slice(ROLLBACK_CACHE_KEY_PREFIX.length)
540 : row.groupKey,
541 );
542 continue;
543 }
544
545 if (
546 row.event !== InstanceEvent.STEP_START &&
547 row.event !== InstanceEvent.STEP_SUCCESS &&
548 row.event !== InstanceEvent.STEP_FAILURE
549 ) {
550 continue;
551 }
552
553 try {
554 if ((JSON.parse(row.metadata) as { hasRollback?: boolean }).hasRollback === true) {
555 rollbackEligibleGroups.add(row.groupKey);
556 }
557 } catch {
558 // Ignore malformed metadata in local persisted logs.
559 }
560 }
561
562 const eligible: Array<{ cacheKey: string; target: string }> = [];
563 for (const { groupKey, target } of stepStartsDesc) {
564 if (rollbackEligibleGroups.has(groupKey) && !rollbackTerminalGroups.has(groupKey)) {
565 eligible.push({ cacheKey: groupKey, target: target ?? groupKey });
566 if (limit !== undefined && eligible.length >= limit) {
567 break;
568 }
569 }
570 }
571
572 return eligible;
573 }
574
575 private getEligibleRollbackSteps(limit?: number): Array<string> {
576 return this.readEligibleRollbackStepsDesc(limit).map(({ cacheKey }) => cacheKey);
577 }
578
579 registerRollbackFn(registration: RollbackRegistration): void {
580 if (
581 this.rollbackPhase === "replay" &&
582 this.rollbackEligibleCacheKeys !== undefined &&
583 !this.rollbackEligibleCacheKeys.has(registration.cacheKey)
584 ) {
585 disposeRollbackStub(registration.fn);
586 return;
587 }
588
589 registerRollbackFn(this.rollbackRegistry, registration);
590 }
591
592 setRollbackPhase(phase: RollbackPhase | undefined): void {
593 this.rollbackPhase = phase;
594 }
595
596 // Lives here for access to the protected DurableObject `ctx`.
597 createRollbackContext(rollbackStep?: { cacheKey: string }): Context {
598 return new Context(this, this.ctx, rollbackStep);
599 }
600
601 readLogsFromStep(_cacheKey: string): Array<RawInstanceLog> {
602 return [];
603 }
604
605 async subscribe(options?: WorkflowSubscriptionOptions): Promise<WorkflowSubscriptionTarget> {
606 const { cursor, filter } = options ?? {};
607 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
608 if (metadata === undefined) {
609 throw createWorkflowError("Instance does not exist", "instance.not_found");
610 }
611
612 const state: WorkflowSubscriptionState = {
613 instanceId: metadata.instance.id,
614 params: metadata.event.payload,
615 lastEventId: cursor ?? -1,
616 filter: filter === undefined ? undefined : new Set(filter),
617 waiter: undefined,
618 closed: false,
619 };
620 const subscription = new WorkflowSubscriptionTarget(
621 () => this.nextWorkflowEvent(state),
622 () => {
623 state.closed = true;
624 this.subscribers.delete(state);
625 state.waiter?.resolve();
626 state.waiter = undefined;
627 },
628 );
629 this.subscribers.add(state);
630 return subscription;
631 }
632
633 private async nextWorkflowEvent(
634 state: WorkflowSubscriptionState,
635 ): Promise<IteratorResult<WorkflowSubscriptionEvent, undefined>> {
636 while (!state.closed) {
637 const row = this.ctx.storage.sql
638 .exec<RawInstanceLog>(
639 "SELECT id, timestamp, event, groupKey, target, metadata FROM states WHERE id > ? ORDER BY id ASC LIMIT 1",
640 state.lastEventId,
641 )
642 .toArray()[0];
643
644 if (row === undefined) {
645 const hasTerminalEvent =
646 this.ctx.storage.sql
647 .exec<{ id: number }>(
648 "SELECT id FROM states WHERE id <= ? AND event IN (?, ?, ?) LIMIT 1",
649 state.lastEventId,
650 InstanceEvent.WORKFLOW_SUCCESS,
651 InstanceEvent.WORKFLOW_FAILURE,
652 InstanceEvent.WORKFLOW_TERMINATED,
653 )
654 .toArray()[0] !== undefined;
655 if (hasTerminalEvent) {
656 state.closed = true;
657 return { done: true, value: undefined };
658 }
659
660 await new Promise<void>((resolve) => {
661 state.waiter = { resolve };
662 });
663 continue;
664 }
665
666 state.lastEventId = row.id;
667 const event = await buildWorkflowSubscriptionEvent(
668 this.ctx.storage,
669 row,
670 state.instanceId,
671 state.params,
672 );
673 if (event.type === "internal") {
674 continue;
675 }
676 if (state.filter === undefined || state.filter.has(event.type)) {
677 return { done: false, value: event };
678 }
679 if (isTerminalEvent(event)) {
680 return { done: true, value: undefined };
681 }
682 }
683
684 return { done: true, value: undefined };
685 }
686
687 readLogs(): EngineLogs {
688 const logs = [
689 ...this.ctx.storage.sql.exec<{
690 event: InstanceEvent;
691 groupKey: string | null;
692 target: string | null;
693 metadata: string;
694 }>("SELECT event, groupKey, target, metadata FROM states"),
695 ];
696
697 return {
698 logs: logs.map((log) => {
699 const metadata = JSON.parse(log.metadata);
700
701 if (!isStepSuccessEvent(log.event) || !metadata.streamOutput) {
702 return { ...log, metadata, group: log.groupKey };
703 }
704
705 const { cacheKey, meta } = metadata.streamOutput as {
706 cacheKey: string;
707 meta: StreamOutputMeta;
708 };
709 try {
710 const preview = getStoredStreamOutputPreview({
711 storage: this.ctx.storage,
712 cacheKey,
713 meta,
714 maxChars: 1024,
715 });
716 metadata.result =
717 preview.type === "text"
718 ? preview.output
719 : `[ReadableStream (binary): ${meta.totalBytes} bytes]`;
720 } catch {
721 metadata.result = `[ReadableStream: ${meta.totalBytes} bytes]`;
722 }
723 delete metadata.streamOutput;
724
725 return { ...log, metadata, group: log.groupKey };
726 }),
727 };
728 }
729
730 /**
731 * Returns detailed logs including timestamps, ordered by ID.
732 * Used by the local explorer to reconstruct step-level detail.
733 */
734 readDetailedLogs(): Array<{
735 id: number;
736 timestamp: string;
737 event: number;
738 group: string | null;
739 target: string | null;
740 metadata: Record<string, unknown>;
741 }> {
742 const rows = [
743 ...this.ctx.storage.sql.exec<{
744 id: number;
745 timestamp: string;
746 event: number;
747 groupKey: string | null;
748 target: string | null;
749 metadata: string;
750 }>("SELECT id, timestamp, event, groupKey, target, metadata FROM states ORDER BY id ASC"),
751 ];
752
753 return rows.map((row) => {
754 const metadata = JSON.parse(row.metadata) as Record<string, unknown>;
755
756 if (!isStepSuccessEvent(row.event) || !metadata.streamOutput) {
757 return {
758 id: row.id,
759 timestamp: String(row.timestamp).replace(" ", "T") + "Z",
760 event: row.event,
761 group: row.groupKey,
762 target: row.target,
763 metadata,
764 };
765 }
766
767 const { cacheKey, meta } = metadata.streamOutput as {
768 cacheKey: string;
769 meta: StreamOutputMeta;
770 };
771 try {
772 const preview = getStoredStreamOutputPreview({
773 storage: this.ctx.storage,
774 cacheKey,
775 meta,
776 maxChars: 1024,
777 });
778 metadata.result =
779 preview.type === "text"
780 ? preview.output
781 : `[ReadableStream (binary): ${meta.totalBytes} bytes]`;
782 } catch {
783 metadata.result = `[ReadableStream: ${meta.totalBytes} bytes]`;
784 }
785 delete metadata.streamOutput;
786
787 return {
788 id: row.id,
789 timestamp: String(row.timestamp).replace(" ", "T") + "Z",
790 event: row.event,
791 group: row.groupKey,
792 target: row.target,
793 metadata,
794 };
795 });
796 }
797
798 readLogsFromEvent(eventType: InstanceEvent): EngineLogs {
799 const logs = [
800 ...this.ctx.storage.sql.exec<{
801 event: InstanceEvent;
802 groupKey: string | null;
803 target: string | null;
804 metadata: string;
805 }>("SELECT event, groupKey, target, metadata FROM states WHERE event = ?", eventType),
806 ];
807
808 return {
809 logs: logs.map((log) => ({
810 ...log,
811 metadata: JSON.parse(log.metadata),
812 group: log.groupKey,
813 })),
814 };
815 }
816
817 async getStatus(): Promise<InstanceStatus> {
818 if (this.accountId === undefined) {
819 // Engine could have restarted, so we try to restore from its state
820 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
821 if (metadata === undefined) {
822 // metadata was never set, so we assume the engine was never started
823 throw new Error("Engine was never started");
824 }
825
826 this.accountId = metadata.accountId;
827 this.instanceId = metadata.instance.id;
828 this.workflowName = metadata.workflow.name;
829 }
830
831 const res = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
832
833 // NOTE(lduarte): if status don't exist, means that engine is running for the first time, so we assume queued
834 if (res === undefined) {
835 return InstanceStatus.Queued;
836 }
837 return res;
838 }
839
840 // Returns instance metadata for the local explorer.
841 async getInstanceMetadata(): Promise<{
842 instanceId: string;
843 status: InstanceStatus;
844 createdOn: string;
845 }> {
846 const status = await this.getStatus();
847 // Read the full metadata to get the created_on timestamp
848 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
849 let createdOn = metadata?.instance?.created_on ?? "";
850
851 // Backfill for instances created before created_on was populated:
852 // try the earliest log entry, then the earliest priority queue entry.
853 if (!createdOn) {
854 const queries = [
855 "SELECT timestamp AS ts FROM states ORDER BY id ASC LIMIT 1",
856 "SELECT created_on AS ts FROM priority_queue ORDER BY id ASC LIMIT 1",
857 ];
858 for (const query of queries) {
859 if (createdOn) {
860 break;
861 }
862 try {
863 for (const row of this.ctx.storage.sql.exec(query)) {
864 if (typeof row.ts === "string" && row.ts) {
865 // SQLite DATETIME('now','subsec') returns "YYYY-MM-DD HH:MM:SS.sss"
866 // which is not valid ISO 8601 (needs T separator and Z timezone).
867 const raw = row.ts.replace(" ", "T") + "Z";
868 createdOn = raw;
869 }
870 }
871 } catch {
872 // Table may not exist
873 }
874 }
875 }
876
877 return {
878 instanceId: this.instanceId ?? "",
879 status,
880 createdOn,
881 };
882 }
883
884 async setStatus(accountId: number, instanceId: string, status: InstanceStatus): Promise<void> {
885 const previousStatus = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
886 await this.ctx.storage.put(ENGINE_STATUS_KEY, status);
887
888 if (previousStatus !== status) {
889 switch (status) {
890 case InstanceStatus.Queued: {
891 // Creation writes WORKFLOW_QUEUED before the first status is stored.
892 const hasQueuedEvent =
893 this.ctx.storage.sql
894 .exec<{ id: number }>(
895 "SELECT id FROM states WHERE event = ? LIMIT 1",
896 InstanceEvent.WORKFLOW_QUEUED,
897 )
898 .toArray()[0] !== undefined;
899 if (!(previousStatus === undefined && hasQueuedEvent)) {
900 this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {});
901 }
902 break;
903 }
904 case InstanceStatus.Running:
905 this.writeLog(InstanceEvent.WORKFLOW_RUNNING, null, null, {});
906 break;
907 case InstanceStatus.Paused:
908 this.writeLog(InstanceEvent.WORKFLOW_PAUSED, null, null, {});
909 break;
910 case InstanceStatus.WaitingForPause:
911 this.writeLog(InstanceEvent.WORKFLOW_WAITING_FOR_PAUSE, null, null, {});
912 break;
913 case InstanceStatus.Waiting:
914 this.writeLog(InstanceEvent.WORKFLOW_WAITING, null, null, {});
915 break;
916 }
917 }
918
919 // check if anyone is waiting for this status
920 this.handleStatusWaiter(status);
921 }
922
923 private statusWaiters: Map<
924 InstanceStatus,
925 { resolve: () => void; reject: (e: unknown) => void }
926 > = new Map();
927 async waitForStatus(status: string): Promise<void> {
928 const targetStatus = toInstanceStatus(status);
929 const currentStatus = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
930
931 // if the workflow has already reached the desired state, resolve immediately
932 if (currentStatus === targetStatus) {
933 return;
934 }
935
936 // if it hasn't reached the desired state, create a new promise and add its resolver to the waiters map
937 return new Promise((resolve, reject) => {
938 this.statusWaiters.set(targetStatus, { resolve, reject });
939 // immediately reconcile against current status in case it's already finite
940 this.handleStatusWaiter(currentStatus as InstanceStatus);
941 });
942 }
943
944 handleStatusWaiter(status: InstanceStatus): void {
945 const waiter = this.statusWaiters.get(status);
946
947 // resolve if it reached the desired status
948 if (waiter) {
949 waiter.resolve();
950 this.statusWaiters.delete(status);
951 return;
952 }
953
954 switch (status) {
955 case InstanceStatus.Errored: {
956 // if it reaches final status "errored", then it can't be waiting for it to complete or terminate
957 const unreachableStatuses = [InstanceStatus.Complete, InstanceStatus.Terminated];
958
959 this.rejectUnreachableStatus(status, unreachableStatuses);
960 break;
961 }
962 case InstanceStatus.Terminated: {
963 // if it reaches final status "terminated", then it can't be waiting for it to complete or error
964 const unreachableStatuses = [InstanceStatus.Complete, InstanceStatus.Errored];
965
966 this.rejectUnreachableStatus(status, unreachableStatuses);
967 break;
968 }
969 case InstanceStatus.Complete: {
970 // if it reaches final status "complete", then it can't be waiting for it to terminate or error
971 const unreachableStatuses = [InstanceStatus.Terminated, InstanceStatus.Errored];
972
973 this.rejectUnreachableStatus(status, unreachableStatuses);
974 break;
975 }
976 default:
977 break;
978 }
979 }
980
981 rejectUnreachableStatus(reachedStatus: number, unreachableStatuses: Array<number>): void {
982 if (unreachableStatuses) {
983 for (const unreachableStatus of unreachableStatuses) {
984 const waiter = this.statusWaiters.get(unreachableStatus);
985 if (waiter) {
986 waiter.reject(
987 new Error(
988 `[WorkflowIntrospector] The Workflow instance ${
989 this.instanceId
990 } has reached status '${instanceStatusName(
991 reachedStatus,
992 )}'. This is a finite status that prevents it from ever reaching the expected status of '${instanceStatusName(
993 unreachableStatus,
994 )}'.`,
995 ),
996 );
997 this.statusWaiters.delete(unreachableStatus);
998 return;
999 }
1000 }
1001 }
1002 }
1003
1004 /**
1005 * Create a replay ReadableStream from stored stream output metadata.
1006 * Returns undefined if the stream data is not in a valid/complete state.
1007 */
1008 private replayStreamFromMeta(streamOutput: {
1009 cacheKey: string;
1010 meta: StreamOutputMeta;
1011 }): ReadableStream<Uint8Array> | undefined {
1012 if (streamOutput.meta.state !== StreamOutputState.Complete) {
1013 return undefined;
1014 }
1015 const integrityError = getInvalidStoredStreamOutputError(
1016 this.ctx.storage,
1017 streamOutput.cacheKey,
1018 streamOutput.meta,
1019 );
1020 if (integrityError !== undefined) {
1021 return undefined;
1022 }
1023 return createReplayReadableStream({
1024 storage: this.ctx.storage,
1025 cacheKey: streamOutput.cacheKey,
1026 meta: streamOutput.meta,
1027 });
1028 }
1029
1030 private stepResultWaiters: Map<
1031 string,
1032 { resolve: (v: unknown) => void; reject: (e: unknown) => void }
1033 > = new Map();
1034 async waitForStepResult(stepName: string, stepCount?: number): Promise<unknown> {
1035 const hash = await computeHash(stepName);
1036 const count = stepCount ?? 1;
1037 const cacheKey = `${hash}-${count}`;
1038
1039 // read latest log from step
1040 const rows = [
1041 ...this.ctx.storage.sql.exec<{
1042 event: InstanceEvent;
1043 metadata: string;
1044 }>(
1045 "SELECT event, metadata FROM states WHERE groupKey = ? ORDER BY id DESC LIMIT 1",
1046 cacheKey,
1047 ),
1048 ];
1049
1050 if (rows.length > 0) {
1051 const { event, metadata } = rows[0];
1052 const parsed = JSON.parse(metadata);
1053 if (event === InstanceEvent.STEP_SUCCESS) {
1054 if (parsed?.streamOutput) {
1055 return this.replayStreamFromMeta(parsed.streamOutput);
1056 }
1057 return parsed?.result;
1058 }
1059 if (event === InstanceEvent.STEP_FAILURE) {
1060 throw parsed?.error ?? parsed;
1061 }
1062 }
1063
1064 // if it hasn't completed the step, create a new promise to later resolve/reject
1065 return new Promise<unknown>((resolve, reject) => {
1066 this.stepResultWaiters.set(cacheKey, { resolve, reject });
1067 });
1068 }
1069
1070 handleStepResultWaiter(group: string, event: InstanceEvent, metadata: Record<string, unknown>) {
1071 const waiter = this.stepResultWaiters.get(group);
1072 if (!waiter) {
1073 return;
1074 }
1075 if (event === InstanceEvent.STEP_SUCCESS) {
1076 if (metadata?.streamOutput) {
1077 waiter.resolve(
1078 this.replayStreamFromMeta(
1079 metadata.streamOutput as {
1080 cacheKey: string;
1081 meta: StreamOutputMeta;
1082 },
1083 ),
1084 );
1085 } else {
1086 waiter.resolve(metadata?.result);
1087 }
1088 this.stepResultWaiters.delete(group);
1089 } else if (event === InstanceEvent.STEP_FAILURE) {
1090 const error = metadata?.error ?? new Error("Step failed");
1091 waiter.reject(error);
1092 this.stepResultWaiters.delete(group);
1093 }
1094 }
1095
1096 async getOutputOrError(isOutput: boolean): Promise<unknown> {
1097 const status = await this.getStatus();
1098
1099 if (isOutput) {
1100 if (status !== InstanceStatus.Complete) {
1101 throw new Error(
1102 `Cannot retrieve output: Workflow instance is in status "${instanceStatusName(
1103 status,
1104 )}" but must be "complete" to have an output available`,
1105 );
1106 }
1107 const logs = this.readLogsFromEvent(InstanceEvent.WORKFLOW_SUCCESS).logs;
1108 return logs.at(0)?.metadata.result;
1109 } else {
1110 if (status !== InstanceStatus.Errored) {
1111 throw new Error(
1112 `Cannot retrieve error: Workflow instance is in status "${instanceStatusName(
1113 status,
1114 )}" but must be "errored" to have error information available`,
1115 );
1116 }
1117 const logs = this.readLogsFromEvent(InstanceEvent.WORKFLOW_FAILURE).logs;
1118 const log = logs.at(0);
1119 if (!log?.metadata.error) {
1120 throw new Error("Cannot retrieve error: No workflow instance failure log found");
1121 }
1122 return log.metadata.error;
1123 }
1124 }
1125
1126 async abort(reason: string) {
1127 await this.ctx.storage.sync();
1128
1129 // Clean up pending JS operations
1130 this.timeoutHandler.dispose();
1131 this.engineAbortController.abort(new Error(reason));
1132
1133 this.ctx.abort(reason);
1134 }
1135
1136 // Called by the dispose function when introspecting the instance in tests
1137 // TODO: Ideally this abort should be done by `abortAllDurableObjects` from worked called by vitest-plugin
1138 async unsafeAbort(reason?: string) {
1139 await this.ctx.storage.sync();
1140 await this.ctx.storage.deleteAll();
1141
1142 this.ctx.abort(reason);
1143 }
1144
1145 async storeEventMap() {
1146 await this.ctx.blockConcurrencyWhile(async () => {
1147 const entries: Record<string, Event> = {};
1148 for (const [type, events] of this.eventMap.entries()) {
1149 for (let i = 0; i < events.length; i++) {
1150 entries[`${EVENT_MAP_PREFIX}\n${type}\n${i}`] = events[i];
1151 }
1152 }
1153 if (Object.keys(entries).length > 0) {
1154 await this.ctx.storage.put(entries);
1155 }
1156 });
1157 }
1158
1159 async restoreEventMap() {
1160 await this.ctx.blockConcurrencyWhile(async () => {
1161 // FIXME(lduarte): can this OoM the DO in the production?
1162 const entries = await this.ctx.storage.list<Event>({
1163 prefix: EVENT_MAP_PREFIX,
1164 });
1165 for (const [key, value] of entries) {
1166 const [_, eventType, _idx] = key.split("\n");
1167 // NOTE(lduarte): safe to do because list returns keys in ascending order, so
1168 // indexes will be correctly ordered
1169 const eventList = this.eventMap.get(eventType) ?? [];
1170 eventList.push(value);
1171 this.eventMap.set(eventType, eventList);
1172 }
1173 });
1174 }
1175
1176 async receiveEvent(event: { timestamp: Date; payload: unknown; type: string }) {
1177 // There are four possible cases here:
1178 // - There is a callback waiting, send it
1179 // - There is no callback waiting but engine is alive, store it
1180 // - Engine is not awake and is in Waiting status, store it and start it up
1181 // - Engine is not awake and is in Paused (or another terminal) status, store it
1182 // - Engine is not awake and is Errored or Terminated, this should not get called
1183 let eventTypeQueue = this.eventMap.get(event.type) ?? [];
1184 eventTypeQueue.push(event as Event);
1185
1186 this.eventMap.set(event.type, eventTypeQueue);
1187 await this.storeEventMap();
1188
1189 // if the engine is running
1190 if (this.isRunning) {
1191 // Attempt to get the callback and run it
1192 const callbacks = this.waiters.get(event.type);
1193 if (callbacks) {
1194 const entry = callbacks[0];
1195 if (entry) {
1196 const [, resolve] = entry;
1197 resolve(event);
1198 // Remove it from the list of callbacks
1199 callbacks.shift();
1200 this.waiters.set(event.type, callbacks);
1201
1202 eventTypeQueue = this.eventMap.get(event.type) ?? [];
1203 eventTypeQueue.shift();
1204 this.eventMap.set(event.type, eventTypeQueue);
1205
1206 return;
1207 }
1208 }
1209 } else {
1210 const mockEvent = await this.ctx.storage.get(`${MODIFIER_KEYS.MOCK_EVENT}${event.type}`);
1211 if (mockEvent) {
1212 return;
1213 }
1214
1215 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1216
1217 if (metadata === undefined) {
1218 throw new Error("Engine was never started");
1219 }
1220
1221 void this.init(
1222 metadata.accountId,
1223 metadata.workflow,
1224 metadata.version,
1225 metadata.instance,
1226 metadata.event,
1227 );
1228 }
1229 }
1230
1231 getInstanceModifier(): WorkflowInstanceModifier {
1232 return new WorkflowInstanceModifier(this, this.ctx);
1233 }
1234
1235 async changeInstanceStatus(
1236 newStatus: "resume" | "pause" | "terminate" | "restart",
1237 from?: RestartFromStep,
1238 terminateOptions?: WorkflowInstanceTerminateOptions,
1239 ): Promise<void> {
1240 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1241
1242 if (metadata === undefined) {
1243 throw createWorkflowError("Instance does not exist", "instance.not_found");
1244 }
1245
1246 switch (newStatus) {
1247 case "pause":
1248 await this.userTriggeredPause();
1249 break;
1250 case "resume": {
1251 const currentStatus = await this.getStatus();
1252 if (currentStatus === InstanceStatus.WaitingForPause) {
1253 // Engine is still running — cancel the pending pause
1254 this.timeoutHandler.cancelWaitingPromisesByType("pause");
1255 await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Running);
1256 } else if (currentStatus === InstanceStatus.Paused) {
1257 await this.attemptResume();
1258 }
1259 break;
1260 }
1261 case "terminate": {
1262 const currentStatus = await this.getStatus();
1263 if (
1264 [InstanceStatus.Terminated, InstanceStatus.Complete, InstanceStatus.Errored].includes(
1265 currentStatus,
1266 )
1267 ) {
1268 throw createWorkflowError(
1269 "Cannot terminate instance since its on a finite state",
1270 "instance.cannot_terminate",
1271 );
1272 }
1273 await this.userTriggeredTerminate(terminateOptions);
1274 break;
1275 }
1276 case "restart":
1277 if (from) {
1278 if (!resolveGroupKeysToWipe(this.ctx.storage.sql, from)) {
1279 throw stepNotFoundError(from.name);
1280 }
1281 await storeRestartFromStep(this.ctx.storage, from);
1282 }
1283 await this.userTriggeredRestart();
1284 break;
1285 }
1286 }
1287
1288 private async replayRollbackRegistry(metadata: InstanceMetadata): Promise<void> {
1289 if (this.rollbackRegistry.size > 0) {
1290 return;
1291 }
1292
1293 const eligible = this.getEligibleRollbackSteps();
1294 if (eligible.length === 0) {
1295 return;
1296 }
1297
1298 this.rollbackEligibleCacheKeys = new Set(eligible);
1299 const stubStep = this.createRollbackContext();
1300 this.setRollbackPhase("replay");
1301 try {
1302 await (this.ctx.props.workflowClass ?? this.env.USER_WORKFLOW).run(
1303 metadata.event,
1304 stubStep as unknown as WorkflowStep,
1305 );
1306 } catch (replayErr) {
1307 // Match the production engine: replay may stop on normal workflow control
1308 // flow; rollback execution uses whatever handlers replay registered.
1309 console.debug("Rollback replay stopped:", replayErr);
1310 } finally {
1311 this.setRollbackPhase(undefined);
1312 this.rollbackEligibleCacheKeys = undefined;
1313 }
1314 }
1315
1316 async userTriggeredTerminate(options?: WorkflowInstanceTerminateOptions) {
1317 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1318
1319 if (metadata === undefined) {
1320 throw createWorkflowError("Instance does not exist", "instance.not_found");
1321 }
1322
1323 if (options?.rollback === true) {
1324 this.priorityQueue ??= new TimePriorityQueue(this.ctx, metadata);
1325 await this.replayRollbackRegistry(metadata);
1326
1327 const error = new Error("Instance terminated during rollback");
1328 error.name = "Terminated";
1329 this.setRollbackPhase("rollback");
1330 try {
1331 await executeRollbacks(this, error);
1332 } catch (rollbackErr) {
1333 console.error("Rollback execution failed:", rollbackErr);
1334 } finally {
1335 this.setRollbackPhase(undefined);
1336 }
1337 }
1338
1339 this.writeLog(InstanceEvent.WORKFLOW_TERMINATED, null, null, {
1340 trigger: {
1341 source: InstanceTrigger.API,
1342 },
1343 });
1344
1345 await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Terminated);
1346
1347 await this.abort(ABORT_REASONS.USER_TERMINATE);
1348 }
1349
1350 /** Deletes all instance state and aborts its current execution. */
1351 async deleteInstance(): Promise<void> {
1352 if ((await this.ctx.storage.get(INSTANCE_METADATA)) === undefined) {
1353 throw createWorkflowError("Instance does not exist", "instance.not_found");
1354 }
1355
1356 await this.ctx.storage.deleteAll();
1357 await this.abort(ABORT_REASONS.USER_DELETE);
1358 }
1359
1360 async userTriggeredPause() {
1361 const status = await this.getStatus();
1362
1363 if (status === InstanceStatus.Paused || status === InstanceStatus.WaitingForPause) {
1364 return;
1365 }
1366
1367 if (status !== InstanceStatus.Running && status !== InstanceStatus.Waiting) {
1368 return;
1369 }
1370
1371 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1372
1373 if (metadata === undefined) {
1374 throw createWorkflowError("Instance does not exist", "instance.not_found");
1375 }
1376
1377 await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.WaitingForPause);
1378
1379 void this.timeoutHandler
1380 .waitUntilNothingIsRunning("pause", async () => {
1381 await this.ctx.storage.put(PAUSE_DATETIME, new Date());
1382 await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Paused);
1383 // Signal the pause controller to interrupt any active
1384 // scheduler.wait (sleep/waitForEvent). The workflow will
1385 // throw a pause error at the next step boundary via
1386 // #checkForPendingPause()
1387 this.pauseController.abort(ABORT_REASONS.USER_PAUSE);
1388 })
1389 .catch(() => {});
1390 }
1391
1392 async userTriggeredRestart() {
1393 // cleanup is done in attemptRestart() on the fresh DO instance
1394
1395 await this.abort(ABORT_REASONS.USER_RESTART);
1396 }
1397
1398 async attemptRestart() {
1399 const restartFromStep = await readAndClearRestartFromStep(this.ctx.storage);
1400
1401 let groupKeysToWipe: Set<string> | null = null;
1402 if (restartFromStep) {
1403 groupKeysToWipe = resolveGroupKeysToWipe(this.ctx.storage.sql, restartFromStep);
1404 if (!groupKeysToWipe) {
1405 throw stepNotFoundError(restartFromStep.name);
1406 }
1407 }
1408
1409 await wipeRestartState(this.ctx.storage, ENGINE_STATUS_KEY, PAUSE_DATETIME, groupKeysToWipe);
1410
1411 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1412
1413 if (metadata === undefined) {
1414 throw createWorkflowError("Instance does not exist", "instance.not_found");
1415 }
1416
1417 const { accountId, workflow, version, instance, event } = metadata;
1418
1419 if (!groupKeysToWipe) {
1420 this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {
1421 params: event.payload,
1422 versionId: version.id,
1423 trigger: {
1424 source: InstanceTrigger.API,
1425 },
1426 });
1427 this.writeLog(InstanceEvent.WORKFLOW_START, null, null, {});
1428 }
1429
1430 void this.init(accountId, workflow, version, instance, event);
1431 }
1432
1433 async attemptResume() {
1434 const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1435
1436 if (metadata === undefined) {
1437 throw createWorkflowError("Instance does not exist", "instance.not_found");
1438 }
1439
1440 const status = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
1441 if (status !== InstanceStatus.Paused) {
1442 return;
1443 }
1444
1445 // Offset priority queue timers by the pause duration so that
1446 // sleeps/retries resume from where they left off
1447 const pausedDate = await this.ctx.storage.get<Date>(PAUSE_DATETIME);
1448 if (pausedDate !== undefined) {
1449 const offset = Date.now() - new Date(pausedDate).valueOf();
1450 if (this.priorityQueue) {
1451 this.priorityQueue.offsetAll(offset);
1452 } else {
1453 const pq = new TimePriorityQueue(this.ctx, metadata);
1454 pq.offsetAll(offset);
1455 }
1456 }
1457 await this.ctx.storage.delete(PAUSE_DATETIME);
1458
1459 const { accountId, workflow, version, instance, event } = metadata;
1460
1461 await this.ctx.storage.put(ENGINE_STATUS_KEY, InstanceStatus.Queued);
1462
1463 // Reset pause state for the new run. The DO stays alive across
1464 // pause/resume (no this.ctx.abort()), so we need to clear stale
1465 // in-memory state from the previous run to avoid duplicates.
1466 this.pauseController = new AbortController();
1467 this.waiters = new Map();
1468 this.eventMap = new Map();
1469 clearRollbackRegistry(this.rollbackRegistry);
1470
1471 void this.init(accountId, workflow, version, instance, event);
1472 }
1473
1474 async hasInstance(): Promise<boolean> {
1475 // INSTANCE_METADATA is written exactly once, by the first init() for this
1476 // id, so its presence is the durable marker that the instance exists.
1477 return (await this.ctx.storage.get(INSTANCE_METADATA)) !== undefined;
1478 }
1479
1480 async init(
1481 accountId: number,
1482 workflow: DatabaseWorkflow,
1483 version: DatabaseVersion,
1484 instance: DatabaseInstance,
1485 event: WorkflowEvent<unknown>,
1486 ) {
1487 if (this.priorityQueue === undefined) {
1488 this.priorityQueue = new TimePriorityQueue(
1489 this.ctx,
1490 // this.env,
1491 {
1492 accountId,
1493 workflow,
1494 version,
1495 instance,
1496 event,
1497 },
1498 );
1499 }
1500
1501 if (this.isRunning) {
1502 return;
1503 }
1504
1505 this.priorityQueue.popPastEntries();
1506 await this.priorityQueue.handleNextAlarm();
1507
1508 // We are not running and are possibly starting a new lifetime
1509 this.accountId = accountId;
1510 this.instanceId = instance.id;
1511 this.workflowName = workflow.name;
1512
1513 const status = await this.getStatus();
1514 if (
1515 [
1516 InstanceStatus.Errored, // TODO (WOR-85): Remove this once upgrade story is done
1517 InstanceStatus.Terminated,
1518 InstanceStatus.Complete,
1519 InstanceStatus.Paused,
1520 ].includes(status)
1521 ) {
1522 return;
1523 }
1524
1525 // If the DO restarted (e.g. from alarm) while in WaitingForPause state,
1526 // transition to Paused and return early — same as production behaviour.
1527 if (status === InstanceStatus.WaitingForPause) {
1528 await this.ctx.storage.put(PAUSE_DATETIME, new Date());
1529 await this.setStatus(accountId, instance.id, InstanceStatus.Paused);
1530 return;
1531 }
1532
1533 if ((await this.ctx.storage.get(INSTANCE_METADATA)) == undefined) {
1534 const instanceMetadata: InstanceMetadata = {
1535 accountId,
1536 workflow,
1537 version,
1538 instance,
1539 event,
1540 };
1541 await this.ctx.storage.put(INSTANCE_METADATA, instanceMetadata);
1542
1543 // TODO (WOR-78): We currently don't have a queue mechanism
1544 this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {
1545 params: event.payload,
1546 versionId: version.id,
1547 trigger: {
1548 source: InstanceTrigger.API,
1549 },
1550 });
1551 this.writeLog(InstanceEvent.WORKFLOW_START, null, null, {});
1552 }
1553
1554 // restore eventMap so that waitForEvent across lifetimes works correctly
1555 await this.restoreEventMap();
1556
1557 const stubStep = new Context(this, this.ctx);
1558
1559 const workflowRunningHandler = async () => {
1560 await this.ctx.storage.transaction(async () => {
1561 // manually start the grace period
1562 // startGracePeriod(this, this.timeoutHandler.timeoutMs);
1563 await this.setStatus(accountId, instance.id, InstanceStatus.Running);
1564 });
1565 };
1566 this.isRunning = true;
1567
1568 void workflowRunningHandler();
1569 try {
1570 const target = this.ctx.props.workflowClass ?? this.env.USER_WORKFLOW;
1571 const result = await target.run(event, stubStep as unknown as WorkflowStep);
1572 await this.ctx.storage.put(WORKFLOW_OUTPUT_KEY, {
1573 value: normalizeForStorage(result),
1574 });
1575 this.writeLog(InstanceEvent.WORKFLOW_SUCCESS, null, null, {
1576 result,
1577 });
1578 // NOTE(lduarte): we want to run this in a transaction to guarentee ordering with running setstatus call
1579 // in case that it returns immediately
1580 await this.ctx.storage.transaction(async () => {
1581 await this.setStatus(accountId, instance.id, InstanceStatus.Complete);
1582 });
1583 // Dispose dup'd stubs; otherwise they leak across DO lifetimes.
1584 clearRollbackRegistry(this.rollbackRegistry);
1585 this.isRunning = false;
1586 } catch (err) {
1587 if (isAbortError(err)) {
1588 this.isRunning = false;
1589 return;
1590 }
1591
1592 // Run before the terminal status so events land before WORKFLOW_FAILURE.
1593 try {
1594 await executeRollbacks(this, err instanceof Error ? err : new Error(String(err)));
1595 } catch (rollbackErr) {
1596 console.error("Rollback execution failed:", rollbackErr);
1597 }
1598
1599 let error;
1600 if (err instanceof Error) {
1601 if (err.name === "NonRetryableError" || err.message.startsWith("NonRetryableError")) {
1602 const fatalError = shouldPreserveNonRetryableError()
1603 ? new PreservedNonRetryableError(err)
1604 : new WorkflowFatalError(
1605 `The execution of the Workflow instance was terminated, as a step threw an NonRetryableError and it was not handled`,
1606 );
1607
1608 this.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
1609 error: fatalError,
1610 });
1611
1612 await this.setStatus(accountId, instance.id, InstanceStatus.Errored);
1613 await this.abort(ABORT_REASONS.NON_RETRYABLE_ERROR);
1614 this.isRunning = false;
1615 return;
1616 }
1617 error = {
1618 message: err.message,
1619 name: err.name,
1620 };
1621 } else {
1622 error = {
1623 name: "Error",
1624 message: err,
1625 };
1626 }
1627
1628 this.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
1629 error,
1630 });
1631 // NOTE(lduarte): we want to run this in a transaction to guarentee ordering with running setstatus call
1632 // in case that it throws immediately
1633 await this.ctx.storage.transaction(async () => {
1634 await this.setStatus(accountId, instance.id, InstanceStatus.Errored);
1635 });
1636 this.isRunning = false;
1637 }
1638
1639 return {
1640 id: instance.id,
1641 };
1642 }
1643 }