@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 1,390 lines 51.2 KB
Raw
1 // Alchemy modifications are licensed under Apache-2.0.
2 // This file includes third-party code; see /THIRD_PARTY_LICENSES.md.
3 // Alchemy modifications: uses Array<T> syntax for non-tuple array types to match the repository convention.
4 import { RpcTarget } from "cloudflare:workers";
5 import type {
6 WorkflowBackoff,
7 WorkflowDelayDuration,
8 WorkflowDelayFunction,
9 WorkflowSleepDuration,
10 WorkflowStepConfig,
11 WorkflowStepEvent,
12 WorkflowStepSensitivity,
13 WorkflowTimeoutDuration,
14 } from "cloudflare:workers";
15 import { ms } from "itty-time";
16 import type { Engine } from "./engine.ts";
17 import { INSTANCE_METADATA, InstanceEvent, InstanceStatus } from "./instance.ts";
18 import type { InstanceMetadata } from "./instance.ts";
19 import { computeHash } from "./lib/cache.ts";
20 import {
21 DEFAULT_RETRY_DELAY_MS,
22 DELAY_FUNCTION_TIMEOUT_MS,
23 invokeDelayFunction,
24 raceAgainstAbort,
25 schedulerWait,
26 } from "./lib/delay.ts";
27 import {
28 ABORT_REASONS,
29 InvalidStepReadableStreamError,
30 NonRetryableDelayError,
31 OversizedStreamChunkError,
32 PreservedNonRetryableError,
33 shouldPreserveNonRetryableError,
34 StreamOutputStorageLimitError,
35 UnsupportedStreamChunkError,
36 WorkflowFatalError,
37 WorkflowInternalError,
38 WorkflowTimeoutError,
39 } from "./lib/errors.ts";
40 import { calcRetryDuration, DelayFunctionError } from "./lib/retries.ts";
41 import { parseRollbackOptions, ROLLBACK_CACHE_KEY_PREFIX } from "./lib/rollback.ts";
42 import type { RollbackFn, WorkflowStepRollbackOptions } from "./lib/rollback.ts";
43 import { normalizeForStorage } from "./lib/serialization.ts";
44 import {
45 cleanupPendingStreamOutput,
46 createReplayReadableStream,
47 getInvalidStoredStreamOutputError,
48 getStreamOutputMetaKey,
49 isReadableStreamLike,
50 rollbackStreamOutput,
51 StreamOutputState,
52 writeStreamOutput,
53 } from "./lib/streams.ts";
54 import type { StreamOutputMeta } from "./lib/streams.ts";
55 import {
56 isValidStepConfig,
57 isValidStepName,
58 MAX_STEP_NAME_LENGTH,
59 SENSITIVE_STEP_OUTPUT,
60 } from "./lib/validators.ts";
61 import { MODIFIER_KEYS } from "./modifier.ts";
62
63 export type Event = {
64 timestamp: Date;
65 payload: unknown;
66 type: string;
67 };
68
69 // Persisted in place of a dynamic delay function (functions can't be
70 // serialized); the live function is re-read from the user's step config on each
71 // execution.
72 const SERIALIZABLE_DELAY_MARKER = "[dynamic]";
73 type SerializableDelayMarker = typeof SERIALIZABLE_DELAY_MARKER;
74
75 export const REDACTED_STEP_OUTPUT = "[REDACTED]";
76
77 // The persisted, fully-merged config. A dynamic delay is stored as the marker.
78 export type ResolvedStepConfig = {
79 retries: {
80 limit: number;
81 delay: WorkflowDelayDuration | SerializableDelayMarker;
82 backoff?: WorkflowBackoff;
83 };
84 timeout: WorkflowTimeoutDuration;
85 sensitive?: WorkflowStepSensitivity;
86 };
87
88 // The user-facing config (passed to step callbacks). The dynamic-delay marker is
89 // never exposed: `delay` is omitted entirely when the delay is a function.
90 export type EngineStepConfig = {
91 retries: {
92 limit: number;
93 delay?: WorkflowDelayDuration;
94 backoff?: WorkflowBackoff;
95 };
96 timeout: WorkflowTimeoutDuration;
97 sensitive?: WorkflowStepSensitivity;
98 };
99
100 export function toEngineStepConfig(config: ResolvedStepConfig): EngineStepConfig {
101 const { delay, ...retries } = config.retries;
102 if (
103 typeof delay === "number" ||
104 (typeof delay === "string" && delay !== SERIALIZABLE_DELAY_MARKER)
105 ) {
106 return { ...config, retries: { ...retries, delay } };
107 }
108 return { ...config, retries };
109 }
110
111 const defaultConfig: ResolvedStepConfig = {
112 retries: {
113 limit: 5,
114 delay: 1000,
115 backoff: "exponential",
116 },
117 timeout: "10 minutes",
118 };
119
120 export interface UserErrorField {
121 isUserError?: boolean;
122 }
123
124 export type StepState = {
125 attemptedCount: number;
126 };
127
128 export type WorkflowStepContext = {
129 step: {
130 name: string;
131 count: number;
132 };
133 attempt: number;
134 config: EngineStepConfig;
135 };
136
137 const PAUSE_DATETIME = "PAUSE_DATETIME";
138
139 export class Context extends RpcTarget {
140 #engine: Engine;
141 #state: DurableObjectState;
142
143 #counters: Map<string, number> = new Map();
144 #lifetimeStepCounter: number = 0;
145
146 #rollbackStep: { cacheKey: string } | undefined;
147
148 constructor(engine: Engine, state: DurableObjectState, rollbackStep?: { cacheKey: string }) {
149 super();
150 this.#engine = engine;
151 this.#state = state;
152 this.#rollbackStep = rollbackStep;
153 }
154
155 async #checkForPendingPause(): Promise<void> {
156 if (this.#engine.timeoutHandler.isRunningStep()) {
157 return;
158 }
159
160 const status = await this.#engine.getStatus();
161
162 if (status === InstanceStatus.Paused) {
163 throw new Error(ABORT_REASONS.USER_PAUSE);
164 }
165
166 if (status === InstanceStatus.WaitingForPause) {
167 await this.#state.storage.put(PAUSE_DATETIME, new Date());
168 const metadata = await this.#state.storage.get<InstanceMetadata>(INSTANCE_METADATA);
169 if (metadata) {
170 await this.#engine.setStatus(
171 metadata.accountId,
172 metadata.instance.id,
173 InstanceStatus.Paused,
174 );
175 }
176 throw new Error(ABORT_REASONS.USER_PAUSE);
177 }
178 }
179
180 #getCount(name: string): number {
181 let val = this.#counters.get(name) ?? 0;
182 // 1-indexed, as we're increasing the value before write
183 val++;
184 this.#counters.set(name, val);
185
186 return val;
187 }
188
189 #registerRollback(options: {
190 cacheKey: string;
191 rollbackFn: RollbackFn | undefined;
192 stepContext: WorkflowStepContext;
193 output?: unknown;
194 rollbackConfig?: WorkflowStepConfig;
195 }): void {
196 const { cacheKey, rollbackFn, stepContext, output, rollbackConfig } = options;
197 if (rollbackFn && this.#rollbackStep === undefined) {
198 this.#engine.registerRollbackFn({
199 cacheKey,
200 fn: rollbackFn,
201 stepContext,
202 ...("output" in options && { output }),
203 ...(rollbackConfig !== undefined && { config: rollbackConfig }),
204 });
205 }
206 }
207
208 do(
209 name: string,
210 callback: (ctx: WorkflowStepContext) => Promise<unknown>,
211 rollbackOptions?: WorkflowStepRollbackOptions,
212 ): Promise<unknown>;
213 do(
214 name: string,
215 config: WorkflowStepConfig,
216 callback: (ctx: WorkflowStepContext) => Promise<unknown>,
217 rollbackOptions?: WorkflowStepRollbackOptions,
218 ): Promise<unknown>;
219
220 async do<T>(name: string, ...rest: Array<unknown>): Promise<unknown> {
221 let closure: (ctx: WorkflowStepContext) => Promise<T>;
222 let stepConfig: WorkflowStepConfig;
223 let rollbackOptions: WorkflowStepRollbackOptions | undefined;
224
225 const first = rest[0];
226 if (typeof first === "function") {
227 closure = first as (ctx: WorkflowStepContext) => Promise<T>;
228 stepConfig = {};
229 rollbackOptions = parseRollbackOptions(name, rest[1]);
230 } else {
231 stepConfig = (first ?? {}) as WorkflowStepConfig;
232 closure = rest[1] as (ctx: WorkflowStepContext) => Promise<T>;
233 if (typeof closure !== "function") {
234 const error = new WorkflowFatalError(
235 `Step "${name}" requires a callback function`,
236 ) as Error & UserErrorField;
237 error.isUserError = true;
238 throw error;
239 }
240 rollbackOptions = parseRollbackOptions(name, rest[2]);
241 }
242 const { rollback: rollbackFn, rollbackConfig } = rollbackOptions ?? {};
243
244 const isRollback = this.#rollbackStep !== undefined;
245 if (this.#engine.rollbackPhase === "rollback" && !isRollback) {
246 throw new WorkflowFatalError("Cannot execute steps during rollback phase");
247 }
248
249 const events = isRollback
250 ? {
251 start: InstanceEvent.ROLLBACK_STEP_START,
252 attemptStart: InstanceEvent.ROLLBACK_ATTEMPT_START,
253 attemptSuccess: InstanceEvent.ROLLBACK_ATTEMPT_SUCCESS,
254 attemptFailure: InstanceEvent.ROLLBACK_ATTEMPT_FAILURE,
255 success: InstanceEvent.ROLLBACK_STEP_SUCCESS,
256 failure: InstanceEvent.ROLLBACK_STEP_FAILURE,
257 }
258 : {
259 start: InstanceEvent.STEP_START,
260 attemptStart: InstanceEvent.ATTEMPT_START,
261 attemptSuccess: InstanceEvent.ATTEMPT_SUCCESS,
262 attemptFailure: InstanceEvent.ATTEMPT_FAILURE,
263 success: InstanceEvent.STEP_SUCCESS,
264 failure: InstanceEvent.STEP_FAILURE,
265 };
266
267 if (!isRollback) {
268 this.#lifetimeStepCounter++;
269
270 const stepLimit = this.#engine.stepLimit;
271 if (this.#lifetimeStepCounter > stepLimit) {
272 throw new WorkflowFatalError(
273 `The limit of ${stepLimit} steps has been reached. This limit can be changed in your worker configuration.`,
274 );
275 }
276 }
277
278 if (!isValidStepName(name)) {
279 // NOTE(lduarte): marking errors as user error allows the observability layer to avoid leaking
280 // user errors to sentry while making everything more observable. `isUserError` is not serialized
281 // into userland code due to how workerd serialzises errors over RPC - we also set it as undefined
282 // in the obs layer in case changes to workerd happen
283 const error = new WorkflowFatalError(
284 `Step name "${name}" exceeds max length (${MAX_STEP_NAME_LENGTH} chars) or invalid characters found`,
285 ) as Error & UserErrorField;
286 error.isUserError = true;
287 throw error;
288 }
289
290 if (!isValidStepConfig(stepConfig)) {
291 const error = new WorkflowFatalError(
292 `Step config for "${name}" is in a invalid format. See https://developers.cloudflare.com/workflows/build/sleeping-and-retrying/`,
293 ) as Error & UserErrorField;
294 error.isUserError = true;
295 throw error;
296 }
297
298 let liveDelay: WorkflowDelayDuration | WorkflowDelayFunction =
299 stepConfig.retries?.delay ?? DEFAULT_RETRY_DELAY_MS;
300
301 let config: ResolvedStepConfig = {
302 ...defaultConfig,
303 ...stepConfig,
304 retries: {
305 ...defaultConfig.retries,
306 ...stepConfig.retries,
307 delay: typeof liveDelay === "function" ? SERIALIZABLE_DELAY_MARKER : liveDelay,
308 },
309 };
310
311 let cacheKey: string;
312 let count: number;
313 let stepNameWithCounter: string;
314 const rollbackStep = this.#rollbackStep;
315 if (rollbackStep !== undefined) {
316 cacheKey = `${ROLLBACK_CACHE_KEY_PREFIX}${rollbackStep.cacheKey}`;
317 count = 1;
318 stepNameWithCounter = name;
319 } else {
320 const hash = await computeHash(name);
321 count = this.#getCount("run-" + name);
322 cacheKey = `${hash}-${count}`;
323 stepNameWithCounter = `${name}-${count}`;
324 }
325
326 const valueKey = `${cacheKey}-value`;
327 const streamMetaKey = getStreamOutputMetaKey(cacheKey);
328 const configKey = `${cacheKey}-config`;
329 const errorKey = `${cacheKey}-error`;
330 const stepStateKey = `${cacheKey}-metadata`;
331 const retryDelayDisableKey = `${MODIFIER_KEYS.DISABLE_RETRY_DELAY}${valueKey}`;
332
333 const maybeMap = await this.#state.storage.get([
334 valueKey,
335 streamMetaKey,
336 configKey,
337 errorKey,
338 stepStateKey,
339 ]);
340
341 // Check cache -- streams first, then plain values
342 const maybeStreamMeta = maybeMap.get(streamMetaKey) as StreamOutputMeta | undefined | null;
343 const cachedConfig = maybeMap.get(configKey) as ResolvedStepConfig | undefined;
344 if (maybeStreamMeta?.state === StreamOutputState.Complete) {
345 const cachedState = maybeMap.get(stepStateKey) as StepState | undefined;
346 const maybeOutputError = getInvalidStoredStreamOutputError(
347 this.#state.storage,
348 cacheKey,
349 maybeStreamMeta,
350 );
351 if (maybeOutputError !== undefined) {
352 throw new WorkflowInternalError(
353 `Stored output for ${stepNameWithCounter} is corrupt or incomplete.`,
354 );
355 }
356
357 const result = createReplayReadableStream({
358 storage: this.#state.storage,
359 cacheKey,
360 meta: maybeStreamMeta,
361 }) as T;
362 this.#registerRollback({
363 cacheKey,
364 rollbackFn,
365 stepContext: {
366 step: { name, count },
367 attempt: cachedState?.attemptedCount ?? 1,
368 config: toEngineStepConfig(cachedConfig ?? config),
369 },
370 output: result,
371 rollbackConfig,
372 });
373 return result;
374 } else if (maybeStreamMeta !== undefined && maybeStreamMeta !== null) {
375 // We're not in a complete state - means we crashed while persisting a stream on a previous invocation - need to cleanup
376 await cleanupPendingStreamOutput(this.#state.storage, cacheKey).catch(() => {});
377 }
378
379 const maybeResult = maybeMap.get(valueKey);
380
381 if (maybeResult) {
382 const cachedState = maybeMap.get(stepStateKey) as StepState | undefined;
383 const result = (maybeResult as { value: T }).value;
384 this.#registerRollback({
385 cacheKey,
386 rollbackFn,
387 stepContext: {
388 step: { name, count },
389 attempt: cachedState?.attemptedCount ?? 1,
390 config: toEngineStepConfig(cachedConfig ?? config),
391 },
392 output: result,
393 rollbackConfig,
394 });
395 return result;
396 }
397
398 const maybeError: (Error & UserErrorField) | undefined = maybeMap.get(errorKey) as
399 | Error
400 | undefined;
401
402 if (maybeError) {
403 const cachedState = maybeMap.get(stepStateKey) as StepState | undefined;
404 this.#registerRollback({
405 cacheKey,
406 rollbackFn,
407 stepContext: {
408 step: { name, count },
409 attempt: cachedState?.attemptedCount ?? 1,
410 config: toEngineStepConfig(cachedConfig ?? config),
411 },
412 rollbackConfig,
413 });
414 maybeError.isUserError = true;
415 throw maybeError;
416 }
417
418 // Persist initial config because user can pass in dynamic config
419 if (cachedConfig === undefined) {
420 await this.#state.storage.put(configKey, config);
421 } else {
422 config = cachedConfig;
423 // Recover the live delay from the (non-serializable) user config: a
424 // persisted marker means the user passed a function, so re-read it.
425 liveDelay =
426 config.retries.delay === SERIALIZABLE_DELAY_MARKER
427 ? typeof stepConfig.retries?.delay === "function"
428 ? stepConfig.retries.delay
429 : DEFAULT_RETRY_DELAY_MS
430 : config.retries.delay;
431 }
432
433 if (this.#engine.rollbackPhase === "replay" && !isRollback) {
434 const cachedState = maybeMap.get(stepStateKey) as StepState | undefined;
435 this.#registerRollback({
436 cacheKey,
437 rollbackFn,
438 stepContext: {
439 step: { name, count },
440 attempt: cachedState?.attemptedCount ?? 1,
441 config: toEngineStepConfig(config),
442 },
443 rollbackConfig,
444 });
445 return undefined;
446 }
447
448 const attemptLogs = this.#engine
449 .readLogsFromStep(cacheKey)
450 .filter((val) =>
451 [events.attemptSuccess, events.attemptFailure, events.attemptStart].includes(val.event),
452 );
453
454 // this means that the the engine died while executing this step - we can mark the latest attempt as failed
455 if (attemptLogs.length > 0 && attemptLogs.at(-1)?.event === events.attemptStart) {
456 // TODO: We should get this from SQL
457 const stepState = ((await this.#state.storage.get(stepStateKey)) as StepState) ?? {
458 attemptedCount: 1,
459 };
460
461 const priorityQueueHash = `${cacheKey}-${stepState.attemptedCount}`;
462
463 // @ts-expect-error priorityQueue is initiated in init
464 const timeoutEntryPQ = this.#engine.priorityQueue.getFirst(
465 (a) => a.hash === priorityQueueHash && a.type === "timeout",
466 );
467 // if there's a timeout on the PQ we pop it, because we wont need it
468 if (timeoutEntryPQ !== undefined) {
469 // @ts-expect-error priorityQueue is initiated in init
470 this.#engine.priorityQueue.remove(timeoutEntryPQ);
471 }
472 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
473 attempt: stepState.attemptedCount,
474 error: {
475 name: "WorkflowInternalError",
476 message: `Attempt failed due to internal workflows error`,
477 },
478 });
479
480 await this.#state.storage.put(stepStateKey, stepState);
481 }
482
483 const doWrapper = async (
484 doWrapperClosure: (ctx: WorkflowStepContext) => Promise<unknown>,
485 ): Promise<unknown> => {
486 const stepState = ((await this.#state.storage.get(stepStateKey)) as StepState) ?? {
487 attemptedCount: 0,
488 };
489 const forwardStepContext = (): WorkflowStepContext => ({
490 step: { name, count },
491 attempt: stepState.attemptedCount,
492 config: toEngineStepConfig(config),
493 });
494
495 // NOTE(caio): this might be a stream returning step - if so cleanup stale data from previous lifetimes
496 await cleanupPendingStreamOutput(this.#state.storage, cacheKey).catch(() => {});
497
498 await this.#engine.timeoutHandler.acquire(this.#engine);
499
500 if (stepState.attemptedCount == 0) {
501 this.#engine.writeLog(events.start, cacheKey, stepNameWithCounter, {
502 config: toEngineStepConfig(config),
503 ...(!isRollback && rollbackFn ? { hasRollback: true } : {}),
504 });
505 } else {
506 // in case the engine dies while retrying and wakes up before the retry period
507 const priorityQueueHash = `${cacheKey}-${stepState.attemptedCount}`;
508 // @ts-expect-error priorityQueue is initiated in init
509 const retryEntryPQ = this.#engine.priorityQueue.getFirst(
510 (a) => a.hash === priorityQueueHash && a.type === "retry",
511 );
512 // complete sleep if it didn't finish for some reason
513 if (retryEntryPQ !== undefined) {
514 const disableAllRetryDelays = await this.#state.storage.get(
515 MODIFIER_KEYS.DISABLE_ALL_RETRY_DELAYS,
516 );
517 const disableThisRetryDelay = await this.#state.storage.get(retryDelayDisableKey);
518 const disableRetryDelay = disableAllRetryDelays || disableThisRetryDelay;
519
520 await this.#engine.timeoutHandler.release(this.#engine);
521 await scheduler.wait(disableRetryDelay ? 0 : retryEntryPQ.targetTimestamp - Date.now());
522 await this.#engine.timeoutHandler.acquire(this.#engine);
523 // @ts-expect-error priorityQueue is initiated in init
524 this.#engine.priorityQueue.remove({
525 hash: priorityQueueHash,
526 type: "retry",
527 });
528 }
529 }
530
531 let result;
532
533 const instanceMetadata = await this.#state.storage.get<InstanceMetadata>(INSTANCE_METADATA);
534 if (!instanceMetadata) {
535 throw new Error("instanceMetadata is undefined");
536 }
537 const { accountId, instance } = instanceMetadata;
538
539 let streamResultSeen = false;
540 let lastStreamMeta: StreamOutputMeta | undefined;
541 const abortController = new AbortController();
542 const stepExecutionSignal = AbortSignal.any([
543 abortController.signal,
544 this.#engine.engineAbortController.signal,
545 ]);
546
547 try {
548 const timeoutPromise = async (): Promise<never> => {
549 const priorityQueueHash = `${cacheKey}-${stepState.attemptedCount}`;
550 let timeout = ms(config.timeout);
551 if (forceStepTimeout) {
552 timeout = 0;
553 }
554 // @ts-expect-error priorityQueue is initiated in init
555 await this.#engine.priorityQueue.add({
556 hash: priorityQueueHash,
557 targetTimestamp: Date.now() + timeout,
558 type: "timeout",
559 });
560 await schedulerWait(timeout, { signal: stepExecutionSignal });
561 if (stepExecutionSignal.aborted) {
562 // Step finished or the engine aborted: the native timer is already
563 // cancelled. Hang so Promise.race does not treat this as a timeout.
564 await new Promise<never>(() => {});
565 }
566 // if we reach here, means that we can try to delete the timeout from the PQ
567 // because we managed to wait in the same lifetime
568 // @ts-expect-error priorityQueue is initiated in init
569 this.#engine.priorityQueue.remove({
570 hash: priorityQueueHash,
571 type: "timeout",
572 });
573 const error = new WorkflowTimeoutError(`Execution timed out after ${timeout}ms`);
574 abortController.abort(error);
575 throw error;
576 };
577
578 this.#engine.writeLog(events.attemptStart, cacheKey, stepNameWithCounter, {
579 attempt: stepState.attemptedCount + 1,
580 });
581 stepState.attemptedCount++;
582 this.#registerRollback({
583 cacheKey,
584 rollbackFn,
585 stepContext: forwardStepContext(),
586 rollbackConfig,
587 });
588 await this.#state.storage.put(stepStateKey, stepState);
589 const priorityQueueHash = `${cacheKey}-${stepState.attemptedCount}`;
590
591 const mockErrorKey = `${MODIFIER_KEYS.MOCK_STEP_ERROR}${valueKey}`;
592 const persistentMockError = await this.#state.storage.get<{
593 name: string;
594 message: string;
595 }>(mockErrorKey);
596 const transientMockError = await this.#state.storage.get<{
597 name: string;
598 message: string;
599 }>(`${mockErrorKey}-${stepState.attemptedCount}`);
600 const mockErrorPayload = persistentMockError || transientMockError;
601
602 // if a mocked error exists, throw it immediately
603 if (mockErrorPayload) {
604 const errorToThrow = new Error(mockErrorPayload.message);
605 errorToThrow.name = mockErrorPayload.name;
606 throw errorToThrow;
607 }
608
609 const replaceResult = await this.#state.storage.get(
610 `${MODIFIER_KEYS.REPLACE_RESULT}${valueKey}`,
611 );
612
613 const forceStepTimeoutKey = `${MODIFIER_KEYS.FORCE_STEP_TIMEOUT}${valueKey}`;
614 const persistentStepTimeout = await this.#state.storage.get(forceStepTimeoutKey);
615 const transientStepTimeout = await this.#state.storage.get(
616 `${forceStepTimeoutKey}-${stepState.attemptedCount}`,
617 );
618 const forceStepTimeout = persistentStepTimeout || transientStepTimeout;
619
620 let timeoutTask: Promise<never> | undefined;
621
622 const persistStepResult = async (
623 value: unknown,
624 activeTimeoutTask?: Promise<never>,
625 ): Promise<unknown> => {
626 if (!isReadableStreamLike(value)) {
627 try {
628 // Non-stream results must be structured-cloneable. Clone before
629 // normalisation, which can discard unsupported array properties,
630 // and avoid forwarding the callback's RPC disposer to the caller.
631 const cloned = structuredClone(value);
632 // Compact typed-array backing buffers only for storage (#14101).
633 const stored = normalizeForStorage(cloned);
634 await this.#state.storage.put(valueKey, { value: stored });
635 abortController.abort("step finished");
636 // @ts-expect-error priorityQueue is initiated in init
637 this.#engine.priorityQueue.remove({
638 hash: priorityQueueHash,
639 type: "timeout",
640 });
641 return cloned;
642 } finally {
643 (value as Partial<Disposable> | null | undefined)?.[Symbol.dispose]?.();
644 }
645 }
646
647 streamResultSeen = true;
648 const streamMeta = await writeStreamOutput({
649 storage: this.#state.storage,
650 cacheKey,
651 attempt: stepState.attemptedCount,
652 stream: value as ReadableStream<unknown>,
653 signal: stepExecutionSignal,
654 timeoutTask: activeTimeoutTask,
655 });
656 lastStreamMeta = streamMeta;
657
658 abortController.abort("step finished");
659 // @ts-expect-error priorityQueue is initiated in init
660 this.#engine.priorityQueue.remove({
661 hash: priorityQueueHash,
662 type: "timeout",
663 });
664 return createReplayReadableStream({
665 storage: this.#state.storage,
666 cacheKey,
667 meta: streamMeta,
668 });
669 };
670
671 if (forceStepTimeout) {
672 result = await timeoutPromise();
673 } else if (replaceResult) {
674 // Check if the mocked result is a stream sentinel (from mockStepResult with ReadableStream)
675 if (
676 replaceResult &&
677 typeof replaceResult === "object" &&
678 (replaceResult as Record<string, unknown>).__mockStreamOutput
679 ) {
680 const sentinel = replaceResult as {
681 __mockStreamOutput: true;
682 cacheKey: string;
683 meta: StreamOutputMeta;
684 };
685 result = createReplayReadableStream({
686 storage: this.#state.storage,
687 cacheKey: sentinel.cacheKey,
688 meta: sentinel.meta,
689 });
690 } else {
691 result = replaceResult;
692 }
693 } else {
694 timeoutTask = timeoutPromise();
695 const callbackTask = doWrapperClosure({
696 step: { name, count },
697 attempt: stepState.attemptedCount,
698 config: toEngineStepConfig(config),
699 });
700 try {
701 result = await Promise.race([callbackTask, timeoutTask]);
702 } catch (error) {
703 // A timeout does not cancel the callback RPC. Release any late
704 // result without waiting for it before retrying. Successful race
705 // winners remain owned by persistStepResult, including streams.
706 // The engine is a Durable Object, which stays active while there
707 // is ongoing work or pending I/O, so this untracked cleanup is
708 // retained without a waitUntil (a documented no-op for Durable
709 // Objects). Once the run() call that owns the callback stub ends,
710 // the stub is torn down with it and there is no result to release.
711 void callbackTask
712 .then(async (value) => {
713 try {
714 if (isReadableStreamLike(value)) {
715 await value.cancel(error);
716 }
717 } finally {
718 (value as Partial<Disposable> | null | undefined)?.[Symbol.dispose]?.();
719 }
720 })
721 // Late rejection or cleanup failure must not replace the
722 // attempt's original error or become an unhandled rejection.
723 .catch(() => {});
724 throw error;
725 }
726 }
727
728 // We store the value of `output` in an object with a `value` property. This allows us to store `undefined`,
729 // in the case that it's returned from the user's code. This is because DO storage will error if you try to
730 // store undefined directly.
731 try {
732 result = await persistStepResult(result, timeoutTask);
733 } catch (e) {
734 abortController.abort("step errored");
735 // @ts-expect-error priorityQueue is initiated in init
736 this.#engine.priorityQueue.remove({
737 hash: priorityQueueHash,
738 type: "timeout",
739 });
740
741 if (e instanceof WorkflowTimeoutError) {
742 throw e;
743 }
744
745 // Fatal serialization/storage errors abort the DO immediately, so
746 // previously registered rollbacks do not run for these paths.
747 // This matches the existing terminal behavior for unrecoverable output.
748 // Stream-specific fatal errors
749 if (
750 e instanceof InvalidStepReadableStreamError ||
751 e instanceof OversizedStreamChunkError ||
752 e instanceof UnsupportedStreamChunkError
753 ) {
754 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
755 attempt: stepState.attemptedCount,
756 error: new WorkflowFatalError(e.message),
757 });
758 this.#engine.writeLog(events.failure, cacheKey, stepNameWithCounter, {});
759 this.#engine.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
760 error: new WorkflowFatalError(
761 `The execution of the Workflow instance was terminated, as the step "${name}" returned an invalid ReadableStream output. ${e.message}`,
762 ),
763 });
764
765 await this.#engine.setStatus(accountId, instance.id, InstanceStatus.Errored);
766 await this.#engine.timeoutHandler.release(this.#engine);
767 await this.#engine.abort(ABORT_REASONS.NOT_SERIALISABLE);
768 return;
769 }
770
771 if (e instanceof StreamOutputStorageLimitError) {
772 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
773 attempt: stepState.attemptedCount,
774 error: new WorkflowFatalError(e.message),
775 });
776 this.#engine.writeLog(events.failure, cacheKey, stepNameWithCounter, {});
777 this.#engine.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
778 error: new WorkflowFatalError("The instance has exceeded the 1GiB storage limit"),
779 });
780
781 await this.#engine.setStatus(accountId, instance.id, InstanceStatus.Errored);
782 await this.#engine.timeoutHandler.release(this.#engine);
783 await this.#engine.abort(ABORT_REASONS.STORAGE_LIMIT_EXCEEDED);
784 return;
785 }
786
787 // something that cannot be written to storage
788 if (e instanceof Error && e.name === "DataCloneError") {
789 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
790 attempt: stepState.attemptedCount,
791 error: new WorkflowFatalError(
792 `Value returned from step "${name}" is not serialisable`,
793 ),
794 });
795 this.#engine.writeLog(events.failure, cacheKey, stepNameWithCounter, {});
796 this.#engine.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
797 error: new WorkflowFatalError(
798 `The execution of the Workflow instance was terminated, as the step "${name}" returned a value which is not serialisable`,
799 ),
800 });
801
802 await this.#engine.setStatus(accountId, instance.id, InstanceStatus.Errored);
803 await this.#engine.timeoutHandler.release(this.#engine);
804 await this.#engine.abort(ABORT_REASONS.NOT_SERIALISABLE);
805 } else if (
806 e instanceof Error &&
807 e.message.includes("string or blob too big: SQLITE_TOOBIG")
808 ) {
809 throw new WorkflowInternalError(
810 `Step ${stepNameWithCounter} output is too large. Maximum allowed size is 1MiB.`,
811 );
812 } else {
813 // TODO (WOR-77): Send this to Sentry
814 throw new WorkflowInternalError(
815 `Storage failure for ${stepNameWithCounter} due to internal error.`,
816 );
817 }
818 return;
819 }
820
821 // if we reach here, means that the closure ran successfully and we can remove the timeout from the PQ
822 // @ts-expect-error priorityQueue is initiated in init
823 this.#engine.priorityQueue.remove({
824 hash: priorityQueueHash,
825 type: "timeout",
826 });
827
828 this.#engine.writeLog(events.attemptSuccess, cacheKey, stepNameWithCounter, {
829 attempt: stepState.attemptedCount,
830 });
831 } catch (e) {
832 const error = e as Error;
833 // Cancel the native timeout timer for this attempt. Removing the PQ
834 // entry alone leaves scheduler.wait() holding a workerd timer until
835 // the original deadline (issue #15788).
836 abortController.abort("step errored");
837 // @ts-expect-error priorityQueue is initiated in init
838 this.#engine.priorityQueue.remove({
839 hash: `${cacheKey}-${stepState.attemptedCount}`,
840 type: "timeout",
841 });
842
843 // Clean up any partial stream output from this failed attempt
844 if (streamResultSeen) {
845 try {
846 await rollbackStreamOutput(this.#state.storage, cacheKey, stepState.attemptedCount);
847 } catch {
848 // Best-effort cleanup
849 }
850 }
851
852 if (
853 e instanceof Error &&
854 (error.name === "NonRetryableError" || error.message.startsWith("NonRetryableError"))
855 ) {
856 const attemptError = shouldPreserveNonRetryableError()
857 ? new PreservedNonRetryableError(e)
858 : new WorkflowFatalError(`Step threw a NonRetryableError with message "${e.message}"`);
859
860 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
861 attempt: stepState.attemptedCount,
862 error: attemptError,
863 });
864 this.#engine.writeLog(
865 events.failure,
866 cacheKey,
867 stepNameWithCounter,
868 !isRollback && rollbackFn ? { hasRollback: true } : {},
869 );
870 this.#registerRollback({
871 cacheKey,
872 rollbackFn,
873 stepContext: forwardStepContext(),
874 rollbackConfig,
875 });
876
877 throw error;
878 }
879
880 await this.#state.storage.put(stepStateKey, stepState);
881
882 const willRetry = stepState.attemptedCount <= config.retries.limit;
883 const priorityQueueHash = `${cacheKey}-${stepState.attemptedCount}`;
884
885 // Resolve the delay once per attempt. A broken delay function turns
886 // the retry into a non-retryable failure.
887 let retryDelayMs: number | undefined;
888 let delayFailure: (Error & UserErrorField) | undefined;
889 if (willRetry) {
890 try {
891 let resolvedDelay: unknown;
892 if (typeof liveDelay === "function") {
893 // Park a default-delayed retry before invoking the user's
894 // delay function so a crash while the function is in flight
895 // still reschedules the step. The PQ table is append-only
896 // with a UNIQUE(action, entryType, hash) constraint, so this
897 // exact entry is reused as the retry entry below — it is
898 // never removed and re-added (that would violate UNIQUE).
899 // @ts-expect-error priorityQueue is initiated in init
900 await this.#engine.priorityQueue.add({
901 hash: priorityQueueHash,
902 targetTimestamp: Date.now() + DEFAULT_RETRY_DELAY_MS,
903 type: "retry",
904 });
905
906 const stepCtx: WorkflowStepContext = {
907 step: { name, count },
908 attempt: stepState.attemptedCount,
909 config: toEngineStepConfig(config),
910 };
911
912 resolvedDelay = await invokeDelayFunction(
913 liveDelay,
914 { ctx: stepCtx, error },
915 {
916 timeoutMs: DELAY_FUNCTION_TIMEOUT_MS,
917 wait: schedulerWait,
918 signal: this.#engine.engineAbortController.signal,
919 },
920 );
921 } else {
922 resolvedDelay = liveDelay;
923 }
924 retryDelayMs = calcRetryDuration(config, stepState, resolvedDelay);
925 } catch (delayErr) {
926 if (!(delayErr instanceof DelayFunctionError)) {
927 throw delayErr;
928 }
929 // @ts-expect-error priorityQueue is initiated in init
930 this.#engine.priorityQueue.remove({
931 hash: priorityQueueHash,
932 type: "retry",
933 });
934 const userError = new NonRetryableDelayError(
935 `The delay function for step "${stepNameWithCounter}" ${delayErr.message}`,
936 ) as Error & UserErrorField;
937 userError.isUserError = true;
938 delayFailure = userError;
939 }
940 }
941
942 this.#engine.writeLog(events.attemptFailure, cacheKey, stepNameWithCounter, {
943 attempt: stepState.attemptedCount,
944 error: {
945 name: error.name,
946 message: error.message,
947 // TODO (WOR-79): Stacks are all incorrect over RPC and need work
948 // stack: error.stack,
949 },
950 ...(retryDelayMs !== undefined ? { retryDelayMs } : {}),
951 });
952
953 if (willRetry && delayFailure === undefined) {
954 // TODO (WOR-71): Think through if every Error should transition
955 const durationMs = retryDelayMs ?? DEFAULT_RETRY_DELAY_MS;
956 const disableAllRetryDelays = await this.#state.storage.get(
957 MODIFIER_KEYS.DISABLE_ALL_RETRY_DELAYS,
958 );
959 const disableThisRetryDelay = await this.#state.storage.get(retryDelayDisableKey);
960 const disableRetryDelay = disableAllRetryDelays || disableThisRetryDelay;
961 const effectiveDuration = disableRetryDelay ? 0 : durationMs;
962
963 // A dynamic delay already parked its retry entry under this hash
964 // above (the append-only PQ rejects re-adding the same hash), so
965 // only static delays add here. The live wait below uses
966 // effectiveDuration regardless; the parked entry's timestamp only
967 // drives crash recovery.
968 //
969 // Known trade-off: for a dynamic delay the parked entry keeps the
970 // DEFAULT_RETRY_DELAY_MS placeholder timestamp set before the delay
971 // function ran (see the add above) — the UNIQUE(action, entryType,
972 // hash) constraint means it can't be overwritten in place with the
973 // computed value without a remove+re-add that would violate the
974 // constraint. So if the engine crashes mid-wait, recovery resumes
975 // the step at the placeholder delay rather than the computed one.
976 // This is accepted for local dev: the step still retries with the
977 // correct semantics, only the post-crash wait can be shorter.
978 if (typeof liveDelay !== "function") {
979 // @ts-expect-error priorityQueue is initiated in init
980 await this.#engine.priorityQueue.add({
981 hash: priorityQueueHash,
982 targetTimestamp: Date.now() + effectiveDuration,
983 type: "retry",
984 });
985 }
986 await this.#engine.timeoutHandler.release(this.#engine);
987 // Race retry wait against the pause signal so pause
988 // takes effect immediately during retries
989 {
990 const retryPauseSignal = this.#engine.pauseController.signal;
991 await raceAgainstAbort(
992 schedulerWait(effectiveDuration, { signal: retryPauseSignal }),
993 retryPauseSignal,
994 );
995 const retryStatus = await this.#engine.getStatus();
996 const pausedDuringRetry =
997 retryStatus === InstanceStatus.Paused ||
998 retryStatus === InstanceStatus.WaitingForPause;
999 if (pausedDuringRetry) {
1000 throw new Error(ABORT_REASONS.USER_PAUSE);
1001 }
1002 }
1003
1004 // if it ever reaches here, we can try to remove it from the priority queue since it's no longer useful
1005 // @ts-expect-error priorityQueue is initiated in init
1006 this.#engine.priorityQueue.remove({
1007 hash: priorityQueueHash,
1008 type: "retry",
1009 });
1010
1011 return doWrapper(doWrapperClosure);
1012 } else {
1013 await this.#engine.timeoutHandler.release(this.#engine);
1014 // Clean up any leftover stream chunks on retry exhaustion
1015 try {
1016 await rollbackStreamOutput(this.#state.storage, cacheKey, stepState.attemptedCount);
1017 } catch {
1018 // Best-effort cleanup
1019 }
1020 this.#engine.writeLog(
1021 events.failure,
1022 cacheKey,
1023 stepNameWithCounter,
1024 !isRollback && rollbackFn ? { hasRollback: true } : {},
1025 );
1026 this.#registerRollback({
1027 cacheKey,
1028 rollbackFn,
1029 stepContext: forwardStepContext(),
1030 rollbackConfig,
1031 });
1032
1033 const finalError = delayFailure ?? error;
1034 await this.#state.storage.put(errorKey, finalError);
1035 throw finalError;
1036 }
1037 }
1038
1039 const redactOutput = config.sensitive === SENSITIVE_STEP_OUTPUT;
1040 this.#engine.writeLog(events.success, cacheKey, stepNameWithCounter, {
1041 // TODO (WOR-86): Add limits, figure out serialization
1042 result: redactOutput ? REDACTED_STEP_OUTPUT : lastStreamMeta ? undefined : result,
1043 ...(!redactOutput &&
1044 lastStreamMeta && {
1045 streamOutput: { cacheKey, meta: lastStreamMeta },
1046 }),
1047 ...(!isRollback && rollbackFn ? { hasRollback: true } : {}),
1048 });
1049 this.#registerRollback({
1050 cacheKey,
1051 rollbackFn,
1052 stepContext: forwardStepContext(),
1053 output: result,
1054 rollbackConfig,
1055 });
1056 await this.#engine.timeoutHandler.release(this.#engine);
1057 return result;
1058 };
1059
1060 const result = await doWrapper(closure);
1061
1062 // Check if a pause was requested while this step was running.
1063 // Rollback steps may run while the instance is paused; don't let stale pause
1064 // state abort rollback cleanup after the rollback step itself succeeds.
1065 if (!isRollback) {
1066 await this.#checkForPendingPause();
1067 }
1068
1069 return result;
1070 }
1071
1072 async sleep(name: string, duration: WorkflowSleepDuration): Promise<void> {
1073 if (this.#engine.rollbackPhase === "replay") {
1074 return;
1075 }
1076
1077 if (typeof duration == "string") {
1078 duration = ms(duration);
1079 }
1080
1081 const hash = await computeHash(name + duration.toString());
1082 const count = this.#getCount("sleep-" + name + duration.toString());
1083 const cacheKey = `${hash}-${count}`;
1084 const sleepNameWithCounter = `${name}-${count}`;
1085
1086 const sleepKey = `${cacheKey}-value`;
1087 const sleepLogWrittenKey = `${cacheKey}-log-written`;
1088 const maybeResult = await this.#state.storage.get(sleepKey);
1089
1090 const sleepNameCountHash = await computeHash(name + this.#getCount("sleep-" + name));
1091 const disableThisSleep = await this.#state.storage.get(
1092 `${MODIFIER_KEYS.DISABLE_SLEEP}${sleepNameCountHash}`,
1093 );
1094 const disableAllSleeps = await this.#state.storage.get(MODIFIER_KEYS.DISABLE_ALL_SLEEPS);
1095
1096 const disableSleep = disableAllSleeps || disableThisSleep;
1097
1098 if (maybeResult != undefined) {
1099 // @ts-expect-error priorityQueue is initiated in init
1100 const entryPQ = this.#engine.priorityQueue.getFirst(
1101 (a) => a.hash === cacheKey && a.type === "sleep",
1102 );
1103 // in case the engine dies while sleeping and wakes up before the retry period
1104 if (entryPQ !== undefined) {
1105 const pauseSignal = this.#engine.pauseController.signal;
1106 const raceResult = await raceAgainstAbort(
1107 schedulerWait(disableSleep ? 0 : entryPQ.targetTimestamp - Date.now(), {
1108 signal: pauseSignal,
1109 }),
1110 pauseSignal,
1111 );
1112 const statusAfterSleep = await this.#engine.getStatus();
1113 const pausedDuringSleep =
1114 raceResult.aborted ||
1115 statusAfterSleep === InstanceStatus.Paused ||
1116 statusAfterSleep === InstanceStatus.WaitingForPause;
1117 if (pausedDuringSleep) {
1118 throw new Error(ABORT_REASONS.USER_PAUSE);
1119 }
1120 // @ts-expect-error priorityQueue is initiated in init
1121 this.#engine.priorityQueue.remove({ hash: cacheKey, type: "sleep" });
1122 }
1123 const shouldWriteLog = (await this.#state.storage.get(sleepLogWrittenKey)) == undefined;
1124 if (shouldWriteLog) {
1125 this.#engine.writeLog(InstanceEvent.SLEEP_COMPLETE, cacheKey, sleepNameWithCounter, {});
1126 await this.#state.storage.put(sleepLogWrittenKey, true);
1127 }
1128 return;
1129 }
1130
1131 this.#engine.writeLog(InstanceEvent.SLEEP_START, cacheKey, sleepNameWithCounter, {
1132 durationMs: duration,
1133 });
1134 const instanceMetadata = await this.#state.storage.get<InstanceMetadata>(INSTANCE_METADATA);
1135 if (!instanceMetadata) {
1136 throw new Error("instanceMetadata is undefined");
1137 }
1138
1139 // TODO(lduarte): not sure of this order of operations
1140 await this.#state.storage.put(sleepKey, true); // Any value will do for cache hit
1141
1142 // @ts-expect-error priorityQueue is initiated in init
1143 await this.#engine.priorityQueue.add({
1144 hash: cacheKey,
1145 targetTimestamp: Date.now() + (disableSleep ? 0 : duration),
1146 type: "sleep",
1147 });
1148
1149 // Race the sleep against the pause signal
1150 const pauseSignal = this.#engine.pauseController.signal;
1151 const sleepDuration = disableSleep ? 0 : duration;
1152
1153 await raceAgainstAbort(schedulerWait(sleepDuration, { signal: pauseSignal }), pauseSignal);
1154
1155 // Check if we were paused during the sleep
1156 const statusAfterSleep = await this.#engine.getStatus();
1157 const pausedDuringSleep =
1158 statusAfterSleep === InstanceStatus.Paused ||
1159 statusAfterSleep === InstanceStatus.WaitingForPause;
1160
1161 if (pausedDuringSleep) {
1162 // Throw pause error
1163 throw new Error(ABORT_REASONS.USER_PAUSE);
1164 }
1165
1166 this.#engine.writeLog(InstanceEvent.SLEEP_COMPLETE, cacheKey, sleepNameWithCounter, {});
1167 await this.#state.storage.put(sleepLogWrittenKey, true);
1168
1169 // @ts-expect-error priorityQueue is initiated in init
1170 this.#engine.priorityQueue.remove({ hash: cacheKey, type: "sleep" });
1171 }
1172
1173 async sleepUntil(name: string, timestamp: Date | number): Promise<void> {
1174 if (this.#engine.rollbackPhase === "replay") {
1175 return;
1176 }
1177
1178 if (timestamp instanceof Date) {
1179 timestamp = timestamp.valueOf();
1180 }
1181
1182 const now = Date.now();
1183 // timestamp needs to be in the future, throw if not
1184 if (timestamp < now) {
1185 throw new Error("You can't sleep until a time in the past, time-traveler");
1186 }
1187
1188 return this.sleep(name, timestamp - now);
1189 }
1190
1191 async waitForEvent<T>(
1192 name: string,
1193 options: {
1194 type: string;
1195 timeout?: string | number;
1196 },
1197 ): Promise<WorkflowStepEvent<T>> {
1198 if (this.#engine.rollbackPhase === "rollback") {
1199 throw new WorkflowFatalError("Cannot execute steps during rollback phase");
1200 }
1201
1202 if (!options.timeout) {
1203 options.timeout = "24 hours";
1204 }
1205
1206 const count = this.#getCount("waitForEvent-" + name);
1207 const waitForEventNameWithCounter = `${name}-${count}`;
1208 const hash = await computeHash(waitForEventNameWithCounter);
1209 const cacheKey = `${hash}-${count}`;
1210 const waitForEventKey = `${cacheKey}-value`;
1211 const errorKey = `${cacheKey}-error`;
1212
1213 const pendingWaiterRegistered = `${cacheKey}-pending`;
1214
1215 const timeoutError = new WorkflowTimeoutError(
1216 `Execution timed out after ${ms(options.timeout)}ms`,
1217 ) as Error & UserErrorField;
1218
1219 const maybeResult = await this.#state.storage.get<Event>(waitForEventKey);
1220
1221 if (this.#engine.rollbackPhase === "replay") {
1222 return maybeResult as WorkflowStepEvent<T>;
1223 }
1224
1225 if (maybeResult) {
1226 const shouldWriteLog = (await this.#state.storage.get(waitForEventKey)) == undefined;
1227 if (shouldWriteLog) {
1228 this.#engine.writeLog(
1229 InstanceEvent.WAIT_COMPLETE,
1230 cacheKey,
1231 waitForEventNameWithCounter,
1232 maybeResult,
1233 );
1234 }
1235 return maybeResult as WorkflowStepEvent<T>;
1236 }
1237 const maybeError: (Error & UserErrorField) | undefined = (await this.#state.storage.get(
1238 errorKey,
1239 )) as Error | undefined;
1240
1241 if (maybeError) {
1242 maybeError.isUserError = true;
1243 throw maybeError;
1244 }
1245
1246 const maybeRegistered = await this.#state.storage.get(pendingWaiterRegistered);
1247
1248 if (!maybeRegistered) {
1249 this.#engine.writeLog(InstanceEvent.WAIT_START, cacheKey, waitForEventNameWithCounter, {
1250 event: options.type,
1251 });
1252
1253 await this.#state.storage.put(pendingWaiterRegistered, true);
1254 }
1255
1256 // if there's a timeout on the PQ we pop it, because we wont need it
1257 // @ts-expect-error priorityQueue is initiated in init
1258 const timeoutEntryPQ = this.#engine.priorityQueue.getFirst(
1259 (a) => a.hash === cacheKey && a.type === "timeout",
1260 );
1261 const forceEventTimeout = await this.#state.storage.get(
1262 `${MODIFIER_KEYS.FORCE_EVENT_TIMEOUT}${waitForEventKey}`,
1263 );
1264 if (
1265 (timeoutEntryPQ === undefined &&
1266 this.#engine.priorityQueue !== undefined &&
1267 this.#engine.priorityQueue.checkIfExistedInPast({
1268 hash: cacheKey,
1269 type: "timeout",
1270 })) ||
1271 (timeoutEntryPQ !== undefined && timeoutEntryPQ.targetTimestamp < Date.now()) ||
1272 forceEventTimeout
1273 ) {
1274 this.#engine.writeLog(InstanceEvent.WAIT_TIMED_OUT, cacheKey, waitForEventNameWithCounter, {
1275 name: timeoutError.name,
1276 message: timeoutError.message,
1277 });
1278 await this.#state.storage.put(errorKey, timeoutError);
1279 throw timeoutError;
1280 }
1281
1282 const pauseSignal = this.#engine.pauseController.signal;
1283 const timeoutAbortController = new AbortController();
1284 const timeoutSignal = AbortSignal.any([timeoutAbortController.signal, pauseSignal]);
1285
1286 const timeoutPromise = async (timeoutToWait: number, addToPQ: boolean) => {
1287 const priorityQueueHash = cacheKey;
1288 if (addToPQ) {
1289 // @ts-expect-error priorityQueue is initiated in init
1290 await this.#engine.priorityQueue.add({
1291 hash: priorityQueueHash,
1292 targetTimestamp: Date.now() + timeoutToWait,
1293 type: "timeout",
1294 });
1295 }
1296 await schedulerWait(timeoutToWait, { signal: timeoutSignal });
1297 if (timeoutSignal.aborted) {
1298 // Event arrived or the engine paused: the timer is already cancelled.
1299 // Hang so Promise.race does not treat this as a timeout.
1300 await new Promise<never>(() => {});
1301 }
1302 // if we reach here, means that we can try to delete the timeout from the PQ
1303 // because we managed to wait in the same lifetime
1304
1305 // @ts-expect-error priorityQueue is initiated in init
1306 this.#engine.priorityQueue.remove({
1307 hash: priorityQueueHash,
1308 type: "timeout",
1309 });
1310 // NOTE(lduarte): marking errors as user error allows the observability layer to avoid leaking
1311 // user errors to sentry while making everything more observable. `isUserError` is not serialized
1312 // into userland code due to how workerd serialzises errors over RPC - we also set it as undefined
1313 // in the obs layer in case changes to workerd happen
1314 const error = timeoutError;
1315 error.isUserError = true;
1316 throw error;
1317 };
1318
1319 const eventPromise = new Promise<Event>((resolve) => {
1320 // TODO: This might need to be the name, not the event type
1321
1322 const eventTypeQueue = this.#engine.eventMap.get(options.type);
1323 if (eventTypeQueue) {
1324 const event = eventTypeQueue.shift();
1325 if (event) {
1326 this.#engine.eventMap.set(options.type, eventTypeQueue);
1327 return resolve(event);
1328 }
1329 }
1330 const callbacks = this.#engine.waiters.get(options.type) ?? [];
1331 callbacks.push([cacheKey, resolve]);
1332
1333 this.#engine.waiters.set(options.type, callbacks);
1334 });
1335
1336 try {
1337 // Race event and timeout against the pause signal.
1338 const raceResult = await raceAgainstAbort(
1339 Promise.race([
1340 eventPromise,
1341 timeoutEntryPQ !== undefined
1342 ? timeoutPromise(timeoutEntryPQ.targetTimestamp - Date.now(), false)
1343 : timeoutPromise(ms(options.timeout), true),
1344 ]),
1345 pauseSignal,
1346 ).catch(async (error) => {
1347 const callbacks = this.#engine.waiters.get(options.type);
1348 if (callbacks) {
1349 const idx = callbacks.findIndex(([key]) => key === cacheKey);
1350 if (idx !== -1) {
1351 callbacks.splice(idx, 1);
1352 }
1353 }
1354
1355 this.#engine.writeLog(
1356 InstanceEvent.WAIT_TIMED_OUT,
1357 cacheKey,
1358 waitForEventNameWithCounter,
1359 error,
1360 );
1361 await this.#state.storage.put(errorKey, error);
1362 throw error;
1363 });
1364
1365 // Pause signal won the race — throw to stop the workflow
1366 if (raceResult.aborted) {
1367 throw new Error(ABORT_REASONS.USER_PAUSE);
1368 }
1369 const event = raceResult.value;
1370
1371 // @ts-expect-error priorityQueue is initiated in init
1372 this.#engine.priorityQueue.remove({
1373 hash: cacheKey,
1374 type: "timeout",
1375 });
1376
1377 this.#engine.writeLog(
1378 InstanceEvent.WAIT_COMPLETE,
1379 cacheKey,
1380 waitForEventNameWithCounter,
1381 event,
1382 );
1383 await this.#state.storage.put(waitForEventKey, event);
1384
1385 return event as WorkflowStepEvent<T>;
1386 } finally {
1387 timeoutAbortController.abort();
1388 }
1389 }
1390 }