@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 2,088 lines 70.9 KB
Raw
1 // Alchemy modifications are licensed under Apache-2.0.
2 // This file includes third-party code; see /THIRD_PARTY_LICENSES.md.
3 // Alchemy modifications: uses Array<T> syntax and expects current workerd error serialization, which preserves the message without prefixing the custom class name.
4 import { runInDurableObject } from "cloudflare:test";
5 import { env } from "cloudflare:workers";
6 import type {
7 WorkflowDelayFunction,
8 WorkflowStep,
9 WorkflowStepConfig,
10 WorkflowStepContext,
11 } from "cloudflare:workers";
12 import { NonRetryableError } from "cloudflare:workflows";
13 import { afterEach, describe, it, vi } from "vitest";
14 import type { TestContext } from "vitest";
15 import workerdUnsafe from "workerd:unsafe";
16 import type { DatabaseInstance, DatabaseVersion, DatabaseWorkflow, EngineLogs } from "../engine.ts";
17 import { DEFAULT_STEP_LIMIT, InstanceEvent, InstanceStatus } from "../index.ts";
18 import { ABORT_REASONS, isAbortError } from "../lib/errors.ts";
19 import type { RollbackContext, RollbackFn, WorkflowStepRollbackOptions } from "../lib/rollback.ts";
20 import { setTestWorkflowCallback } from "./test-entry.ts";
21 import { runWorkflow, runWorkflowAndAwait } from "./utils.ts";
22
23 afterEach(async () => {
24 await workerdUnsafe.abortAllDurableObjects();
25 });
26
27 /** Keep first-attempt failures and progress visible even when Vitest retries pass. */
28 function trackTimerQuotaTest({ task, onTestFinished }: TestContext) {
29 const start = performance.now();
30 const attempt = (task.result?.retryCount ?? 0) + 1;
31 const previousErrors = task.result?.errors?.length ?? 0;
32 const progress = { completed: 0, phase: "steps", stepsMs: 0 };
33 onTestFinished(() => {
34 console.info(
35 "Workflows timer quota:",
36 JSON.stringify({
37 test: task.name,
38 attempt,
39 ...progress,
40 elapsedMs: Math.round(performance.now() - start),
41 errors: task.result?.errors?.slice(previousErrors).map(({ name, message }) => ({
42 name,
43 message,
44 })),
45 }),
46 );
47 });
48 return { attempt, start, progress };
49 }
50
51 describe("Engine", () => {
52 it("should not retry after NonRetryableError is thrown", async ({ expect }) => {
53 const instanceId = "NON-RETRYABLE-ERROR";
54 const engineId = env.ENGINE.idFromName(instanceId);
55
56 await runWorkflow(instanceId, async (_event, step) => {
57 await step.do("should only have one retry", async () => {
58 throw new NonRetryableError("Should only retry once");
59 });
60 });
61
62 await vi.waitUntil(
63 async () => {
64 try {
65 const logs = (await env.ENGINE.get(engineId).readLogs()) as EngineLogs;
66 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE);
67 } catch (e) {
68 // DO may still be aborting — retry
69 if (isAbortError(e)) {
70 return false;
71 }
72 throw e;
73 }
74 },
75 { timeout: 5000 },
76 );
77
78 const logs = (await env.ENGINE.get(engineId).readLogs()) as EngineLogs;
79
80 expect(logs.logs.filter((val) => val.event === InstanceEvent.ATTEMPT_START)).toHaveLength(1);
81 });
82
83 it("should preserve NonRetryableError message when compat flag is enabled", async ({
84 expect,
85 }) => {
86 const instanceId = "NON-RETRYABLE-PRESERVE-MSG";
87 const engineId = env.ENGINE.idFromName(instanceId);
88 const engineStub = env.ENGINE.get(engineId);
89
90 vi.stubGlobal("Cloudflare", {
91 compatibilityFlags: {
92 workflows_preserve_non_retryable_error_message: true,
93 },
94 });
95
96 try {
97 setTestWorkflowCallback(async (_event, step) => {
98 await step.do("failing-step", async () => {
99 throw new NonRetryableError("my custom error message");
100 });
101 });
102
103 await engineStub
104 .init(
105 12346,
106 {} as DatabaseWorkflow,
107 {} as DatabaseVersion,
108 { id: instanceId } as DatabaseInstance,
109 { payload: {}, timestamp: new Date(), instanceId, workflowName: "" },
110 )
111 .catch(() => {});
112
113 await vi.waitUntil(
114 async () => {
115 try {
116 const logs = (await env.ENGINE.get(engineId).readLogs()) as EngineLogs;
117 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE);
118 } catch (e) {
119 if (isAbortError(e)) {
120 return false;
121 }
122 throw e;
123 }
124 },
125 { timeout: 3000 },
126 );
127
128 const logs = (await env.ENGINE.get(engineId).readLogs()) as EngineLogs;
129
130 const workflowFailure = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_FAILURE);
131 expect(workflowFailure?.metadata.error).toEqual({
132 name: "NonRetryableError",
133 message: "my custom error message",
134 });
135
136 const attemptFailure = logs.logs.find((val) => val.event === InstanceEvent.ATTEMPT_FAILURE);
137 expect(attemptFailure?.metadata.error).toEqual({
138 name: "NonRetryableError",
139 message: "my custom error message",
140 });
141 } finally {
142 vi.unstubAllGlobals();
143 }
144 });
145
146 it("should not error out if step fails but is try-catched", async ({ expect }) => {
147 const engineStub = await runWorkflow("MOCK-INSTANCE-ID-TRY-CATCH", async (_event, step) => {
148 try {
149 await step.do(
150 "always errors out",
151 {
152 retries: {
153 limit: 0,
154 delay: 1000,
155 },
156 },
157 async () => {
158 throw new Error("Step errors out");
159 },
160 );
161 } catch {}
162 return "finished";
163 });
164
165 await vi.waitUntil(
166 async () => {
167 const logs = (await engineStub.readLogs()) as EngineLogs;
168 return logs.logs.some((val) => val.event == InstanceEvent.WORKFLOW_SUCCESS);
169 },
170 { timeout: 5000 },
171 );
172
173 const logs = (await engineStub.readLogs()) as EngineLogs;
174
175 expect(logs.logs.some((val) => val.event == InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
176
177 expect(logs.logs.filter((val) => val.event == InstanceEvent.ATTEMPT_FAILURE)).toHaveLength(1);
178 });
179
180 it("applies a dynamic delay function and succeeds after retries", async ({ expect }) => {
181 const instanceId = "DYNAMIC-DELAY-APPLIED";
182 const dynamicDelay: WorkflowDelayFunction = () => 50;
183 let attempts = 0;
184
185 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
186 await step.do(
187 "dynamic-delay-step",
188 {
189 retries: { limit: 3, delay: dynamicDelay, backoff: "constant" },
190 },
191 async () => {
192 attempts++;
193 if (attempts < 2) {
194 throw new Error("transient");
195 }
196 return "ok";
197 },
198 );
199 return "done";
200 });
201
202 await vi.waitUntil(
203 async () => {
204 const logs = (await engineStub.readLogs()) as EngineLogs;
205 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
206 },
207 { timeout: 5000 },
208 );
209
210 const logs = (await engineStub.readLogs()) as EngineLogs;
211 const attemptFailures = logs.logs.filter((val) => val.event === InstanceEvent.ATTEMPT_FAILURE);
212
213 expect(attemptFailures.length).toBeGreaterThanOrEqual(1);
214 expect(attemptFailures[0]?.metadata).toMatchObject({ retryDelayMs: 50 });
215 });
216
217 it("fails non-retryably when the delay function throws", async ({ expect }) => {
218 const instanceId = "DYNAMIC-DELAY-BROKEN";
219 const throwingDelay: WorkflowDelayFunction = () => {
220 throw new Error("delay boom");
221 };
222
223 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
224 await step.do(
225 "broken-delay-step",
226 {
227 retries: { limit: 5, delay: throwingDelay, backoff: "constant" },
228 },
229 async () => {
230 throw new Error("step failed");
231 },
232 );
233 return "done";
234 });
235
236 await vi.waitUntil(
237 async () => {
238 const logs = (await engineStub.readLogs()) as EngineLogs;
239 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE);
240 },
241 { timeout: 5000 },
242 );
243
244 const logs = (await engineStub.readLogs()) as EngineLogs;
245
246 // The broken delay function turns the retry into a non-retryable failure,
247 // so the step is only attempted once.
248 expect(logs.logs.filter((val) => val.event === InstanceEvent.ATTEMPT_START)).toHaveLength(1);
249
250 // Current workerd preserves the error message across the RPC boundary
251 // without folding the custom class name into it.
252 const workflowFailure = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_FAILURE);
253 expect(workflowFailure?.metadata.error.message).toContain("delay function");
254 expect(workflowFailure?.metadata.error.message).toContain("delay boom");
255 });
256
257 it("omits delay from ctx.config for a dynamic delay but includes it for a static delay", async ({
258 expect,
259 }) => {
260 const dynamicDelay: WorkflowDelayFunction = () => 10;
261 let dynamicConfig: WorkflowStepContext<WorkflowDelayFunction>["config"] | undefined;
262 let staticConfig: WorkflowStepContext<number>["config"] | undefined;
263
264 const dynamicStub = await runWorkflow("DYNAMIC-DELAY-CTX-HIDDEN", async (_event, step) => {
265 await step.do(
266 "dynamic-config-step",
267 {
268 retries: { limit: 1, delay: dynamicDelay, backoff: "constant" },
269 },
270 async (ctx) => {
271 dynamicConfig = ctx.config;
272 return "ok";
273 },
274 );
275 return "done";
276 });
277
278 await vi.waitUntil(
279 async () => {
280 const logs = (await dynamicStub.readLogs()) as EngineLogs;
281 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
282 },
283 { timeout: 5000 },
284 );
285
286 const staticStub = await runWorkflow("STATIC-DELAY-CTX-VISIBLE", async (_event, step) => {
287 await step.do(
288 "static-config-step",
289 {
290 retries: { limit: 1, delay: 1234, backoff: "constant" },
291 },
292 async (ctx) => {
293 staticConfig = ctx.config;
294 return "ok";
295 },
296 );
297 return "done";
298 });
299
300 await vi.waitUntil(
301 async () => {
302 const logs = (await staticStub.readLogs()) as EngineLogs;
303 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
304 },
305 { timeout: 5000 },
306 );
307
308 const dynamicRetries = dynamicConfig?.retries;
309 expect(dynamicRetries).toBeDefined();
310 expect(dynamicRetries && "delay" in dynamicRetries).toBe(false);
311 expect(staticConfig?.retries?.delay).toBe(1234);
312 });
313
314 it("waitForEvent should receive events while active", async ({ expect }) => {
315 const engineStub = await runWorkflow("MOCK-INSTANCE-ID-WAIT-FOR-EVENT", async (_, step) => {
316 return await step.waitForEvent("i'm a event!", {
317 type: "event-type-1",
318 timeout: "10 seconds",
319 });
320 });
321
322 await vi.waitUntil(
323 async () => {
324 const logs = (await engineStub.readLogs()) as EngineLogs;
325 return logs.logs.some((val) => val.event === InstanceEvent.WAIT_START);
326 },
327 { timeout: 5000 },
328 );
329
330 await engineStub.receiveEvent({
331 type: "event-type-1",
332 timestamp: new Date(),
333 payload: {},
334 });
335
336 await vi.waitUntil(
337 async () => {
338 const logs = (await engineStub.readLogs()) as EngineLogs;
339 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
340 },
341 { timeout: 5000 },
342 );
343
344 const logs = (await engineStub.readLogs()) as EngineLogs;
345 expect(logs.logs.some((v) => v.event === InstanceEvent.WAIT_START)).toBe(true);
346 expect(logs.logs.some((v) => v.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
347 });
348
349 it("waitForEvent should receive events even if not active", async ({ expect }) => {
350 const engineStub = await runWorkflow(
351 "MOCK-INSTANCE-ID-WAIT-FOR-EVENT-NOT-ACTIVE",
352 async (_, step) => {
353 return await step.waitForEvent("i'm a event!", {
354 type: "event-type-1",
355 timeout: "10 seconds",
356 });
357 },
358 );
359
360 await vi.waitUntil(
361 async () => {
362 const logs = (await engineStub.readLogs()) as EngineLogs;
363 return logs.logs.some((val) => val.event === InstanceEvent.WAIT_START);
364 },
365 { timeout: 5000 },
366 );
367
368 try {
369 await runInDurableObject(engineStub, async (engine) => {
370 await engine.abort("kabooom");
371 });
372 } catch {
373 // supposed to error out
374 }
375
376 // Get a new stub since we've just aborted the durable object
377 const newStub = env.ENGINE.get(
378 env.ENGINE.idFromName("MOCK-INSTANCE-ID-WAIT-FOR-EVENT-NOT-ACTIVE"),
379 );
380
381 await newStub.receiveEvent({
382 type: "event-type-1",
383 timestamp: new Date(),
384 payload: {},
385 });
386
387 await vi.waitUntil(
388 async () => {
389 const logs = (await newStub.readLogs()) as EngineLogs;
390 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
391 },
392 { timeout: 5000 },
393 );
394
395 const logs = (await newStub.readLogs()) as EngineLogs;
396 expect(logs.logs.some((v) => v.event === InstanceEvent.WAIT_START)).toBe(true);
397 expect(logs.logs.some((v) => v.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
398 });
399
400 it("waitForEvent should not deliver events to timed-out events with the same type", async ({
401 expect,
402 }) => {
403 const instanceId = "WAIT-FOR-EVENT-STALE-WAITER";
404 const engineStub = await runWorkflow(instanceId, async (_, step) => {
405 const results: Array<{
406 iteration: number;
407 received: boolean;
408 }> = [];
409
410 for (let i = 1; i <= 3; i++) {
411 try {
412 await step.waitForEvent(`my-event-waiter-${i}`, {
413 type: "my-event",
414 timeout: 500,
415 });
416 results.push({ iteration: i, received: true });
417 } catch {
418 results.push({ iteration: i, received: false });
419 }
420 }
421
422 return { results };
423 });
424
425 // 1st waitForEvent iteration - should receive event
426 await vi.waitUntil(
427 async () => {
428 const logs = (await engineStub.readLogs()) as EngineLogs;
429 return logs.logs.filter((val) => val.event === InstanceEvent.WAIT_START).length >= 1;
430 },
431 { timeout: 5000 },
432 );
433
434 await engineStub.receiveEvent({
435 type: "my-event",
436 timestamp: new Date(),
437 payload: { iteration: 1 },
438 });
439
440 await vi.waitUntil(
441 async () => {
442 const logs = (await engineStub.readLogs()) as EngineLogs;
443 return logs.logs.filter((val) => val.event === InstanceEvent.WAIT_START).length >= 2;
444 },
445 { timeout: 5000 },
446 );
447
448 // 2nd waitForEvent iteration - should timeout (500ms)
449 await vi.waitUntil(
450 async () => {
451 const logs = (await engineStub.readLogs()) as EngineLogs;
452 return logs.logs.some((val) => val.event === InstanceEvent.WAIT_TIMED_OUT);
453 },
454 { timeout: 5000 },
455 );
456
457 // 3rd waitForEvent iteration - should receive event
458 await vi.waitUntil(
459 async () => {
460 const logs = (await engineStub.readLogs()) as EngineLogs;
461 return logs.logs.filter((val) => val.event === InstanceEvent.WAIT_START).length >= 3;
462 },
463 { timeout: 5000 },
464 );
465
466 await engineStub.receiveEvent({
467 type: "my-event",
468 timestamp: new Date(),
469 payload: { iteration: 3 },
470 });
471
472 await vi.waitUntil(
473 async () => {
474 const logs = (await engineStub.readLogs()) as EngineLogs;
475 return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS);
476 },
477 { timeout: 5000 },
478 );
479
480 const logs = (await engineStub.readLogs()) as EngineLogs;
481 // Iterations 1 and 3 received events; iteration 2 timed out
482 expect(logs.logs.filter((val) => val.event === InstanceEvent.WAIT_COMPLETE)).toHaveLength(2);
483 expect(logs.logs.filter((val) => val.event === InstanceEvent.WAIT_TIMED_OUT)).toHaveLength(1);
484 });
485
486 it("should restore state from storage when accountId is undefined", async ({ expect }) => {
487 const instanceId = "RESTORE-TEST-INSTANCE";
488 const accountId = 12345;
489 const workflow: DatabaseWorkflow = {
490 name: "test-workflow",
491 id: "workflow-123",
492 created_on: new Date().toISOString(),
493 modified_on: new Date().toISOString(),
494 script_name: "test-script",
495 class_name: "TestWorkflow",
496 triggered_on: null,
497 };
498 const version: DatabaseVersion = {
499 id: "version-123",
500 class_name: "TestWorkflow",
501 created_on: new Date().toISOString(),
502 modified_on: new Date().toISOString(),
503 workflow_id: workflow.id,
504 mutable_pipeline_id: "pipeline-123",
505 };
506 const instance: DatabaseInstance = {
507 id: instanceId,
508 created_on: new Date().toISOString(),
509 modified_on: new Date().toISOString(),
510 workflow_id: workflow.id,
511 version_id: version.id,
512 status: InstanceStatus.Running,
513 started_on: new Date().toISOString(),
514 ended_on: null,
515 };
516 const event = {
517 payload: {},
518 timestamp: new Date(),
519 instanceId: instanceId,
520 workflowName: "",
521 };
522
523 const engineStub = await runWorkflow(instanceId, async () => {
524 return "test";
525 });
526
527 try {
528 await runInDurableObject(engineStub, async (engine) => {
529 await engine.init(accountId, workflow, version, instance, event);
530 await engine.setStatus(accountId, instanceId, InstanceStatus.Running);
531 await engine.abort(ABORT_REASONS.GRACE_PERIOD_COMPLETE);
532 });
533 } catch (e) {
534 // Expected - abort throws to break the DO
535 if (!isAbortError(e)) {
536 throw e;
537 }
538 }
539
540 const engineId = env.ENGINE.idFromName(instanceId);
541 const restartedStub = env.ENGINE.get(engineId);
542
543 const status = await runInDurableObject(restartedStub, (engine) => {
544 return engine.getStatus();
545 });
546
547 expect(status).toBe(InstanceStatus.Running);
548
549 const logs = (await restartedStub.readLogs()) as EngineLogs;
550 expect(logs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_START)).toBe(true);
551 });
552
553 it("should complete a step that returns a large Uint8Array without SQLITE_TOOBIG", async ({
554 expect,
555 }) => {
556 // Regression: JSON.stringify(Uint8Array) encodes each byte as a numeric key,
557 // producing a string far larger than byteLength. A 200 KB Uint8Array → ~2 MB JSON
558 // → SQLITE_TOOBIG. writeLog must use a replacer to sanitize TypedArrays.
559 const instanceId = "LARGE-UINT8ARRAY-RESULT";
560 const engineId = env.ENGINE.idFromName(instanceId);
561 const engineStub = env.ENGINE.get(engineId);
562
563 await runWorkflowAndAwait(instanceId, async (_event, step) => {
564 await step.do("large-binary-step", async () => {
565 return new Uint8Array(200_000); // ~200 KB, triggers SQLITE_TOOBIG without fix
566 });
567 });
568
569 const logs = (await engineStub.readLogs()) as EngineLogs;
570 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
571 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false);
572 });
573
574 it("should complete a step that returns an object containing a large Uint8Array without SQLITE_TOOBIG", async ({
575 expect,
576 }) => {
577 // Regression: nested TypedArrays inside objects also cause SQLITE_TOOBIG without
578 // the JSON.stringify replacer, since sanitization must be recursive.
579 const instanceId = "NESTED-UINT8ARRAY-RESULT";
580 const engineId = env.ENGINE.idFromName(instanceId);
581 const engineStub = env.ENGINE.get(engineId);
582
583 await runWorkflowAndAwait(instanceId, async (_event, step) => {
584 await step.do("nested-binary-step", async () => {
585 return { payload: new Uint8Array(200_000), label: "test" };
586 });
587 });
588
589 const logs = (await engineStub.readLogs()) as EngineLogs;
590 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
591 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false);
592 });
593
594 describe("step limits", () => {
595 it("should enforce step limit when exceeded", async ({ expect }) => {
596 const stepLimit = 3;
597 const instanceId = "STEP-LIMIT-EXCEEDED";
598 const engineId = env.ENGINE.idFromName(instanceId);
599 const engineStub = env.ENGINE.get(engineId);
600
601 await runInDurableObject(engineStub, (engine) => {
602 engine.stepLimit = stepLimit;
603 });
604
605 setTestWorkflowCallback(async (_event, step) => {
606 for (let i = 0; i < stepLimit + 1; i++) {
607 await step.do(`step-${i}`, async () => `result-${i}`);
608 }
609 });
610
611 await engineStub.init(
612 12346,
613 {} as DatabaseWorkflow,
614 {} as DatabaseVersion,
615 { id: instanceId } as DatabaseInstance,
616 { payload: {}, timestamp: new Date(), instanceId, workflowName: "" },
617 );
618
619 const logs = (await engineStub.readLogs()) as EngineLogs;
620
621 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(true);
622 });
623
624 it("should succeed when steps are exactly at the limit", async ({ expect }) => {
625 const stepLimit = 3;
626 const instanceId = "STEP-LIMIT-AT-LIMIT";
627 const engineId = env.ENGINE.idFromName(instanceId);
628 const engineStub = env.ENGINE.get(engineId);
629
630 await runInDurableObject(engineStub, (engine) => {
631 engine.stepLimit = stepLimit;
632 });
633
634 setTestWorkflowCallback(async (_event, step) => {
635 for (let i = 0; i < stepLimit; i++) {
636 await step.do(`step-${i}`, async () => `result-${i}`);
637 }
638 return "done";
639 });
640
641 await engineStub.init(
642 12346,
643 {} as DatabaseWorkflow,
644 {} as DatabaseVersion,
645 { id: instanceId } as DatabaseInstance,
646 { payload: {}, timestamp: new Date(), instanceId, workflowName: "" },
647 );
648
649 const logs = (await engineStub.readLogs()) as EngineLogs;
650
651 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
652 expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false);
653 });
654
655 it("should use DEFAULT_STEP_LIMIT when no limit is configured", async ({ expect }) => {
656 const engineId = env.ENGINE.idFromName("STEP-LIMIT-DEFAULT");
657 const freshStub = env.ENGINE.get(engineId);
658
659 const stepLimit = await runInDurableObject(freshStub, (engine) => engine.stepLimit);
660
661 expect(stepLimit).toBe(DEFAULT_STEP_LIMIT);
662 });
663 });
664
665 describe("step timeout timers", () => {
666 it(
667 "cancels completed step timeout timers so sequential steps stay under the active-timeout quota",
668 // Windows needs room for all 5,100 real callbacks; a timeout followed
669 // by replaying stored results previously hid incomplete attempts.
670 { timeout: 300_000 },
671 async (context) => {
672 const { expect } = context;
673 const { attempt, start, progress } = trackTimerQuotaTest(context);
674 // workerd's isolate allows 10,000 active timers. Each step.do() starts a
675 // 10-minute scheduler.wait() for the step timeout. If that wait is not
676 // cancelled when the step finishes, 5,000 sequential immediate steps hit
677 // QuotaExceededError (issue #15788).
678 // Vitest retries retain storage. A fresh ID ensures every attempt
679 // executes real callbacks instead of replaying earlier step results.
680 const steps = 5_100;
681 const engineStub = await runWorkflowAndAwait(
682 `STEP-TIMEOUT-TIMER-QUOTA-${attempt}`,
683 async (_event, step) => {
684 for (let i = 0; i < steps; i++) {
685 await step.do(
686 `n-${i}`,
687 {
688 retries: { limit: 0, delay: "0 seconds" },
689 timeout: "10 minutes",
690 },
691 async () => {
692 progress.completed++;
693 return i;
694 },
695 );
696 }
697 return { completed: steps };
698 },
699 );
700
701 progress.stepsMs = Math.round(performance.now() - start);
702 progress.phase = "terminal events";
703 // Avoid returning 25,000+ step events over RPC when only the terminal
704 // events are asserted. All real steps and storage operations still run.
705 const failure = (await engineStub.readLogsFromEvent(InstanceEvent.WORKFLOW_FAILURE))
706 .logs[0];
707 expect(failure, JSON.stringify(failure)).toBeUndefined();
708 const success = (await engineStub.readLogsFromEvent(InstanceEvent.WORKFLOW_SUCCESS))
709 .logs[0];
710 expect(success).toBeDefined();
711 expect(success?.metadata.result).toEqual({ completed: steps });
712 expect(progress.completed).toBe(steps);
713 progress.phase = "complete";
714 },
715 );
716
717 it(
718 "cancels failed step timeout timers so sequential try/caught steps stay under the active-timeout quota",
719 // Windows needs room for all 5,100 real callbacks; a timeout followed
720 // by replaying stored results previously hid incomplete attempts.
721 { timeout: 300_000 },
722 async (context) => {
723 const { expect } = context;
724 const { attempt, start, progress } = trackTimerQuotaTest(context);
725 // Same 10,000-timer quota as the success-path case. Failed steps used
726 // to leave scheduler.wait() running because the error handler never
727 // aborted stepExecutionSignal (issue #15788).
728 // Vitest retries retain storage. A fresh ID ensures every attempt
729 // executes real callbacks instead of replaying earlier step results.
730 const steps = 5_100;
731 const engineStub = await runWorkflowAndAwait(
732 `STEP-TIMEOUT-TIMER-QUOTA-FAILURES-${attempt}`,
733 async (_event, step) => {
734 for (let i = 0; i < steps; i++) {
735 try {
736 await step.do(
737 `n-${i}`,
738 {
739 retries: { limit: 0, delay: "0 seconds" },
740 timeout: "10 minutes",
741 },
742 async () => {
743 progress.completed++;
744 throw new Error("immediate step failure");
745 },
746 );
747 } catch (e) {
748 const msg = e instanceof Error ? e.message : String(e);
749 if (!msg.includes("immediate step failure")) {
750 throw e;
751 }
752 }
753 }
754 return { completed: steps };
755 },
756 );
757
758 progress.stepsMs = Math.round(performance.now() - start);
759 progress.phase = "terminal events";
760 // Avoid returning 25,000+ step events over RPC when only the terminal
761 // events are asserted. All real steps and storage operations still run.
762 const failure = (await engineStub.readLogsFromEvent(InstanceEvent.WORKFLOW_FAILURE))
763 .logs[0];
764 expect(failure, JSON.stringify(failure)).toBeUndefined();
765 const success = (await engineStub.readLogsFromEvent(InstanceEvent.WORKFLOW_SUCCESS))
766 .logs[0];
767 expect(success).toBeDefined();
768 expect(success?.metadata.result).toEqual({ completed: steps });
769 expect(progress.completed).toBe(steps);
770 progress.phase = "complete";
771 },
772 );
773 });
774
775 describe("lifecycle methods", () => {
776 it.for([InstanceStatus.Complete, InstanceStatus.Errored, InstanceStatus.Terminated])(
777 "should throw when calling terminate on instance in finite state: %s",
778 async (finiteStatus, { expect }) => {
779 const engineStub = await runWorkflowAndAwait(
780 `TERMINATE-${finiteStatus}-INSTANCE`,
781 async () => "done",
782 );
783
784 // If not Complete, manually set the status
785 if (finiteStatus !== InstanceStatus.Complete) {
786 await runInDurableObject(engineStub, async (_engine, state) => {
787 await state.storage.put("ENGINE_STATUS", finiteStatus);
788 });
789 }
790
791 await expect(
792 runInDurableObject(engineStub, async (engine) => {
793 await engine.changeInstanceStatus("terminate");
794 }),
795 ).rejects.toThrow("Cannot terminate instance since its on a finite state");
796 },
797 );
798
799 it.for([
800 InstanceStatus.Complete,
801 InstanceStatus.Errored,
802 InstanceStatus.Terminated,
803 InstanceStatus.Running,
804 InstanceStatus.Paused,
805 ])("should restart workflow from status: %s", async (initialStatus, { expect }) => {
806 const instanceId = `RESTART-${initialStatus}-INSTANCE`;
807 const engineId = env.ENGINE.idFromName(instanceId);
808
809 const engineStub = await runWorkflowAndAwait(
810 instanceId,
811 async (_event: unknown, step: WorkflowStep) => {
812 await step.do("test-step", async () => "step-result");
813 return "done";
814 },
815 );
816
817 // Set the status to initialStatus
818 await runInDurableObject(engineStub, async (_engine, state) => {
819 await state.storage.put("ENGINE_STATUS", initialStatus);
820 });
821
822 try {
823 await runInDurableObject(engineStub, async (engine) => {
824 await engine.changeInstanceStatus("restart");
825 });
826 } catch (e) {
827 // Expected - abort throws to break the DO
828 if (!isAbortError(e)) {
829 throw e;
830 }
831 }
832
833 const restartedStub = env.ENGINE.get(engineId);
834
835 await runInDurableObject(restartedStub, async (engine) => {
836 await engine.attemptRestart();
837 });
838
839 await vi.waitUntil(
840 async () => {
841 const status = await runInDurableObject(restartedStub, (engine) => engine.getStatus());
842 return status === InstanceStatus.Complete;
843 },
844 { timeout: 5000 },
845 );
846
847 // Verify the workflow ran again by checking logs
848 const logs = (await restartedStub.readLogs()) as EngineLogs;
849
850 expect(logs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_START)).toBe(true);
851
852 expect(logs.logs.some((log) => log.event === InstanceEvent.STEP_START)).toBe(true);
853
854 expect(logs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
855 });
856
857 it("should restart from a specific step and preserve earlier results", async ({ expect }) => {
858 const instanceId = "RESTART-FROM-STEP";
859 const engineId = env.ENGINE.idFromName(instanceId);
860
861 const engineStub = await runWorkflowAndAwait(
862 instanceId,
863 async (_event: unknown, step: WorkflowStep) => {
864 const a = await step.do("step-a", async () => "result-a");
865 const b = await step.do("step-b", async () => "result-b");
866 return { a, b };
867 },
868 );
869
870 const statusBefore = await runInDurableObject(engineStub, (engine) => engine.getStatus());
871 expect(statusBefore).toBe(InstanceStatus.Complete);
872
873 try {
874 await runInDurableObject(engineStub, async (engine) => {
875 await engine.changeInstanceStatus("restart", {
876 name: "step-b",
877 });
878 });
879 } catch (e) {
880 if (!isAbortError(e)) {
881 throw e;
882 }
883 }
884
885 const restartedStub = env.ENGINE.get(engineId);
886
887 await runInDurableObject(restartedStub, async (engine) => {
888 await engine.attemptRestart();
889 });
890
891 await vi.waitUntil(
892 async () => {
893 const status = await runInDurableObject(restartedStub, (engine) => engine.getStatus());
894 return status === InstanceStatus.Complete;
895 },
896 { timeout: 5000 },
897 );
898
899 const logs = (await restartedStub.readLogs()) as EngineLogs;
900
901 // step-a should still have its cached result from the first run
902 const stepSuccessLogs = logs.logs.filter((log) => log.event === InstanceEvent.STEP_SUCCESS);
903 expect(stepSuccessLogs.length).toBeGreaterThanOrEqual(2);
904
905 expect(logs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true);
906 });
907
908 it("should default restart from step type to step.do", async ({ expect }) => {
909 const instanceId = "RESTART-FROM-STEP-DEFAULT-DO";
910 const engineId = env.ENGINE.idFromName(instanceId);
911
912 const engineStub = await runWorkflowAndAwait(
913 instanceId,
914 async (_event: unknown, step: WorkflowStep) => {
915 const setup = await step.do("setup", async () => crypto.randomUUID());
916 await step.sleep("checkpoint", 1);
917 const between = await step.do("between", async () => crypto.randomUUID());
918 const checkpoint = await step.do("checkpoint", async () => crypto.randomUUID());
919 const after = await step.do("after", async () => crypto.randomUUID());
920 return { setup, between, checkpoint, after };
921 },
922 );
923
924 const logsBefore = (await engineStub.readLogs()) as EngineLogs;
925 const stepResultsBefore = logsBefore.logs
926 .filter((log) => log.event === InstanceEvent.STEP_SUCCESS)
927 .map((log) => log.metadata.result);
928
929 try {
930 await runInDurableObject(engineStub, async (engine) => {
931 await engine.changeInstanceStatus("restart", {
932 name: "checkpoint",
933 });
934 });
935 } catch (e) {
936 if (!isAbortError(e)) {
937 throw e;
938 }
939 }
940
941 const restartedStub = env.ENGINE.get(engineId);
942
943 await runInDurableObject(restartedStub, async (engine) => {
944 await engine.attemptRestart();
945 });
946
947 await vi.waitUntil(
948 async () => {
949 const status = await runInDurableObject(restartedStub, (engine) => engine.getStatus());
950 return status === InstanceStatus.Complete;
951 },
952 { timeout: 5000 },
953 );
954
955 const logsAfter = (await restartedStub.readLogs()) as EngineLogs;
956 const stepResultsAfter = logsAfter.logs
957 .filter((log) => log.event === InstanceEvent.STEP_SUCCESS)
958 .map((log) => log.metadata.result);
959
960 expect(stepResultsAfter[0]).toEqual(stepResultsBefore[0]);
961 expect(stepResultsAfter[1]).toEqual(stepResultsBefore[1]);
962 expect(stepResultsAfter[2]).not.toEqual(stepResultsBefore[2]);
963 expect(stepResultsAfter[3]).not.toEqual(stepResultsBefore[3]);
964 });
965
966 it("should throw when restarting from a non-existing step", async ({ expect }) => {
967 const instanceId = "RESTART-FROM-BAD-STEP";
968
969 const engineStub = await runWorkflowAndAwait(
970 instanceId,
971 async (_event: unknown, step: WorkflowStep) => {
972 await step.do("step-a", async () => "result-a");
973 return "done";
974 },
975 );
976
977 await expect(
978 runInDurableObject(engineStub, async (engine) => {
979 await engine.changeInstanceStatus("restart", {
980 name: "nonexistent-step",
981 });
982 }),
983 ).rejects.toThrow("not found in execution history");
984 });
985
986 it("should pause after in-flight step.do finishes", async ({ expect }) => {
987 const instanceId = "PAUSE-AFTER-DO";
988 const engineId = env.ENGINE.idFromName(instanceId);
989
990 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
991 await step.do("long-step", async () => {
992 await scheduler.wait(500);
993 return "first";
994 });
995 // step-2 should never run because pause takes effect after long-step
996 await step.do("step-2", async () => "second");
997 return "done";
998 });
999
1000 // Wait for long-step to start
1001 await vi.waitUntil(
1002 async () => {
1003 return await runInDurableObject(engineStub, (engine) => {
1004 const logs = engine.readLogs() as unknown as EngineLogs;
1005 return logs.logs.some((log) => log.event === InstanceEvent.STEP_START);
1006 });
1007 },
1008 { timeout: 5000 },
1009 );
1010
1011 // Request pause while long-step is in flight
1012 try {
1013 await runInDurableObject(engineStub, async (engine) => {
1014 await engine.changeInstanceStatus("pause");
1015 });
1016 } catch (e) {
1017 if (!isAbortError(e)) {
1018 throw e;
1019 }
1020 }
1021
1022 await vi.waitUntil(
1023 async () => {
1024 try {
1025 return await runInDurableObject(
1026 env.ENGINE.get(engineId),
1027 async (engine) => (await engine.getStatus()) === InstanceStatus.Paused,
1028 );
1029 } catch (e) {
1030 if (isAbortError(e)) {
1031 return false;
1032 }
1033 throw e;
1034 }
1035 },
1036 { timeout: 5000 },
1037 );
1038
1039 const freshStub = env.ENGINE.get(engineId);
1040 const finalStatus = await runInDurableObject(freshStub, (engine) => engine.getStatus());
1041 expect(finalStatus).toBe(InstanceStatus.Paused);
1042
1043 // Verify long-step completed but step-2 never ran
1044 const logs = await runInDurableObject(freshStub, (engine) => {
1045 return engine.readLogs() as unknown as EngineLogs;
1046 });
1047 const stepSuccesses = logs.logs.filter((log) => log.event === InstanceEvent.STEP_SUCCESS);
1048 expect(stepSuccesses).toHaveLength(1);
1049 });
1050
1051 it("should pause after multiple concurrent in-flight step.dos finish", async ({ expect }) => {
1052 const instanceId = "PAUSE-AFTER-CONCURRENT-DOS";
1053 const engineId = env.ENGINE.idFromName(instanceId);
1054
1055 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
1056 const [resultA, resultB] = await Promise.all([
1057 step.do("slow-step-a", async () => {
1058 await scheduler.wait(500);
1059 return "a-done";
1060 }),
1061 step.do("slow-step-b", async () => {
1062 await scheduler.wait(500);
1063 return "b-done";
1064 }),
1065 ]);
1066
1067 // This step should never run
1068 await step.do("step-after-pause", async () => "should-not-run");
1069 return { resultA, resultB };
1070 });
1071
1072 await vi.waitUntil(
1073 async () => {
1074 return await runInDurableObject(engineStub, (engine) => {
1075 const logs = engine.readLogs() as unknown as EngineLogs;
1076 return logs.logs.filter((log) => log.event === InstanceEvent.STEP_START).length >= 2;
1077 });
1078 },
1079 { timeout: 5000 },
1080 );
1081
1082 // Request pause while both slow steps are in flight
1083 try {
1084 await runInDurableObject(engineStub, async (engine) => {
1085 await engine.changeInstanceStatus("pause");
1086 });
1087 } catch (e) {
1088 if (!isAbortError(e)) {
1089 throw e;
1090 }
1091 }
1092
1093 await vi.waitUntil(
1094 async () => {
1095 try {
1096 return await runInDurableObject(
1097 env.ENGINE.get(engineId),
1098 async (engine) => (await engine.getStatus()) === InstanceStatus.Paused,
1099 );
1100 } catch (e) {
1101 if (isAbortError(e)) {
1102 return false;
1103 }
1104 throw e;
1105 }
1106 },
1107 { timeout: 5000 },
1108 );
1109
1110 const freshStub = env.ENGINE.get(engineId);
1111 const finalStatus = await runInDurableObject(freshStub, (engine) => engine.getStatus());
1112 expect(finalStatus).toBe(InstanceStatus.Paused);
1113
1114 // Both concurrent steps should have completed, but step-after-pause should not
1115 const logs = await runInDurableObject(freshStub, (engine) => {
1116 return engine.readLogs() as unknown as EngineLogs;
1117 });
1118 const stepSuccesses = logs.logs.filter((log) => log.event === InstanceEvent.STEP_SUCCESS);
1119 expect(stepSuccesses).toHaveLength(2);
1120 });
1121
1122 it("should unblock concurrent steps blocked when resume cancels pending pause", async ({
1123 expect,
1124 }) => {
1125 const instanceId = "RESUME-UNBLOCKS-WAITING-STEPS";
1126 const engineId = env.ENGINE.idFromName(instanceId);
1127
1128 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
1129 const [slowResult, fastResult] = await Promise.all([
1130 step.do("slow-step", async () => {
1131 await scheduler.wait(1000);
1132 return "slow-done";
1133 }),
1134 step.do("fast-step", async () => {
1135 return "fast-done";
1136 }),
1137 ]);
1138
1139 await step.do("step-after-resume", async () => "after-resume");
1140 return { slowResult, fastResult };
1141 });
1142
1143 await vi.waitUntil(
1144 async () => {
1145 return await runInDurableObject(engineStub, (engine) => {
1146 const logs = engine.readLogs() as unknown as EngineLogs;
1147 return logs.logs.some((log) => log.event === InstanceEvent.STEP_START);
1148 });
1149 },
1150 { timeout: 5000 },
1151 );
1152
1153 await runInDurableObject(engineStub, async (engine) => {
1154 await engine.changeInstanceStatus("pause");
1155 });
1156
1157 await vi.waitUntil(
1158 async () =>
1159 runInDurableObject(
1160 env.ENGINE.get(engineId),
1161 async (engine) => (await engine.getStatus()) === InstanceStatus.WaitingForPause,
1162 ),
1163 { timeout: 5000 },
1164 );
1165
1166 // Resume before slow-step finishes
1167 await runInDurableObject(engineStub, async (engine) => {
1168 await engine.changeInstanceStatus("resume");
1169 });
1170
1171 // Verify status goes back to Running
1172 const statusAfterResume = await runInDurableObject(env.ENGINE.get(engineId), (engine) =>
1173 engine.getStatus(),
1174 );
1175 expect(statusAfterResume).toBe(InstanceStatus.Running);
1176
1177 await vi.waitUntil(
1178 async () =>
1179 runInDurableObject(
1180 env.ENGINE.get(engineId),
1181 async (engine) => (await engine.getStatus()) === InstanceStatus.Complete,
1182 ),
1183 { timeout: 5000 },
1184 );
1185
1186 const freshStub = env.ENGINE.get(engineId);
1187 const finalStatus = await runInDurableObject(freshStub, (engine) => engine.getStatus());
1188 expect(finalStatus).toBe(InstanceStatus.Complete);
1189
1190 // All three steps should have completed
1191 const logs = await runInDurableObject(freshStub, (engine) => {
1192 return engine.readLogs() as unknown as EngineLogs;
1193 });
1194 const stepSuccesses = logs.logs.filter((log) => log.event === InstanceEvent.STEP_SUCCESS);
1195 expect(stepSuccesses).toHaveLength(3);
1196 });
1197
1198 it("should pause during a step.sleep, then resume and complete", async ({ expect }) => {
1199 const instanceId = "PAUSE-DURING-SLEEP-RESUME";
1200 const engineId = env.ENGINE.idFromName(instanceId);
1201
1202 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
1203 await step.do("first-step", async () => "first-done");
1204
1205 await step.sleep("short-sleep", "1 second");
1206
1207 await step.do("after-sleep", async () => "after-sleep-done");
1208 return "done";
1209 });
1210
1211 // Wait for the first step to complete (workflow is now in the sleep)
1212 await vi.waitUntil(
1213 async () => {
1214 try {
1215 return await runInDurableObject(engineStub, (engine) => {
1216 const logs = engine.readLogs() as unknown as EngineLogs;
1217 return logs.logs.some((log) => log.event === InstanceEvent.STEP_SUCCESS);
1218 });
1219 } catch (e) {
1220 if (isAbortError(e)) {
1221 return false;
1222 }
1223 throw e;
1224 }
1225 },
1226 { timeout: 5000 },
1227 );
1228
1229 // Request pause while in step.sleep — should pause immediately
1230 try {
1231 await runInDurableObject(engineStub, async (engine) => {
1232 await engine.changeInstanceStatus("pause");
1233 });
1234 } catch (e) {
1235 if (!isAbortError(e)) {
1236 throw e;
1237 }
1238 }
1239
1240 await vi.waitUntil(
1241 async () => {
1242 try {
1243 return await runInDurableObject(
1244 env.ENGINE.get(engineId),
1245 async (engine) => (await engine.getStatus()) === InstanceStatus.Paused,
1246 );
1247 } catch (e) {
1248 if (isAbortError(e)) {
1249 return false;
1250 }
1251 throw e;
1252 }
1253 },
1254 { timeout: 5000 },
1255 );
1256
1257 expect(
1258 await runInDurableObject(env.ENGINE.get(engineId), (engine) => engine.getStatus()),
1259 ).toBe(InstanceStatus.Paused);
1260
1261 // Only the first step should have succeeded — sleep was interrupted
1262 const logsBeforeResume = await runInDurableObject(
1263 env.ENGINE.get(engineId),
1264 (engine) => engine.readLogs() as unknown as EngineLogs,
1265 );
1266 expect(
1267 logsBeforeResume.logs.filter((log) => log.event === InstanceEvent.STEP_SUCCESS),
1268 ).toHaveLength(1);
1269
1270 // Resume the workflow
1271 try {
1272 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
1273 await engine.changeInstanceStatus("resume");
1274 });
1275 } catch (e) {
1276 if (!isAbortError(e)) {
1277 throw e;
1278 }
1279 }
1280
1281 // Wait for the workflow to complete — sleep should finish and after-sleep step should run
1282 await vi.waitUntil(
1283 async () => {
1284 try {
1285 return await runInDurableObject(
1286 env.ENGINE.get(engineId),
1287 async (engine) => (await engine.getStatus()) === InstanceStatus.Complete,
1288 );
1289 } catch (e) {
1290 if (isAbortError(e)) {
1291 return false;
1292 }
1293 throw e;
1294 }
1295 },
1296 { timeout: 5000 },
1297 );
1298
1299 const finalLogs = await runInDurableObject(
1300 env.ENGINE.get(engineId),
1301 (engine) => engine.readLogs() as unknown as EngineLogs,
1302 );
1303 const stepSuccesses = finalLogs.logs.filter(
1304 (log) => log.event === InstanceEvent.STEP_SUCCESS,
1305 );
1306 // first-step + after-sleep = 2 step successes
1307 expect(stepSuccesses).toHaveLength(2);
1308 });
1309
1310 it("should pause during a waitForEvent, then resume and complete", async ({ expect }) => {
1311 const instanceId = "PAUSE-DURING-WAIT-FOR-EVENT";
1312 const engineId = env.ENGINE.idFromName(instanceId);
1313
1314 const engineStub = await runWorkflow(instanceId, async (_event, step) => {
1315 await step.do("first-step", async () => "first-done");
1316
1317 await step.waitForEvent("wait-for-signal", {
1318 type: "continue",
1319 timeout: "30 seconds",
1320 });
1321
1322 await step.do("after-event", async () => "after-event-done");
1323 return "done";
1324 });
1325
1326 // Wait for the first step to complete (workflow is now waiting for event)
1327 await vi.waitUntil(
1328 async () => {
1329 try {
1330 return await runInDurableObject(engineStub, (engine) => {
1331 const logs = engine.readLogs() as unknown as EngineLogs;
1332 return logs.logs.some((log) => log.event === InstanceEvent.WAIT_START);
1333 });
1334 } catch (e) {
1335 if (isAbortError(e)) {
1336 return false;
1337 }
1338 throw e;
1339 }
1340 },
1341 { timeout: 5000 },
1342 );
1343
1344 // Request pause while in waitForEvent
1345 try {
1346 await runInDurableObject(engineStub, async (engine) => {
1347 await engine.changeInstanceStatus("pause");
1348 });
1349 } catch (e) {
1350 if (!isAbortError(e)) {
1351 throw e;
1352 }
1353 }
1354
1355 await vi.waitUntil(
1356 async () => {
1357 try {
1358 return await runInDurableObject(
1359 env.ENGINE.get(engineId),
1360 async (engine) => (await engine.getStatus()) === InstanceStatus.Paused,
1361 );
1362 } catch (e) {
1363 if (isAbortError(e)) {
1364 return false;
1365 }
1366 throw e;
1367 }
1368 },
1369 { timeout: 5000 },
1370 );
1371
1372 expect(
1373 await runInDurableObject(env.ENGINE.get(engineId), (engine) => engine.getStatus()),
1374 ).toBe(InstanceStatus.Paused);
1375
1376 // Resume the workflow
1377 try {
1378 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
1379 await engine.changeInstanceStatus("resume");
1380 });
1381 } catch (e) {
1382 if (!isAbortError(e)) {
1383 throw e;
1384 }
1385 }
1386
1387 // Send the event after resume
1388 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
1389 await engine.receiveEvent({
1390 timestamp: new Date(),
1391 payload: { done: true },
1392 type: "continue",
1393 });
1394 });
1395
1396 // Wait for the workflow to complete
1397 await vi.waitUntil(
1398 async () => {
1399 try {
1400 return await runInDurableObject(
1401 env.ENGINE.get(engineId),
1402 async (engine) => (await engine.getStatus()) === InstanceStatus.Complete,
1403 );
1404 } catch (e) {
1405 if (isAbortError(e)) {
1406 return false;
1407 }
1408 throw e;
1409 }
1410 },
1411 { timeout: 5000 },
1412 );
1413
1414 const finalLogs = await runInDurableObject(
1415 env.ENGINE.get(engineId),
1416 (engine) => engine.readLogs() as unknown as EngineLogs,
1417 );
1418 const stepSuccesses = finalLogs.logs.filter(
1419 (log) => log.event === InstanceEvent.STEP_SUCCESS,
1420 );
1421 // first-step + after-event = 2 step successes
1422 expect(stepSuccesses).toHaveLength(2);
1423 });
1424
1425 it("should transition WaitingForPause to Paused on init() entry", async ({ expect }) => {
1426 const instanceId = "WAITING-FOR-PAUSE-INIT";
1427
1428 const engineStub = await runWorkflowAndAwait(instanceId, async () => "done");
1429
1430 // Manually set status to WaitingForPause (simulating a DO restart scenario)
1431 await runInDurableObject(engineStub, async (_engine, state) => {
1432 await state.storage.put("ENGINE_STATUS", InstanceStatus.WaitingForPause);
1433 });
1434
1435 // Now call init() — it should detect WaitingForPause and transition to Paused
1436 await runInDurableObject(engineStub, async (engine) => {
1437 // Reset isRunning so init() doesn't short-circuit
1438 engine.isRunning = false;
1439 await engine.init(
1440 12346,
1441 {} as DatabaseWorkflow,
1442 {} as DatabaseVersion,
1443 { id: instanceId } as DatabaseInstance,
1444 {
1445 payload: {},
1446 timestamp: new Date(),
1447 instanceId,
1448 workflowName: "",
1449 },
1450 );
1451 });
1452
1453 const status = await runInDurableObject(engineStub, (engine) => engine.getStatus());
1454 expect(status).toBe(InstanceStatus.Paused);
1455 });
1456 });
1457 });
1458
1459 // RPC-stubbed rollback fns can't reliably mutate test-scope closures —
1460 // assert via engine log events instead.
1461 describe("Rollback", () => {
1462 async function readLogsAfter(
1463 stub: { readLogs(): Promise<EngineLogs> | EngineLogs },
1464 predicate: (logs: EngineLogs) => boolean,
1465 timeout = 5000,
1466 ): Promise<EngineLogs> {
1467 await vi.waitUntil(async () => predicate((await stub.readLogs()) as EngineLogs), { timeout });
1468 return (await stub.readLogs()) as EngineLogs;
1469 }
1470
1471 function targetsOf(logs: EngineLogs, event: InstanceEvent): Array<string | null> {
1472 return logs.logs.filter((l) => l.event === event).map((l) => l.target);
1473 }
1474
1475 function countOf(logs: EngineLogs, event: InstanceEvent): number {
1476 return logs.logs.filter((l) => l.event === event).length;
1477 }
1478
1479 function doWithRollback<T>(
1480 step: WorkflowStep,
1481 name: string,
1482 callback: (ctx: unknown) => Promise<T>,
1483 options: WorkflowStepRollbackOptions,
1484 ): Promise<unknown>;
1485 function doWithRollback<T>(
1486 step: WorkflowStep,
1487 name: string,
1488 config: WorkflowStepConfig,
1489 callback: (ctx: unknown) => Promise<T>,
1490 options: WorkflowStepRollbackOptions,
1491 ): Promise<unknown>;
1492 function doWithRollback<T>(
1493 step: WorkflowStep,
1494 name: string,
1495 configOrCallback: WorkflowStepConfig | ((ctx: unknown) => Promise<T>),
1496 callbackOrOptions: ((ctx: unknown) => Promise<T>) | WorkflowStepRollbackOptions,
1497 options?: WorkflowStepRollbackOptions,
1498 ): Promise<unknown> {
1499 if (typeof configOrCallback === "function") {
1500 // @ts-expect-error -- rollback options are not in workers-types yet
1501 return step.do(name, configOrCallback, callbackOrOptions);
1502 }
1503 // @ts-expect-error -- rollback options are not in workers-types yet
1504 return step.do(name, configOrCallback, callbackOrOptions, options);
1505 }
1506
1507 async function noopRollback(_ctx: RollbackContext): Promise<void> {}
1508
1509 function rollbackOptions(fn: RollbackFn = noopRollback): WorkflowStepRollbackOptions {
1510 return { rollback: fn };
1511 }
1512
1513 function rollbackOptionsWithConfig(
1514 rollbackConfig: WorkflowStepConfig,
1515 fn: RollbackFn = noopRollback,
1516 ): WorkflowStepRollbackOptions {
1517 return { rollback: fn, rollbackConfig };
1518 }
1519
1520 it("runs rollback fns in LIFO order on workflow failure", async ({ expect }) => {
1521 const stub = await runWorkflowAndAwait("RB-LIFO", async (_e, step) => {
1522 await doWithRollback(step, "step-1", async () => "out-1", rollbackOptions());
1523 await doWithRollback(step, "step-2", async () => "out-2", rollbackOptions());
1524 await doWithRollback(step, "step-3", async () => "out-3", rollbackOptions());
1525 throw new Error("boom");
1526 });
1527 const logs = await readLogsAfter(stub, (l) =>
1528 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1529 );
1530 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual([
1531 "step-3-1",
1532 "step-2-1",
1533 "step-1-1",
1534 ]);
1535 expect(targetsOf(logs, InstanceEvent.ROLLBACK_ATTEMPT_SUCCESS)).toEqual([
1536 "step-3-1",
1537 "step-2-1",
1538 "step-1-1",
1539 ]);
1540 expect(countOf(logs, InstanceEvent.ROLLBACK_FAILED)).toBe(0);
1541 });
1542
1543 it("runs parallel rollbacks in reverse step start order", async ({ expect }) => {
1544 let firstStarted: () => void = () => {};
1545 let firstMayFinish: () => void = () => {};
1546 const firstStartedPromise = new Promise<void>((resolve) => {
1547 firstStarted = resolve;
1548 });
1549 const firstMayFinishPromise = new Promise<void>((resolve) => {
1550 firstMayFinish = resolve;
1551 });
1552
1553 const stub = await runWorkflowAndAwait("RB-PARALLEL", async (_e, step) => {
1554 const first = doWithRollback(
1555 step,
1556 "first",
1557 async () => {
1558 firstStarted();
1559 await firstMayFinishPromise;
1560 await scheduler.wait(50);
1561 return "out-1";
1562 },
1563 rollbackOptions(),
1564 );
1565 await firstStartedPromise;
1566
1567 const second = doWithRollback(
1568 step,
1569 "second",
1570 async () => {
1571 firstMayFinish();
1572 return "out-2";
1573 },
1574 rollbackOptions(),
1575 );
1576
1577 await Promise.all([first, second]);
1578 throw new Error("boom");
1579 });
1580 const logs = await readLogsAfter(stub, (l) =>
1581 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1582 );
1583 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["second-1", "first-1"]);
1584 });
1585
1586 it("uses rollbackConfig when executing rollback", async ({ expect }) => {
1587 const rollbackConfig = {
1588 retries: { limit: 0, delay: 0, backoff: "constant" },
1589 timeout: "30 seconds",
1590 } satisfies WorkflowStepConfig;
1591 const stub = await runWorkflowAndAwait("RB-CONFIG", async (_e, step) => {
1592 await doWithRollback(
1593 step,
1594 "configured-step",
1595 async () => "out",
1596 rollbackOptionsWithConfig(rollbackConfig),
1597 );
1598 throw new Error("boom");
1599 });
1600 const logs = await readLogsAfter(stub, (l) =>
1601 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1602 );
1603 expect(
1604 logs.logs.find((l) => l.event === InstanceEvent.ROLLBACK_STEP_START)?.metadata,
1605 ).toMatchObject({ config: rollbackConfig });
1606 });
1607
1608 it("passes rollback context to rollback fn", async ({ expect }) => {
1609 const stub = await runWorkflowAndAwait("RB-CONTEXT", async (_e, step) => {
1610 await doWithRollback(
1611 step,
1612 "ctx-step",
1613 async () => "out",
1614 rollbackOptions(async (ctx) => {
1615 if (
1616 ctx.error.name !== "Error" ||
1617 ctx.error.message !== "boom" ||
1618 ctx.output !== "out" ||
1619 ctx.ctx.step.name !== "ctx-step" ||
1620 ctx.ctx.step.count !== 1 ||
1621 `${ctx.ctx.step.name}-${ctx.ctx.step.count}` !== "ctx-step-1"
1622 ) {
1623 throw new Error("unexpected rollback context");
1624 }
1625 }),
1626 );
1627 throw new Error("boom");
1628 });
1629 const logs = await readLogsAfter(stub, (l) =>
1630 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1631 );
1632 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["ctx-step-1"]);
1633 });
1634
1635 it("runs rollback for failed step", async ({ expect }) => {
1636 const stub = await runWorkflowAndAwait("RB-FAILED-STEP", async (_e, step) => {
1637 await doWithRollback(
1638 step,
1639 "failed-step",
1640 { retries: { limit: 0, delay: 0, backoff: "constant" } },
1641 async () => {
1642 throw new Error("step-boom");
1643 },
1644 rollbackOptions(async (ctx) => {
1645 if (
1646 ctx.error.message !== "step-boom" ||
1647 ctx.output !== undefined ||
1648 ctx.ctx.step.name !== "failed-step" ||
1649 ctx.ctx.step.count !== 1 ||
1650 `${ctx.ctx.step.name}-${ctx.ctx.step.count}` !== "failed-step-1"
1651 ) {
1652 throw new Error("unexpected failed-step rollback context");
1653 }
1654 }),
1655 );
1656 });
1657 const logs = await readLogsAfter(stub, (l) =>
1658 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1659 );
1660 const rollbackStepTargets = targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS);
1661 const stepFailureTargets = targetsOf(logs, InstanceEvent.STEP_FAILURE);
1662
1663 expect(rollbackStepTargets).toEqual(["failed-step-1"]);
1664 expect(stepFailureTargets).toContain("failed-step-1");
1665 });
1666
1667 it("only runs rollbacks for steps with a registered fn", async ({ expect }) => {
1668 const stub = await runWorkflowAndAwait("RB-PARTIAL", async (_e, step) => {
1669 await step.do("plain-step", async () => "v1");
1670 await doWithRollback(step, "step-with-rollback", async () => "v2", rollbackOptions());
1671 await step.do("plain-step-after", async () => "v3");
1672 throw new Error("boom");
1673 });
1674 const logs = await readLogsAfter(stub, (l) =>
1675 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_COMPLETE),
1676 );
1677 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["step-with-rollback-1"]);
1678 expect(logs.logs.find((l) => l.event === InstanceEvent.ROLLBACK_START)?.metadata).toMatchObject(
1679 { totalSteps: 1 },
1680 );
1681 });
1682
1683 it("stops at the first failing rollback and logs ROLLBACK_FAILED", async ({ expect }) => {
1684 const stub = await runWorkflowAndAwait("RB-FAILS", async (_e, step) => {
1685 await doWithRollback(step, "step-1", async () => "v1", rollbackOptions());
1686 await doWithRollback(
1687 step,
1688 "step-2",
1689 async () => "v2",
1690 rollbackOptionsWithConfig(
1691 { retries: { limit: 0, delay: 0, backoff: "constant" } },
1692 async () => {
1693 throw new Error("rollback-boom");
1694 },
1695 ),
1696 );
1697 await doWithRollback(step, "step-3", async () => "v3", rollbackOptions());
1698 throw new Error("boom");
1699 });
1700 const logs = await readLogsAfter(stub, (l) =>
1701 l.logs.some((r) => r.event === InstanceEvent.ROLLBACK_FAILED),
1702 );
1703 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["step-3-1"]);
1704 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_FAILURE)).toEqual(["step-2-1"]);
1705 expect(targetsOf(logs, InstanceEvent.ROLLBACK_ATTEMPT_FAILURE)).toEqual(["step-2-1"]);
1706 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(0);
1707 });
1708
1709 it("runs rollback when terminate requests it", async ({ expect }) => {
1710 const instanceId = "RB-TERMINATE";
1711 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1712 await doWithRollback(step, "setup-resource", async () => "resource-id", rollbackOptions());
1713 await step.sleep("wait-forever", "1 hour");
1714 });
1715
1716 await readLogsAfter(engineStub, (currentLogs) =>
1717 currentLogs.logs.some(
1718 (log) => log.event === InstanceEvent.STEP_SUCCESS && log.target === "setup-resource-1",
1719 ),
1720 );
1721
1722 try {
1723 await runInDurableObject(engineStub, async (engine) => {
1724 await engine.changeInstanceStatus("terminate", undefined, {
1725 rollback: true,
1726 });
1727 });
1728 } catch (error) {
1729 if (!(error instanceof Error) || !error.message.startsWith("Aborting engine:")) {
1730 throw error;
1731 }
1732 }
1733
1734 const logs = await readLogsAfter(
1735 env.ENGINE.get(env.ENGINE.idFromName(instanceId)),
1736 (currentLogs) =>
1737 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
1738 );
1739 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["setup-resource-1"]);
1740 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
1741 });
1742
1743 it("replays cached steps before terminate rollback when registry is empty", async ({
1744 expect,
1745 }) => {
1746 const instanceId = "RB-TERMINATE-REPLAY";
1747 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1748 await doWithRollback(step, "setup-resource", async () => "resource-id", rollbackOptions());
1749 await step.sleep("wait-forever", "1 hour");
1750 });
1751
1752 await readLogsAfter(engineStub, (currentLogs) =>
1753 currentLogs.logs.some(
1754 (log) => log.event === InstanceEvent.STEP_SUCCESS && log.target === "setup-resource-1",
1755 ),
1756 );
1757
1758 await runInDurableObject(engineStub, async (engine) => {
1759 engine.rollbackRegistry.clear();
1760 });
1761
1762 try {
1763 await runInDurableObject(engineStub, async (engine) => {
1764 await engine.changeInstanceStatus("terminate", undefined, {
1765 rollback: true,
1766 });
1767 });
1768 } catch (error) {
1769 if (!(error instanceof Error) || !error.message.startsWith("Aborting engine:")) {
1770 throw error;
1771 }
1772 }
1773
1774 const logs = await readLogsAfter(
1775 env.ENGINE.get(env.ENGINE.idFromName(instanceId)),
1776 (currentLogs) =>
1777 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
1778 );
1779 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["setup-resource-1"]);
1780 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
1781 });
1782
1783 it("replays started unfinished steps before terminate rollback when registry is empty", async ({
1784 expect,
1785 }) => {
1786 const instanceId = `RB-TERMINATE-STARTED-REPLAY-${crypto.randomUUID()}`;
1787 const engineId = env.ENGINE.idFromName(instanceId);
1788 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1789 await doWithRollback(
1790 step,
1791 "started-setup",
1792 async () => {
1793 await scheduler.wait(30_000);
1794 return "late-resource-id";
1795 },
1796 rollbackOptions(),
1797 );
1798 });
1799
1800 await readLogsAfter(engineStub, (currentLogs) =>
1801 currentLogs.logs.some(
1802 (log) => log.event === InstanceEvent.ATTEMPT_START && log.target === "started-setup-1",
1803 ),
1804 );
1805
1806 try {
1807 await runInDurableObject(engineStub, async (engine) => {
1808 await engine.abort("test restart before step completion");
1809 });
1810 } catch (error) {
1811 if (!(error instanceof Error) || error.message !== "test restart before step completion") {
1812 throw error;
1813 }
1814 }
1815
1816 const restartedStub = env.ENGINE.get(engineId);
1817 await runInDurableObject(restartedStub, async (engine) => {
1818 engine.rollbackRegistry.clear();
1819 });
1820
1821 try {
1822 await runInDurableObject(restartedStub, async (engine) => {
1823 await engine.changeInstanceStatus("terminate", undefined, {
1824 rollback: true,
1825 });
1826 });
1827 } catch (error) {
1828 if (!isAbortError(error)) {
1829 throw error;
1830 }
1831 }
1832
1833 let logs: EngineLogs | undefined;
1834 await vi.waitUntil(
1835 async () => {
1836 try {
1837 logs = (await env.ENGINE.get(engineId).readLogs()) as EngineLogs;
1838 return logs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED);
1839 } catch (error) {
1840 if (isAbortError(error)) {
1841 return false;
1842 }
1843 throw error;
1844 }
1845 },
1846 { timeout: 5000 },
1847 );
1848 if (logs === undefined) {
1849 throw new Error("Expected workflow logs after terminate");
1850 }
1851 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["started-setup-1"]);
1852 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
1853 });
1854
1855 it("fails terminate rollback when replay cannot rebuild an eligible handler", async ({
1856 expect,
1857 }) => {
1858 const instanceId = `RB-TERMINATE-MISSING-REPLAY-${crypto.randomUUID()}`;
1859 const engineId = env.ENGINE.idFromName(instanceId);
1860 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1861 await doWithRollback(step, "branchy-setup", async () => "resource-id", rollbackOptions());
1862 await step.sleep("wait-forever", "1 hour");
1863 });
1864
1865 await readLogsAfter(engineStub, (currentLogs) =>
1866 currentLogs.logs.some(
1867 (log) => log.event === InstanceEvent.STEP_SUCCESS && log.target === "branchy-setup-1",
1868 ),
1869 );
1870
1871 try {
1872 await runInDurableObject(engineStub, async (engine) => {
1873 await engine.abort("test restart before replay branch changes");
1874 });
1875 } catch (error) {
1876 if (
1877 !(error instanceof Error) ||
1878 error.message !== "test restart before replay branch changes"
1879 ) {
1880 throw error;
1881 }
1882 }
1883
1884 setTestWorkflowCallback(async () => {
1885 // Simulate nondeterministic replay control flow that no longer reaches
1886 // the persisted eligible step. Production treats this as rollback failure.
1887 });
1888
1889 try {
1890 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
1891 await engine.changeInstanceStatus("terminate", undefined, {
1892 rollback: true,
1893 });
1894 });
1895 } catch (error) {
1896 if (!isAbortError(error)) {
1897 throw error;
1898 }
1899 }
1900
1901 const logs = await readLogsAfter(env.ENGINE.get(engineId), (currentLogs) =>
1902 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
1903 );
1904 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_FAILURE)).toEqual(["branchy-setup-1"]);
1905 expect(countOf(logs, InstanceEvent.ROLLBACK_FAILED)).toBe(1);
1906 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(0);
1907 });
1908
1909 it("replays cached failed steps before terminate rollback when registry is empty", async ({
1910 expect,
1911 }) => {
1912 const instanceId = "RB-TERMINATE-FAILED-REPLAY";
1913 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1914 try {
1915 await doWithRollback(
1916 step,
1917 "failed-setup",
1918 { retries: { limit: 0, delay: 0, backoff: "constant" } },
1919 async () => {
1920 throw new Error("step-boom");
1921 },
1922 rollbackOptions(),
1923 );
1924 } catch {
1925 // Keep the workflow running so terminate rollback can replay the cached
1926 // failed step and re-register its rollback handler.
1927 }
1928 await step.sleep("wait-forever", "1 hour");
1929 });
1930
1931 await readLogsAfter(engineStub, (currentLogs) =>
1932 currentLogs.logs.some(
1933 (log) => log.event === InstanceEvent.STEP_FAILURE && log.target === "failed-setup-1",
1934 ),
1935 );
1936
1937 await runInDurableObject(engineStub, async (engine) => {
1938 engine.rollbackRegistry.clear();
1939 });
1940
1941 try {
1942 await runInDurableObject(engineStub, async (engine) => {
1943 await engine.changeInstanceStatus("terminate", undefined, {
1944 rollback: true,
1945 });
1946 });
1947 } catch (error) {
1948 if (!(error instanceof Error) || !error.message.startsWith("Aborting engine:")) {
1949 throw error;
1950 }
1951 }
1952
1953 const logs = await readLogsAfter(
1954 env.ENGINE.get(env.ENGINE.idFromName(instanceId)),
1955 (currentLogs) =>
1956 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
1957 );
1958 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["failed-setup-1"]);
1959 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
1960 });
1961
1962 it("omits dynamic delay from rollback ctx.config on cached-error replay", async ({ expect }) => {
1963 const dynamicDelay: WorkflowDelayFunction = () => 1000;
1964 const instanceId = "RB-TERMINATE-FAILED-REPLAY-DYNAMIC-DELAY";
1965 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
1966 try {
1967 await doWithRollback(
1968 step,
1969 "failed-setup",
1970 { retries: { limit: 0, delay: dynamicDelay, backoff: "constant" } },
1971 async () => {
1972 throw new Error("step-boom");
1973 },
1974 rollbackOptions(async (ctx) => {
1975 // A dynamic delay must be omitted from the user-facing config,
1976 // never surfaced as the internal "[dynamic]" marker string.
1977 if ("delay" in ctx.ctx.config.retries) {
1978 throw new Error("dynamic delay marker leaked into rollback ctx.config");
1979 }
1980 }),
1981 );
1982 } catch {
1983 // Keep the workflow running so terminate rollback can replay the cached
1984 // failed step and re-register its rollback handler.
1985 }
1986 await step.sleep("wait-forever", "1 hour");
1987 });
1988
1989 await readLogsAfter(engineStub, (currentLogs) =>
1990 currentLogs.logs.some(
1991 (log) => log.event === InstanceEvent.STEP_FAILURE && log.target === "failed-setup-1",
1992 ),
1993 );
1994
1995 await runInDurableObject(engineStub, async (engine) => {
1996 engine.rollbackRegistry.clear();
1997 });
1998
1999 try {
2000 await runInDurableObject(engineStub, async (engine) => {
2001 await engine.changeInstanceStatus("terminate", undefined, {
2002 rollback: true,
2003 });
2004 });
2005 } catch (error) {
2006 if (!(error instanceof Error) || !error.message.startsWith("Aborting engine:")) {
2007 throw error;
2008 }
2009 }
2010
2011 const logs = await readLogsAfter(
2012 env.ENGINE.get(env.ENGINE.idFromName(instanceId)),
2013 (currentLogs) =>
2014 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
2015 );
2016 // ROLLBACK_STEP_SUCCESS only fires if the rollback fn did not throw, i.e.
2017 // the leaked marker was stripped by toEngineStepConfig on the replay path.
2018 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["failed-setup-1"]);
2019 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
2020 });
2021
2022 it("runs terminate rollback while paused with an empty registry", async ({ expect }) => {
2023 const instanceId = "RB-TERMINATE-PAUSED";
2024 const engineId = env.ENGINE.idFromName(instanceId);
2025 const engineStub = await runWorkflow(instanceId, async (_e, step) => {
2026 await doWithRollback(step, "setup-resource", async () => "resource-id", rollbackOptions());
2027 await step.sleep("wait-forever", "1 hour");
2028 });
2029
2030 await readLogsAfter(engineStub, (currentLogs) =>
2031 currentLogs.logs.some(
2032 (log) => log.event === InstanceEvent.STEP_SUCCESS && log.target === "setup-resource-1",
2033 ),
2034 );
2035
2036 try {
2037 await runInDurableObject(engineStub, async (engine) => {
2038 await engine.changeInstanceStatus("pause");
2039 });
2040 } catch (error) {
2041 if (!isAbortError(error)) {
2042 throw error;
2043 }
2044 }
2045
2046 await vi.waitUntil(
2047 async () =>
2048 runInDurableObject(
2049 env.ENGINE.get(engineId),
2050 async (engine) => (await engine.getStatus()) === InstanceStatus.Paused,
2051 ),
2052 { timeout: 5000 },
2053 );
2054
2055 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
2056 engine.rollbackRegistry.clear();
2057 });
2058
2059 try {
2060 await runInDurableObject(env.ENGINE.get(engineId), async (engine) => {
2061 await engine.changeInstanceStatus("terminate", undefined, {
2062 rollback: true,
2063 });
2064 });
2065 } catch (error) {
2066 if (!isAbortError(error)) {
2067 throw error;
2068 }
2069 }
2070
2071 const logs = await readLogsAfter(env.ENGINE.get(engineId), (currentLogs) =>
2072 currentLogs.logs.some((log) => log.event === InstanceEvent.WORKFLOW_TERMINATED),
2073 );
2074 expect(targetsOf(logs, InstanceEvent.ROLLBACK_STEP_SUCCESS)).toEqual(["setup-resource-1"]);
2075 expect(countOf(logs, InstanceEvent.ROLLBACK_COMPLETE)).toBe(1);
2076 });
2077
2078 it("does not run rollback when workflow succeeds", async ({ expect }) => {
2079 const stub = await runWorkflowAndAwait("RB-NOOP", async (_e, step) => {
2080 await doWithRollback(step, "a", async () => "ok", rollbackOptions());
2081 return "done";
2082 });
2083 const logs = await readLogsAfter(stub, (l) =>
2084 l.logs.some((r) => r.event === InstanceEvent.WORKFLOW_SUCCESS),
2085 );
2086 expect(countOf(logs, InstanceEvent.ROLLBACK_START)).toBe(0);
2087 });
2088 });