The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // Alchemy modifications are licensed under Apache-2.0. |
| 2 | // This file includes third-party code; see /THIRD_PARTY_LICENSES.md. |
| 3 | // Alchemy modifications: uses Array<T> syntax for non-tuple array types to match the repository convention. |
| 4 | import { runInDurableObject } from "cloudflare:test"; |
| 5 | import { env } from "cloudflare:workers"; |
| 6 | import { NonRetryableError } from "cloudflare:workflows"; |
| 7 | import { afterEach, describe, it, vi } from "vitest"; |
| 8 | import workerdUnsafe from "workerd:unsafe"; |
| 9 | import { REDACTED_STEP_OUTPUT } from "../context.ts"; |
| 10 | import type { Engine, EngineLogs } from "../engine.ts"; |
| 11 | import { InstanceEvent } from "../index.ts"; |
| 12 | import { computeHash } from "../lib/cache.ts"; |
| 13 | import { |
| 14 | InvalidStepReadableStreamError, |
| 15 | OversizedStreamChunkError, |
| 16 | UnsupportedStreamChunkError, |
| 17 | WorkflowTimeoutError, |
| 18 | isAbortError, |
| 19 | } from "../lib/errors.ts"; |
| 20 | import { |
| 21 | STREAMING_STEP_CHUNKS_TABLE, |
| 22 | getStreamOutputMetaKey, |
| 23 | StreamOutputState, |
| 24 | rollbackStreamOutput, |
| 25 | writeStreamOutput, |
| 26 | } from "../lib/streams.ts"; |
| 27 | import type { StreamOutputMeta } from "../lib/streams.ts"; |
| 28 | import { MODIFIER_KEYS } from "../modifier.ts"; |
| 29 | import { runWorkflow, runWorkflowAndAwait } from "./utils.ts"; |
| 30 | |
| 31 | afterEach(async () => { |
| 32 | await workerdUnsafe.abortAllDurableObjects(); |
| 33 | }); |
| 34 | |
| 35 | describe("Context", () => { |
| 36 | it("should provide attempt count 1 on first successful attempt", async ({ expect }) => { |
| 37 | let receivedAttempt: number | undefined; |
| 38 | |
| 39 | const engineStub = await runWorkflow("MOCK-INSTANCE-ID", async (_event, step) => { |
| 40 | const result = await step.do("a successful step", async (ctx) => { |
| 41 | receivedAttempt = ctx.attempt; |
| 42 | return "success"; |
| 43 | }); |
| 44 | return result; |
| 45 | }); |
| 46 | |
| 47 | await vi.waitUntil( |
| 48 | async () => { |
| 49 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 50 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 51 | }, |
| 52 | { timeout: 5000 }, |
| 53 | ); |
| 54 | |
| 55 | expect(receivedAttempt).toBe(1); |
| 56 | }); |
| 57 | |
| 58 | it("should provide attempt count to callback", async ({ expect }) => { |
| 59 | const receivedAttempts: Array<number> = []; |
| 60 | |
| 61 | const engineStub = await runWorkflow("MOCK-INSTANCE-ID-RETRY", async (_event, step) => { |
| 62 | const result = await step.do( |
| 63 | "retrying step", |
| 64 | { |
| 65 | retries: { |
| 66 | limit: 2, |
| 67 | delay: 0, |
| 68 | }, |
| 69 | }, |
| 70 | async (ctx) => { |
| 71 | receivedAttempts.push(ctx.attempt); |
| 72 | throw new Error(`Throwing`); |
| 73 | }, |
| 74 | ); |
| 75 | return result; |
| 76 | }); |
| 77 | |
| 78 | await vi.waitUntil( |
| 79 | async () => { |
| 80 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 81 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE); |
| 82 | }, |
| 83 | { timeout: 5000 }, |
| 84 | ); |
| 85 | |
| 86 | // Should have received attempts 1, 2, and 3 |
| 87 | expect(receivedAttempts).toEqual([1, 2, 3]); |
| 88 | }); |
| 89 | |
| 90 | it("should wait for retry delays by default", async ({ expect }) => { |
| 91 | const start = Date.now(); |
| 92 | const engineStub = await runWorkflowAndAwait( |
| 93 | "MOCK-INSTANCE-RETRY-DELAYS", |
| 94 | async (_event, step) => { |
| 95 | const result = await step.do( |
| 96 | "retrying step with delay", |
| 97 | { |
| 98 | retries: { |
| 99 | limit: 1, |
| 100 | delay: "1 second", |
| 101 | backoff: "constant", |
| 102 | }, |
| 103 | }, |
| 104 | async () => { |
| 105 | throw new Error("Always fails"); |
| 106 | }, |
| 107 | ); |
| 108 | return result; |
| 109 | }, |
| 110 | ); |
| 111 | const elapsed = Date.now() - start; |
| 112 | |
| 113 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 114 | expect(logs.logs.filter((val) => val.event === InstanceEvent.ATTEMPT_START)).toHaveLength(2); |
| 115 | // Should have waited at least ~1 second for the retry delay |
| 116 | expect(elapsed).toBeGreaterThanOrEqual(900); |
| 117 | }); |
| 118 | |
| 119 | it("should skip retry delays when disableRetryDelays is set", async ({ expect }) => { |
| 120 | const engineId = env.ENGINE.idFromName("MOCK-INSTANCE-DISABLE-RETRY"); |
| 121 | const engineStub = env.ENGINE.get(engineId); |
| 122 | |
| 123 | // Set the disableRetryDelays flag before running the workflow |
| 124 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 125 | await state.storage.put(MODIFIER_KEYS.DISABLE_ALL_RETRY_DELAYS, true); |
| 126 | }); |
| 127 | |
| 128 | const start = Date.now(); |
| 129 | const stub = await runWorkflowAndAwait("MOCK-INSTANCE-DISABLE-RETRY", async (_event, step) => { |
| 130 | const result = await step.do( |
| 131 | "retrying step with delay", |
| 132 | { |
| 133 | retries: { |
| 134 | limit: 2, |
| 135 | delay: "10 seconds", |
| 136 | backoff: "constant", |
| 137 | }, |
| 138 | }, |
| 139 | async () => { |
| 140 | throw new Error("Always fails"); |
| 141 | }, |
| 142 | ); |
| 143 | return result; |
| 144 | }); |
| 145 | const elapsed = Date.now() - start; |
| 146 | |
| 147 | const logs = (await stub.readLogs()) as EngineLogs; |
| 148 | expect(logs.logs.filter((val) => val.event === InstanceEvent.ATTEMPT_START)).toHaveLength(3); |
| 149 | // Without disableRetryDelays, this would take 20+ seconds (10s + 10s) |
| 150 | expect(elapsed).toBeLessThan(5000); |
| 151 | }); |
| 152 | |
| 153 | // NOTE: The `step` and `config` fields on WorkflowStepContext are new additions |
| 154 | // that haven't been published to @cloudflare/workers-types yet. The runtime |
| 155 | // provides them, but TypeScript doesn't know about them, so we capture ctx as |
| 156 | // `unknown` and assert on the shape. |
| 157 | // TODO: Remove these workarounds once https://github.com/cloudflare/workerd/pull/6523 lands. |
| 158 | |
| 159 | it("should provide step name and count in context", async ({ expect }) => { |
| 160 | let receivedCtx: unknown; |
| 161 | |
| 162 | const engineStub = await runWorkflow("MOCK-INSTANCE-STEP-CTX", async (_event, step) => { |
| 163 | const result = await step.do("my step", async (ctx) => { |
| 164 | receivedCtx = ctx; |
| 165 | return "done"; |
| 166 | }); |
| 167 | return result; |
| 168 | }); |
| 169 | |
| 170 | await vi.waitUntil( |
| 171 | async () => { |
| 172 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 173 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 174 | }, |
| 175 | { timeout: 5000 }, |
| 176 | ); |
| 177 | |
| 178 | expect(receivedCtx).toMatchObject({ |
| 179 | step: { name: "my step", count: 1 }, |
| 180 | }); |
| 181 | }); |
| 182 | |
| 183 | it("should increment step count for steps with the same name", async ({ expect }) => { |
| 184 | const receivedContexts: Array<unknown> = []; |
| 185 | |
| 186 | const engineStub = await runWorkflow("MOCK-INSTANCE-STEP-COUNT", async (_event, step) => { |
| 187 | await step.do("repeated step", async (ctx) => { |
| 188 | receivedContexts.push(ctx); |
| 189 | return "first"; |
| 190 | }); |
| 191 | await step.do("repeated step", async (ctx) => { |
| 192 | receivedContexts.push(ctx); |
| 193 | return "second"; |
| 194 | }); |
| 195 | await step.do("different step", async (ctx) => { |
| 196 | receivedContexts.push(ctx); |
| 197 | return "third"; |
| 198 | }); |
| 199 | return "done"; |
| 200 | }); |
| 201 | |
| 202 | // Needs extra headroom: 3 sequential step.do calls + cold DO startup on Windows |
| 203 | await vi.waitUntil( |
| 204 | async () => { |
| 205 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 206 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 207 | }, |
| 208 | { timeout: 10000 }, |
| 209 | ); |
| 210 | |
| 211 | expect(receivedContexts[0]).toMatchObject({ |
| 212 | step: { name: "repeated step", count: 1 }, |
| 213 | }); |
| 214 | expect(receivedContexts[1]).toMatchObject({ |
| 215 | step: { name: "repeated step", count: 2 }, |
| 216 | }); |
| 217 | expect(receivedContexts[2]).toMatchObject({ |
| 218 | step: { name: "different step", count: 1 }, |
| 219 | }); |
| 220 | }); |
| 221 | |
| 222 | it("should provide resolved config with defaults in context", async ({ expect }) => { |
| 223 | let receivedCtx: unknown; |
| 224 | |
| 225 | const engineStub = await runWorkflow( |
| 226 | "MOCK-INSTANCE-CTX-CONFIG-DEFAULTS", |
| 227 | async (_event, step) => { |
| 228 | // No config provided — ctx.config should have all defaults |
| 229 | const result = await step.do("default config step", async (ctx) => { |
| 230 | receivedCtx = ctx; |
| 231 | return "done"; |
| 232 | }); |
| 233 | return result; |
| 234 | }, |
| 235 | ); |
| 236 | |
| 237 | await vi.waitUntil( |
| 238 | async () => { |
| 239 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 240 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 241 | }, |
| 242 | { timeout: 5000 }, |
| 243 | ); |
| 244 | |
| 245 | expect(receivedCtx).toMatchObject({ |
| 246 | config: { |
| 247 | retries: { |
| 248 | limit: 5, |
| 249 | delay: 1000, |
| 250 | backoff: "exponential", |
| 251 | }, |
| 252 | timeout: "10 minutes", |
| 253 | }, |
| 254 | }); |
| 255 | }); |
| 256 | |
| 257 | it("should provide resolved config with user overrides merged in context", async ({ expect }) => { |
| 258 | let receivedCtx: unknown; |
| 259 | |
| 260 | const engineStub = await runWorkflow( |
| 261 | "MOCK-INSTANCE-CTX-CONFIG-OVERRIDES", |
| 262 | async (_event, step) => { |
| 263 | const result = await step.do( |
| 264 | "custom config step", |
| 265 | { |
| 266 | retries: { |
| 267 | limit: 3, |
| 268 | delay: 500, |
| 269 | }, |
| 270 | timeout: "5 minutes", |
| 271 | }, |
| 272 | async (ctx) => { |
| 273 | receivedCtx = ctx; |
| 274 | return "done"; |
| 275 | }, |
| 276 | ); |
| 277 | return result; |
| 278 | }, |
| 279 | ); |
| 280 | |
| 281 | await vi.waitUntil( |
| 282 | async () => { |
| 283 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 284 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 285 | }, |
| 286 | { timeout: 5000 }, |
| 287 | ); |
| 288 | |
| 289 | // User overrides should be merged with defaults |
| 290 | expect(receivedCtx).toMatchObject({ |
| 291 | config: { |
| 292 | retries: { |
| 293 | limit: 3, |
| 294 | delay: 500, |
| 295 | backoff: "exponential", |
| 296 | }, |
| 297 | timeout: "5 minutes", |
| 298 | }, |
| 299 | }); |
| 300 | }); |
| 301 | |
| 302 | it("should not allow user callback to mutate engine retry config", async ({ expect }) => { |
| 303 | const receivedAttempts: Array<number> = []; |
| 304 | |
| 305 | const engineStub = await runWorkflow( |
| 306 | "MOCK-INSTANCE-CTX-CONFIG-MUTATION", |
| 307 | async (_event, step) => { |
| 308 | const result = await step.do( |
| 309 | "mutating step", |
| 310 | { |
| 311 | retries: { |
| 312 | limit: 1, |
| 313 | delay: 0, |
| 314 | }, |
| 315 | }, |
| 316 | async (ctx) => { |
| 317 | receivedAttempts.push(ctx.attempt); |
| 318 | // Attempt to escalate retries from 1 to 100 |
| 319 | (ctx as unknown as { config: { retries: { limit: number } } }).config.retries.limit = |
| 320 | 100; |
| 321 | throw new Error("retry me"); |
| 322 | }, |
| 323 | ); |
| 324 | return result; |
| 325 | }, |
| 326 | ); |
| 327 | |
| 328 | await vi.waitUntil( |
| 329 | async () => { |
| 330 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 331 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE); |
| 332 | }, |
| 333 | { timeout: 5000 }, |
| 334 | ); |
| 335 | |
| 336 | // With limit: 1, the engine should execute attempt 1 + 1 retry = 2 attempts total. |
| 337 | // If the mutation leaked, this would keep going far beyond 2. |
| 338 | expect(receivedAttempts).toEqual([1, 2]); |
| 339 | }); |
| 340 | }); |
| 341 | |
| 342 | // ── Helpers for stream tests ──────────────────────────────────────────────── |
| 343 | |
| 344 | function encodeUtf8(str: string): Uint8Array { |
| 345 | return new TextEncoder().encode(str); |
| 346 | } |
| 347 | |
| 348 | function decodeUtf8(bytes: Uint8Array): string { |
| 349 | return new TextDecoder().decode(bytes); |
| 350 | } |
| 351 | |
| 352 | async function readStreamBytes(stream: ReadableStream<Uint8Array>): Promise<Uint8Array> { |
| 353 | const chunks: Array<Uint8Array> = []; |
| 354 | const reader = stream.getReader(); |
| 355 | while (true) { |
| 356 | const { done, value } = await reader.read(); |
| 357 | if (done) { |
| 358 | break; |
| 359 | } |
| 360 | chunks.push(value); |
| 361 | } |
| 362 | let totalLength = 0; |
| 363 | for (const chunk of chunks) { |
| 364 | totalLength += chunk.byteLength; |
| 365 | } |
| 366 | const result = new Uint8Array(totalLength); |
| 367 | let offset = 0; |
| 368 | for (const chunk of chunks) { |
| 369 | result.set(chunk, offset); |
| 370 | offset += chunk.byteLength; |
| 371 | } |
| 372 | return result; |
| 373 | } |
| 374 | |
| 375 | function countStreamOutputChunks(state: DurableObjectState, cacheKey: string): number { |
| 376 | const row = state.storage.sql |
| 377 | .exec<{ cnt: number }>( |
| 378 | `SELECT COUNT(*) AS cnt FROM streaming_step_chunks WHERE cache_key = ?`, |
| 379 | cacheKey, |
| 380 | ) |
| 381 | .one(); |
| 382 | return row?.cnt ?? 0; |
| 383 | } |
| 384 | |
| 385 | describe("Context - ReadableStream step outputs", () => { |
| 386 | it("should persist a readable stream output and replay from cache", async ({ expect }) => { |
| 387 | const payload = "hello from a readable stream ".repeat(500); // ~14KB |
| 388 | const payloadBytes = encodeUtf8(payload); |
| 389 | let callCount = 0; |
| 390 | |
| 391 | const engineStub = await runWorkflow("STREAM-BASIC", async (_event, step) => { |
| 392 | const stream = await step.do("stream step", async () => { |
| 393 | callCount++; |
| 394 | return new ReadableStream<Uint8Array>({ |
| 395 | start(controller) { |
| 396 | // Enqueue in two chunks |
| 397 | const mid = Math.floor(payloadBytes.length / 2); |
| 398 | controller.enqueue(payloadBytes.slice(0, mid)); |
| 399 | controller.enqueue(payloadBytes.slice(mid)); |
| 400 | controller.close(); |
| 401 | }, |
| 402 | }); |
| 403 | }); |
| 404 | |
| 405 | // The result should be a ReadableStream we can read |
| 406 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 407 | return decodeUtf8(bytes); |
| 408 | }); |
| 409 | |
| 410 | await vi.waitUntil( |
| 411 | async () => { |
| 412 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 413 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 414 | }, |
| 415 | { timeout: 5000 }, |
| 416 | ); |
| 417 | |
| 418 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 419 | const successLog = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 420 | expect(successLog?.metadata.result).toBe(payload); |
| 421 | |
| 422 | // The closure should have been called exactly once (cache hit on replay) |
| 423 | expect(callCount).toBe(1); |
| 424 | |
| 425 | // Verify the stream metadata is stored and marked complete |
| 426 | const hash = await computeHash("stream step"); |
| 427 | const cacheKey = `${hash}-1`; |
| 428 | const metaKey = getStreamOutputMetaKey(cacheKey); |
| 429 | |
| 430 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 431 | const meta = (await state.storage.get(metaKey)) as StreamOutputMeta; |
| 432 | expect(meta).toBeDefined(); |
| 433 | expect(meta.state).toBe(StreamOutputState.Complete); |
| 434 | expect(meta.totalBytes).toBe(payloadBytes.byteLength); |
| 435 | expect(meta.chunkCount).toBeGreaterThanOrEqual(1); |
| 436 | }); |
| 437 | }); |
| 438 | |
| 439 | it("should persist an empty readable stream", async ({ expect }) => { |
| 440 | const engineStub = await runWorkflow("STREAM-EMPTY", async (_event, step) => { |
| 441 | const stream = await step.do("empty stream step", async () => { |
| 442 | return new ReadableStream<Uint8Array>({ |
| 443 | start(controller) { |
| 444 | controller.close(); |
| 445 | }, |
| 446 | }); |
| 447 | }); |
| 448 | |
| 449 | // Should be a readable stream that yields no data |
| 450 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 451 | return bytes.byteLength; |
| 452 | }); |
| 453 | |
| 454 | await vi.waitUntil( |
| 455 | async () => { |
| 456 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 457 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 458 | }, |
| 459 | { timeout: 5000 }, |
| 460 | ); |
| 461 | |
| 462 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 463 | const successLog = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 464 | expect(successLog?.metadata.result).toBe(0); |
| 465 | |
| 466 | // Verify stream metadata |
| 467 | const hash = await computeHash("empty stream step"); |
| 468 | const cacheKey = `${hash}-1`; |
| 469 | const metaKey = getStreamOutputMetaKey(cacheKey); |
| 470 | |
| 471 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 472 | const meta = (await state.storage.get(metaKey)) as StreamOutputMeta; |
| 473 | expect(meta).toBeDefined(); |
| 474 | expect(meta.state).toBe(StreamOutputState.Complete); |
| 475 | expect(meta.totalBytes).toBe(0); |
| 476 | expect(meta.chunkCount).toBe(0); |
| 477 | }); |
| 478 | }); |
| 479 | |
| 480 | // ── Tests that exercise stream error paths directly inside the DO ──── |
| 481 | // |
| 482 | // The tests below call writeStreamOutput() inside runInDurableObject() |
| 483 | // rather than running a full workflow through step.do(). |
| 484 | // |
| 485 | // Why: In the test harness the user callback runs in a separate |
| 486 | // WorkerEntrypoint (TestWorkflow) and communicates with the Engine |
| 487 | // Durable Object over RPC. Returning a problematic ReadableStream from |
| 488 | // that callback fails at the *RPC transfer* layer with native workerd |
| 489 | // errors that are different from the custom errors the engine raises: |
| 490 | // |
| 491 | // • Locked stream → TypeError: "The ReadableStream has been locked |
| 492 | // to a reader." |
| 493 | // • Non-byte chunks → TypeError: "This ReadableStream did not return |
| 494 | // bytes." |
| 495 | // • Very large chunks → Error: "Network connection lost." (the RPC |
| 496 | // pipe disconnects under load) |
| 497 | // |
| 498 | // In production the callback executes in the *same* isolate as the |
| 499 | // engine, so the stream reaches writeStreamOutput() directly and the |
| 500 | // engine's own validation (iterateStreamChunks, normalizeChunkToUint8Array, |
| 501 | // chunk-size check) surfaces the correct custom error types. |
| 502 | // |
| 503 | // By calling writeStreamOutput() inside the DO we replicate the |
| 504 | // production code-path and avoid the RPC transfer artefacts. |
| 505 | |
| 506 | it("should surface a locked readable stream as a fatal error", async ({ expect }) => { |
| 507 | const engineId = env.ENGINE.idFromName("STREAM-LOCKED"); |
| 508 | const engineStub = env.ENGINE.get(engineId); |
| 509 | |
| 510 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 511 | // Create the streaming table (normally done during engine init) |
| 512 | // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- DO SQL exec, not child_process |
| 513 | state.storage.sql.exec(` |
| 514 | CREATE TABLE IF NOT EXISTS ${STREAMING_STEP_CHUNKS_TABLE} ( |
| 515 | cache_key TEXT NOT NULL, |
| 516 | attempt INTEGER NOT NULL, |
| 517 | chunk_index INTEGER NOT NULL, |
| 518 | chunk BLOB NOT NULL, |
| 519 | PRIMARY KEY (cache_key, attempt, chunk_index) |
| 520 | ) WITHOUT ROWID |
| 521 | `); |
| 522 | |
| 523 | const stream = new ReadableStream<Uint8Array>({ |
| 524 | start(controller) { |
| 525 | controller.enqueue(encodeUtf8("data")); |
| 526 | controller.close(); |
| 527 | }, |
| 528 | }); |
| 529 | // Lock the stream by acquiring a reader |
| 530 | stream.getReader(); |
| 531 | |
| 532 | try { |
| 533 | await writeStreamOutput({ |
| 534 | storage: state.storage, |
| 535 | cacheKey: "locked-stream-test", |
| 536 | attempt: 1, |
| 537 | stream, |
| 538 | }); |
| 539 | expect.unreachable("writeStreamOutput should have thrown"); |
| 540 | } catch (e) { |
| 541 | expect(e).toBeInstanceOf(InvalidStepReadableStreamError); |
| 542 | } |
| 543 | }); |
| 544 | }); |
| 545 | |
| 546 | it("should surface an unsupported chunk type as a fatal error", async ({ expect }) => { |
| 547 | const engineId = env.ENGINE.idFromName("STREAM-UNSUPPORTED-CHUNK"); |
| 548 | const engineStub = env.ENGINE.get(engineId); |
| 549 | |
| 550 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 551 | // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- DO SQL exec, not child_process |
| 552 | state.storage.sql.exec(` |
| 553 | CREATE TABLE IF NOT EXISTS ${STREAMING_STEP_CHUNKS_TABLE} ( |
| 554 | cache_key TEXT NOT NULL, |
| 555 | attempt INTEGER NOT NULL, |
| 556 | chunk_index INTEGER NOT NULL, |
| 557 | chunk BLOB NOT NULL, |
| 558 | PRIMARY KEY (cache_key, attempt, chunk_index) |
| 559 | ) WITHOUT ROWID |
| 560 | `); |
| 561 | |
| 562 | const stream = new ReadableStream({ |
| 563 | start(controller) { |
| 564 | // Enqueue a string -- not supported (only ArrayBuffer / TypedArray) |
| 565 | controller.enqueue("this is a string, not bytes"); |
| 566 | controller.close(); |
| 567 | }, |
| 568 | }); |
| 569 | |
| 570 | try { |
| 571 | await writeStreamOutput({ |
| 572 | storage: state.storage, |
| 573 | cacheKey: "unsupported-chunk-test", |
| 574 | attempt: 1, |
| 575 | stream, |
| 576 | }); |
| 577 | expect.unreachable("writeStreamOutput should have thrown"); |
| 578 | } catch (e) { |
| 579 | expect(e).toBeInstanceOf(UnsupportedStreamChunkError); |
| 580 | } |
| 581 | }); |
| 582 | }); |
| 583 | |
| 584 | it("should surface an oversized stream chunk as a fatal error", async ({ expect }) => { |
| 585 | const engineId = env.ENGINE.idFromName("STREAM-OVERSIZED-CHUNK"); |
| 586 | const engineStub = env.ENGINE.get(engineId); |
| 587 | |
| 588 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 589 | // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- DO SQL exec, not child_process |
| 590 | state.storage.sql.exec(` |
| 591 | CREATE TABLE IF NOT EXISTS ${STREAMING_STEP_CHUNKS_TABLE} ( |
| 592 | cache_key TEXT NOT NULL, |
| 593 | attempt INTEGER NOT NULL, |
| 594 | chunk_index INTEGER NOT NULL, |
| 595 | chunk BLOB NOT NULL, |
| 596 | PRIMARY KEY (cache_key, attempt, chunk_index) |
| 597 | ) WITHOUT ROWID |
| 598 | `); |
| 599 | |
| 600 | const stream = new ReadableStream<Uint8Array>({ |
| 601 | start(controller) { |
| 602 | // 17 MiB chunk -- exceeds the 16 MiB per-chunk limit |
| 603 | controller.enqueue(new Uint8Array(17 * 1024 * 1024)); |
| 604 | controller.close(); |
| 605 | }, |
| 606 | }); |
| 607 | |
| 608 | try { |
| 609 | await writeStreamOutput({ |
| 610 | storage: state.storage, |
| 611 | cacheKey: "oversized-chunk-test", |
| 612 | attempt: 1, |
| 613 | stream, |
| 614 | }); |
| 615 | expect.unreachable("writeStreamOutput should have thrown"); |
| 616 | } catch (e) { |
| 617 | expect(e).toBeInstanceOf(OversizedStreamChunkError); |
| 618 | } |
| 619 | }); |
| 620 | }); |
| 621 | |
| 622 | it("should clean up streaming_step_chunks on restart", async ({ expect }) => { |
| 623 | const instanceId = "STREAM-RESTART-CLEANUP"; |
| 624 | const engineId = env.ENGINE.idFromName(instanceId); |
| 625 | |
| 626 | const engineStub = await runWorkflowAndAwait(instanceId, async (_event, step) => { |
| 627 | const stream = await step.do("stream before restart", async () => { |
| 628 | return new ReadableStream<Uint8Array>({ |
| 629 | start(controller) { |
| 630 | controller.enqueue(encodeUtf8("data for restart test")); |
| 631 | controller.close(); |
| 632 | }, |
| 633 | }); |
| 634 | }); |
| 635 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 636 | return decodeUtf8(bytes); |
| 637 | }); |
| 638 | |
| 639 | // Verify chunks exist before restart |
| 640 | const hash = await computeHash("stream before restart"); |
| 641 | const cacheKey = `${hash}-1`; |
| 642 | |
| 643 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 644 | const chunkCount = countStreamOutputChunks(state, cacheKey); |
| 645 | expect(chunkCount).toBeGreaterThanOrEqual(1); |
| 646 | }); |
| 647 | |
| 648 | // Trigger restart |
| 649 | try { |
| 650 | await runInDurableObject(engineStub, async (engine) => { |
| 651 | await engine.changeInstanceStatus("restart"); |
| 652 | }); |
| 653 | } catch (e) { |
| 654 | if (!isAbortError(e)) { |
| 655 | throw e; |
| 656 | } |
| 657 | } |
| 658 | |
| 659 | const restartedStub: DurableObjectStub<Engine> = env.ENGINE.get(engineId); |
| 660 | await runInDurableObject(restartedStub, async (engine, state) => { |
| 661 | await engine.attemptRestart(); |
| 662 | |
| 663 | // Check immediately after attemptRestart returns, before the |
| 664 | // fire-and-forget init() call (engine.ts:892) has a chance to |
| 665 | // re-execute the workflow and recreate the chunks. |
| 666 | const chunkCount = countStreamOutputChunks(state, cacheKey); |
| 667 | expect(chunkCount).toBe(0); |
| 668 | }); |
| 669 | }); |
| 670 | |
| 671 | it("should preserve mock stream chunks across restart", async ({ expect }) => { |
| 672 | const mockPayload = "mock stream survives restart"; |
| 673 | const mockPayloadBytes = encodeUtf8(mockPayload); |
| 674 | |
| 675 | const instanceId = "STREAM-MOCK-SURVIVES-RESTART"; |
| 676 | const engineId = env.ENGINE.idFromName(instanceId); |
| 677 | const engineStub = env.ENGINE.get(engineId); |
| 678 | |
| 679 | // Set up a mocked stream step via the modifier |
| 680 | await runInDurableObject(engineStub, async (engine) => { |
| 681 | const modifier = engine.getInstanceModifier(); |
| 682 | await modifier.mockStepResult( |
| 683 | { name: "mocked restart stream" }, |
| 684 | new ReadableStream<Uint8Array>({ |
| 685 | start(controller) { |
| 686 | controller.enqueue(mockPayloadBytes); |
| 687 | controller.close(); |
| 688 | }, |
| 689 | }), |
| 690 | ); |
| 691 | }); |
| 692 | |
| 693 | const hash = await computeHash("mocked restart stream"); |
| 694 | const baseCacheKey = `${hash}-1`; |
| 695 | |
| 696 | // Run the workflow once so the engine is initialised and can be restarted |
| 697 | const stub = await runWorkflow(instanceId, async (_event, step) => { |
| 698 | const stream = await step.do("mocked restart stream", async () => { |
| 699 | return new ReadableStream<Uint8Array>({ |
| 700 | start(controller) { |
| 701 | controller.enqueue(encodeUtf8("WRONG - real step ran")); |
| 702 | controller.close(); |
| 703 | }, |
| 704 | }); |
| 705 | }); |
| 706 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 707 | return decodeUtf8(bytes); |
| 708 | }); |
| 709 | |
| 710 | await vi.waitUntil( |
| 711 | async () => { |
| 712 | const logs = (await stub.readLogs()) as EngineLogs; |
| 713 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 714 | }, |
| 715 | { timeout: 5000 }, |
| 716 | ); |
| 717 | |
| 718 | // Trigger restart |
| 719 | try { |
| 720 | await runInDurableObject(stub, async (engine) => { |
| 721 | await engine.changeInstanceStatus("restart"); |
| 722 | }); |
| 723 | } catch (e) { |
| 724 | if (!isAbortError(e)) { |
| 725 | throw e; |
| 726 | } |
| 727 | } |
| 728 | |
| 729 | // After restart, mock stream chunks (attempt=0) must still exist |
| 730 | const restartedStub: DurableObjectStub<Engine> = env.ENGINE.get(engineId); |
| 731 | await runInDurableObject(restartedStub, async (engine, state) => { |
| 732 | await engine.attemptRestart(); |
| 733 | |
| 734 | const chunkCount = countStreamOutputChunks(state, baseCacheKey); |
| 735 | expect(chunkCount).toBeGreaterThanOrEqual(1); |
| 736 | }); |
| 737 | |
| 738 | // Run the workflow again — the mock sentinel (KV) and its SQL chunks |
| 739 | // should both be present, allowing the mocked stream to replay correctly. |
| 740 | const stub2 = await runWorkflow(instanceId, async (_event, step) => { |
| 741 | const stream = await step.do("mocked restart stream", async () => { |
| 742 | return new ReadableStream<Uint8Array>({ |
| 743 | start(controller) { |
| 744 | controller.enqueue(encodeUtf8("WRONG - real step ran after restart")); |
| 745 | controller.close(); |
| 746 | }, |
| 747 | }); |
| 748 | }); |
| 749 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 750 | return decodeUtf8(bytes); |
| 751 | }); |
| 752 | |
| 753 | await vi.waitUntil( |
| 754 | async () => { |
| 755 | const logs = (await stub2.readLogs()) as EngineLogs; |
| 756 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 757 | }, |
| 758 | { timeout: 5000 }, |
| 759 | ); |
| 760 | |
| 761 | const logs = (await stub2.readLogs()) as EngineLogs; |
| 762 | const successLog = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 763 | expect(successLog?.metadata.result).toBe(mockPayload); |
| 764 | }); |
| 765 | |
| 766 | it("should normalize TypedArray chunks to Uint8Array on replay", async ({ expect }) => { |
| 767 | // Return Int16Array chunks -- they should be stored as raw bytes |
| 768 | // and replayed as Uint8Array |
| 769 | const int16Data = new Int16Array([1, 2, 3, 256, -1]); |
| 770 | const expectedBytes = new Uint8Array( |
| 771 | int16Data.buffer, |
| 772 | int16Data.byteOffset, |
| 773 | int16Data.byteLength, |
| 774 | ); |
| 775 | |
| 776 | const engineStub = await runWorkflow("STREAM-TYPED-ARRAY", async (_event, step) => { |
| 777 | const stream = await step.do("typed array step", async () => { |
| 778 | return new ReadableStream({ |
| 779 | start(controller) { |
| 780 | controller.enqueue(int16Data); |
| 781 | controller.close(); |
| 782 | }, |
| 783 | }); |
| 784 | }); |
| 785 | |
| 786 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 787 | return Array.from(bytes); |
| 788 | }); |
| 789 | |
| 790 | await vi.waitUntil( |
| 791 | async () => { |
| 792 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 793 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 794 | }, |
| 795 | { timeout: 5000 }, |
| 796 | ); |
| 797 | |
| 798 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 799 | const successLog = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 800 | expect(successLog?.metadata.result).toEqual(Array.from(expectedBytes)); |
| 801 | }); |
| 802 | |
| 803 | it("should return a replay ReadableStream from waitForStepResult", async ({ expect }) => { |
| 804 | const payload = "stream content for waitForStepResult"; |
| 805 | const payloadBytes = encodeUtf8(payload); |
| 806 | |
| 807 | const engineStub = await runWorkflow("STREAM-WAIT-FOR-STEP", async (_event, step) => { |
| 808 | const stream = await step.do("stream step", async () => { |
| 809 | return new ReadableStream<Uint8Array>({ |
| 810 | start(controller) { |
| 811 | controller.enqueue(payloadBytes); |
| 812 | controller.close(); |
| 813 | }, |
| 814 | }); |
| 815 | }); |
| 816 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 817 | return decodeUtf8(bytes); |
| 818 | }); |
| 819 | |
| 820 | // Use engine's waitForStepResult to get the stream |
| 821 | const stepResult = await engineStub.waitForStepResult("stream step"); |
| 822 | |
| 823 | // Should be a ReadableStream |
| 824 | expect(stepResult).toBeInstanceOf(ReadableStream); |
| 825 | |
| 826 | const replayBytes = await readStreamBytes(stepResult as ReadableStream<Uint8Array>); |
| 827 | expect(decodeUtf8(replayBytes)).toBe(payload); |
| 828 | |
| 829 | // Wait for workflow to finish |
| 830 | await vi.waitUntil( |
| 831 | async () => { |
| 832 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 833 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 834 | }, |
| 835 | { timeout: 5000 }, |
| 836 | ); |
| 837 | }); |
| 838 | |
| 839 | it("should mock a step result with a ReadableStream via modifier", async ({ expect }) => { |
| 840 | const mockPayload = "mocked stream content from modifier"; |
| 841 | const mockPayloadBytes = encodeUtf8(mockPayload); |
| 842 | |
| 843 | const instanceId = "STREAM-MOCK-STEP-RESULT"; |
| 844 | const engineId = env.ENGINE.idFromName(instanceId); |
| 845 | const engineStub = env.ENGINE.get(engineId); |
| 846 | |
| 847 | // Set up the mock stream result via the modifier before running the workflow |
| 848 | await runInDurableObject(engineStub, async (engine) => { |
| 849 | const modifier = engine.getInstanceModifier(); |
| 850 | await modifier.mockStepResult( |
| 851 | { name: "mocked stream step" }, |
| 852 | new ReadableStream<Uint8Array>({ |
| 853 | start(controller) { |
| 854 | controller.enqueue(mockPayloadBytes); |
| 855 | controller.close(); |
| 856 | }, |
| 857 | }), |
| 858 | ); |
| 859 | }); |
| 860 | |
| 861 | const hash = await computeHash("mocked stream step"); |
| 862 | const baseCacheKey = `${hash}-1`; |
| 863 | const valueKey = `${baseCacheKey}-value`; |
| 864 | |
| 865 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 866 | // normal non-mocked key should not exist |
| 867 | expect(await state.storage.get(getStreamOutputMetaKey(baseCacheKey))).toBeUndefined(); |
| 868 | |
| 869 | const replaceResult = await state.storage.get<{ |
| 870 | __mockStreamOutput: true; |
| 871 | cacheKey: string; |
| 872 | meta: StreamOutputMeta; |
| 873 | }>(`${MODIFIER_KEYS.REPLACE_RESULT}${valueKey}`); |
| 874 | |
| 875 | expect(replaceResult?.__mockStreamOutput).toBe(true); |
| 876 | expect(replaceResult?.cacheKey).toBe(baseCacheKey); |
| 877 | expect(replaceResult?.meta.state).toBe(StreamOutputState.Complete); |
| 878 | expect(replaceResult?.meta.attempt).toBe(0); |
| 879 | expect(countStreamOutputChunks(state, baseCacheKey)).toBeGreaterThanOrEqual(1); |
| 880 | }); |
| 881 | |
| 882 | // Run a workflow that uses the mocked step |
| 883 | const stub = await runWorkflow(instanceId, async (_event, step) => { |
| 884 | const stream = await step.do("mocked stream step", async () => { |
| 885 | // This should NOT be called -- the mock replaces it |
| 886 | return new ReadableStream<Uint8Array>({ |
| 887 | start(controller) { |
| 888 | controller.enqueue(encodeUtf8("WRONG - real step ran")); |
| 889 | controller.close(); |
| 890 | }, |
| 891 | }); |
| 892 | }); |
| 893 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 894 | return decodeUtf8(bytes); |
| 895 | }); |
| 896 | |
| 897 | await vi.waitUntil( |
| 898 | async () => { |
| 899 | const logs = (await stub.readLogs()) as EngineLogs; |
| 900 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 901 | }, |
| 902 | { timeout: 5000 }, |
| 903 | ); |
| 904 | |
| 905 | const logs = (await stub.readLogs()) as EngineLogs; |
| 906 | const successLog = logs.logs.find((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 907 | // The workflow should have received the mocked stream content |
| 908 | expect(successLog?.metadata.result).toBe(mockPayload); |
| 909 | }); |
| 910 | |
| 911 | it("should resolve stream output to a text preview in readLogs", async ({ expect }) => { |
| 912 | const payload = "hello from stream preview test"; |
| 913 | const payloadBytes = encodeUtf8(payload); |
| 914 | |
| 915 | const engineStub = await runWorkflow("STREAM-PREVIEW-TEXT", async (_event, step) => { |
| 916 | const stream = await step.do("preview step", async () => { |
| 917 | return new ReadableStream<Uint8Array>({ |
| 918 | start(controller) { |
| 919 | controller.enqueue(payloadBytes); |
| 920 | controller.close(); |
| 921 | }, |
| 922 | }); |
| 923 | }); |
| 924 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 925 | return decodeUtf8(bytes); |
| 926 | }); |
| 927 | |
| 928 | await vi.waitUntil( |
| 929 | async () => { |
| 930 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 931 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 932 | }, |
| 933 | { timeout: 5000 }, |
| 934 | ); |
| 935 | |
| 936 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 937 | const stepLog = logs.logs.find((val) => val.event === InstanceEvent.STEP_SUCCESS); |
| 938 | // readLogs() should resolve the stream output metadata to the preview text |
| 939 | expect(stepLog?.metadata.result).toBe(payload); |
| 940 | }); |
| 941 | |
| 942 | it("should truncate a long stream preview in readLogs", async ({ expect }) => { |
| 943 | // Generate a payload longer than the 1024-char preview limit |
| 944 | const payload = "A".repeat(2048); |
| 945 | const payloadBytes = encodeUtf8(payload); |
| 946 | |
| 947 | const engineStub = await runWorkflow("STREAM-PREVIEW-TRUNCATED", async (_event, step) => { |
| 948 | const stream = await step.do("long preview step", async () => { |
| 949 | return new ReadableStream<Uint8Array>({ |
| 950 | start(controller) { |
| 951 | controller.enqueue(payloadBytes); |
| 952 | controller.close(); |
| 953 | }, |
| 954 | }); |
| 955 | }); |
| 956 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 957 | return decodeUtf8(bytes); |
| 958 | }); |
| 959 | |
| 960 | await vi.waitUntil( |
| 961 | async () => { |
| 962 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 963 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 964 | }, |
| 965 | { timeout: 5000 }, |
| 966 | ); |
| 967 | |
| 968 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 969 | const stepLog = logs.logs.find((val) => val.event === InstanceEvent.STEP_SUCCESS); |
| 970 | // readLogs() should truncate the preview to 1024 chars |
| 971 | expect(stepLog?.metadata.result).toBe("A".repeat(1024) + "[truncated output]"); |
| 972 | }); |
| 973 | |
| 974 | it("should resolve non-UTF-8 stream output to a binary summary in readLogs", async ({ |
| 975 | expect, |
| 976 | }) => { |
| 977 | // Write raw bytes that are not valid UTF-8 |
| 978 | const invalidUtf8 = new Uint8Array([0xff, 0xfe, 0x80, 0x81]); |
| 979 | |
| 980 | const engineStub = await runWorkflow("STREAM-PREVIEW-BINARY", async (_event, step) => { |
| 981 | const stream = await step.do("binary step", async () => { |
| 982 | return new ReadableStream<Uint8Array>({ |
| 983 | start(controller) { |
| 984 | controller.enqueue(invalidUtf8); |
| 985 | controller.close(); |
| 986 | }, |
| 987 | }); |
| 988 | }); |
| 989 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 990 | return bytes.byteLength; |
| 991 | }); |
| 992 | |
| 993 | await vi.waitUntil( |
| 994 | async () => { |
| 995 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 996 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 997 | }, |
| 998 | { timeout: 5000 }, |
| 999 | ); |
| 1000 | |
| 1001 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1002 | const stepLog = logs.logs.find((val) => val.event === InstanceEvent.STEP_SUCCESS); |
| 1003 | // readLogs() should fall back to a binary size summary |
| 1004 | expect(stepLog?.metadata.result).toBe( |
| 1005 | `[ReadableStream (binary): ${invalidUtf8.byteLength} bytes]`, |
| 1006 | ); |
| 1007 | }); |
| 1008 | |
| 1009 | it("should time out during stream write and roll back", async ({ expect }) => { |
| 1010 | // In the full workflow path a stream-write timeout triggers retries |
| 1011 | // (default limit: 5, exponential backoff from 1 s), so the test would |
| 1012 | // exceed its timeout before WORKFLOW_FAILURE is ever logged. Testing |
| 1013 | // writeStreamOutput() directly lets us verify timeout + rollback |
| 1014 | // behaviour without retry overhead. |
| 1015 | |
| 1016 | const engineId = env.ENGINE.idFromName("STREAM-TIMEOUT"); |
| 1017 | const engineStub = env.ENGINE.get(engineId); |
| 1018 | |
| 1019 | const cacheKey = "timeout-stream-test"; |
| 1020 | |
| 1021 | await runInDurableObject(engineStub, async (_engine, state) => { |
| 1022 | // eslint-disable-next-line workers-sdk/no-unsafe-command-execution -- DO SQL exec, not child_process |
| 1023 | state.storage.sql.exec(` |
| 1024 | CREATE TABLE IF NOT EXISTS ${STREAMING_STEP_CHUNKS_TABLE} ( |
| 1025 | cache_key TEXT NOT NULL, |
| 1026 | attempt INTEGER NOT NULL, |
| 1027 | chunk_index INTEGER NOT NULL, |
| 1028 | chunk BLOB NOT NULL, |
| 1029 | PRIMARY KEY (cache_key, attempt, chunk_index) |
| 1030 | ) WITHOUT ROWID |
| 1031 | `); |
| 1032 | |
| 1033 | const abortController = new AbortController(); |
| 1034 | |
| 1035 | // Create a timeout promise that rejects after 1 second, mirroring |
| 1036 | // the timeoutPromise() used by Context.do() in production. |
| 1037 | const timeoutTask = new Promise<never>((_, reject) => { |
| 1038 | setTimeout(() => { |
| 1039 | const error = new WorkflowTimeoutError("Execution timed out after 1000ms"); |
| 1040 | abortController.abort(error); |
| 1041 | reject(error); |
| 1042 | }, 1000); |
| 1043 | }); |
| 1044 | |
| 1045 | // Stream that never closes — each pull emits a small chunk then |
| 1046 | // waits longer than the timeout. |
| 1047 | const stream = new ReadableStream<Uint8Array>({ |
| 1048 | async pull(controller) { |
| 1049 | controller.enqueue(encodeUtf8("chunk ")); |
| 1050 | await new Promise((resolve) => setTimeout(resolve, 2000)); |
| 1051 | }, |
| 1052 | }); |
| 1053 | |
| 1054 | try { |
| 1055 | await writeStreamOutput({ |
| 1056 | storage: state.storage, |
| 1057 | cacheKey, |
| 1058 | attempt: 1, |
| 1059 | stream, |
| 1060 | signal: abortController.signal, |
| 1061 | timeoutTask, |
| 1062 | }); |
| 1063 | expect.unreachable("writeStreamOutput should have thrown"); |
| 1064 | } catch (e) { |
| 1065 | expect(e).toBeInstanceOf(WorkflowTimeoutError); |
| 1066 | } |
| 1067 | |
| 1068 | // Mimic the rollback that context.ts performs in the outer catch |
| 1069 | // (line ~704) when a stream write times out. |
| 1070 | await rollbackStreamOutput(state.storage, cacheKey, 1); |
| 1071 | |
| 1072 | // Verify that all chunks and metadata were cleaned up |
| 1073 | const chunkCount = countStreamOutputChunks(state, cacheKey); |
| 1074 | expect(chunkCount).toBe(0); |
| 1075 | |
| 1076 | const metaKey = getStreamOutputMetaKey(cacheKey); |
| 1077 | const meta = await state.storage.get(metaKey); |
| 1078 | expect(meta).toBeUndefined(); |
| 1079 | }); |
| 1080 | }); |
| 1081 | }); |
| 1082 | |
| 1083 | describe("Context - typed-array step outputs (issue #14101)", () => { |
| 1084 | it("should persist a 200KB Uint8Array step output on a tight backing buffer", async ({ |
| 1085 | expect, |
| 1086 | }) => { |
| 1087 | // Baseline: a 200KB view sized exactly to its backing buffer must |
| 1088 | // succeed. This case worked on `main` too — anchors the regression test |
| 1089 | // suite to "small enough to fit, regardless of normalisation". |
| 1090 | const engineStub = await runWorkflow("UINT8-TIGHT-200K", async (_event, step) => { |
| 1091 | return await step.do("emit-tight-bytes", async () => { |
| 1092 | return new Uint8Array(200_000); |
| 1093 | }); |
| 1094 | }); |
| 1095 | |
| 1096 | await vi.waitUntil( |
| 1097 | async () => { |
| 1098 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1099 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1100 | }, |
| 1101 | { timeout: 10000 }, |
| 1102 | ); |
| 1103 | |
| 1104 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1105 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true); |
| 1106 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false); |
| 1107 | }); |
| 1108 | |
| 1109 | it("should persist a 200KB Uint8Array view sliced from an 800KB backing buffer (regression #14101)", async ({ |
| 1110 | expect, |
| 1111 | }) => { |
| 1112 | // The exact reporter's repro: a Uint8Array view sized 200KB but |
| 1113 | // sliced from a much larger backing ArrayBuffer (800KB here). Prior |
| 1114 | // to the fix, workerd's v8::ValueSerializer would serialise the |
| 1115 | // entire 800KB backing buffer along with the view, blowing past the |
| 1116 | // 1MiB SQL blob limit and surfacing a raw `SQLITE_TOOBIG` error. |
| 1117 | const engineStub = await runWorkflow("UINT8-SLICED-200K-OF-800K", async (_event, step) => { |
| 1118 | return await step.do("emit-sliced-bytes", async () => { |
| 1119 | const backing = new ArrayBuffer(800_000); |
| 1120 | return new Uint8Array(backing, 0, 200_000); |
| 1121 | }); |
| 1122 | }); |
| 1123 | |
| 1124 | await vi.waitUntil( |
| 1125 | async () => { |
| 1126 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1127 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1128 | }, |
| 1129 | { timeout: 10000 }, |
| 1130 | ); |
| 1131 | |
| 1132 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1133 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true); |
| 1134 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false); |
| 1135 | }); |
| 1136 | |
| 1137 | it("should return the original Uint8Array to the next step in the live execution path", async ({ |
| 1138 | expect, |
| 1139 | }) => { |
| 1140 | // The persisted form preserves the view type (`Uint8Array` stays |
| 1141 | // `Uint8Array`) so the live execution path and cached replays observe |
| 1142 | // the same shape — downstream user code that branches on |
| 1143 | // `value instanceof Uint8Array` works either way. |
| 1144 | let observedInNext: unknown = "not-seen"; |
| 1145 | const engineStub = await runWorkflow("UINT8-ROUND-TRIP", async (_event, step) => { |
| 1146 | const bytes = await step.do("emit-bytes", async () => { |
| 1147 | return new Uint8Array([1, 2, 3, 4, 5]); |
| 1148 | }); |
| 1149 | await step.do("read-bytes", async () => { |
| 1150 | observedInNext = bytes; |
| 1151 | return "ok"; |
| 1152 | }); |
| 1153 | }); |
| 1154 | |
| 1155 | await vi.waitUntil( |
| 1156 | async () => { |
| 1157 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1158 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1159 | }, |
| 1160 | { timeout: 10000 }, |
| 1161 | ); |
| 1162 | |
| 1163 | expect(observedInNext).toBeInstanceOf(Uint8Array); |
| 1164 | expect(Array.from(observedInNext as Uint8Array)).toEqual([1, 2, 3, 4, 5]); |
| 1165 | }); |
| 1166 | |
| 1167 | it("should preserve typed-array view types other than Uint8Array on the live execution path", async ({ |
| 1168 | expect, |
| 1169 | }) => { |
| 1170 | // Persisting must preserve the constructor type (`Int16Array` stays |
| 1171 | // `Int16Array`, etc.) so downstream `instanceof` checks behave the |
| 1172 | // same whether the step ran live or via cached replay. |
| 1173 | let observedInNext: unknown = "not-seen"; |
| 1174 | const engineStub = await runWorkflow("INT16-ROUND-TRIP", async (_event, step) => { |
| 1175 | const bytes = await step.do("emit-int16", async () => { |
| 1176 | return new Int16Array([1, 2, 3, 4, 5]); |
| 1177 | }); |
| 1178 | await step.do("read-int16", async () => { |
| 1179 | observedInNext = bytes; |
| 1180 | return "ok"; |
| 1181 | }); |
| 1182 | }); |
| 1183 | |
| 1184 | await vi.waitUntil( |
| 1185 | async () => { |
| 1186 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1187 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1188 | }, |
| 1189 | { timeout: 10000 }, |
| 1190 | ); |
| 1191 | |
| 1192 | expect(observedInNext).toBeInstanceOf(Int16Array); |
| 1193 | expect(Array.from(observedInNext as Int16Array)).toEqual([1, 2, 3, 4, 5]); |
| 1194 | }); |
| 1195 | |
| 1196 | it("should compact typed-array views nested inside an object (deep-walk regression)", async ({ |
| 1197 | expect, |
| 1198 | }) => { |
| 1199 | // A view sliced from a much larger backing buffer would bloat past |
| 1200 | // SQLITE_TOOBIG if nested-level normalisation were missing. Recurses |
| 1201 | // into plain objects. |
| 1202 | const engineStub = await runWorkflow("NESTED-UINT8-SLICED", async (_event, step) => { |
| 1203 | return await step.do("emit-nested-sliced-bytes", async () => { |
| 1204 | const backing = new ArrayBuffer(800_000); |
| 1205 | return { |
| 1206 | image: new Uint8Array(backing, 0, 200_000), |
| 1207 | meta: { width: 100, height: 100 }, |
| 1208 | }; |
| 1209 | }); |
| 1210 | }); |
| 1211 | |
| 1212 | await vi.waitUntil( |
| 1213 | async () => { |
| 1214 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1215 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1216 | }, |
| 1217 | { timeout: 10000 }, |
| 1218 | ); |
| 1219 | |
| 1220 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1221 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true); |
| 1222 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false); |
| 1223 | }); |
| 1224 | |
| 1225 | it("should compact typed-array views nested deep inside arrays of objects", async ({ |
| 1226 | expect, |
| 1227 | }) => { |
| 1228 | // Two levels of nesting (array of object of view). Confirms the |
| 1229 | // walker descends through both arrays and plain objects. |
| 1230 | const engineStub = await runWorkflow( |
| 1231 | "ARRAY-OF-OBJECTS-WITH-SLICED-VIEWS", |
| 1232 | async (_event, step) => { |
| 1233 | return await step.do("emit-array-of-objects", async () => { |
| 1234 | const backing = new ArrayBuffer(800_000); |
| 1235 | return [ |
| 1236 | { data: new Uint8Array(backing, 0, 100_000) }, |
| 1237 | { data: new Uint8Array(backing, 100_000, 100_000) }, |
| 1238 | ]; |
| 1239 | }); |
| 1240 | }, |
| 1241 | ); |
| 1242 | |
| 1243 | await vi.waitUntil( |
| 1244 | async () => { |
| 1245 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1246 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1247 | }, |
| 1248 | { timeout: 10000 }, |
| 1249 | ); |
| 1250 | |
| 1251 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1252 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS)).toBe(true); |
| 1253 | expect(logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_FAILURE)).toBe(false); |
| 1254 | }); |
| 1255 | }); |
| 1256 | |
| 1257 | describe("Sensitive step output", () => { |
| 1258 | it("should redact a sensitive step's output in logs while passing the real value downstream", async ({ |
| 1259 | expect, |
| 1260 | }) => { |
| 1261 | let downstreamValue: unknown; |
| 1262 | |
| 1263 | const engineStub = await runWorkflowAndAwait("SENSITIVE-STEP-OUTPUT", async (_event, step) => { |
| 1264 | const secret = await step.do("sensitive step", { sensitive: "output" }, async () => { |
| 1265 | return { token: "super-secret" }; |
| 1266 | }); |
| 1267 | await step.do("downstream step", async () => { |
| 1268 | downstreamValue = secret; |
| 1269 | return "ok"; |
| 1270 | }); |
| 1271 | }); |
| 1272 | |
| 1273 | await vi.waitUntil( |
| 1274 | async () => { |
| 1275 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1276 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1277 | }, |
| 1278 | { timeout: 5000 }, |
| 1279 | ); |
| 1280 | |
| 1281 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1282 | const stepLog = logs.logs.find( |
| 1283 | (val) => val.event === InstanceEvent.STEP_SUCCESS && val.target === "sensitive step-1", |
| 1284 | ); |
| 1285 | expect(stepLog?.metadata.result).toBe(REDACTED_STEP_OUTPUT); |
| 1286 | |
| 1287 | // The real value is still cached and handed to the running workflow. |
| 1288 | expect(downstreamValue).toEqual({ token: "super-secret" }); |
| 1289 | }); |
| 1290 | |
| 1291 | it("should redact a sensitive step's output from waitForStepResult", async ({ expect }) => { |
| 1292 | const engineStub = await runWorkflowAndAwait( |
| 1293 | "SENSITIVE-STEP-WAIT-RESULT", |
| 1294 | async (_event, step) => { |
| 1295 | await step.do("sensitive step", { sensitive: "output" }, async () => "super-secret"); |
| 1296 | }, |
| 1297 | ); |
| 1298 | |
| 1299 | await vi.waitUntil( |
| 1300 | async () => { |
| 1301 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1302 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1303 | }, |
| 1304 | { timeout: 5000 }, |
| 1305 | ); |
| 1306 | |
| 1307 | const result = await engineStub.waitForStepResult("sensitive step"); |
| 1308 | expect(result).toBe(REDACTED_STEP_OUTPUT); |
| 1309 | }); |
| 1310 | |
| 1311 | it("should not redact a sensitive step's error", async ({ expect }) => { |
| 1312 | const engineStub = await runWorkflowAndAwait("SENSITIVE-STEP-ERROR", async (_event, step) => { |
| 1313 | try { |
| 1314 | await step.do( |
| 1315 | "sensitive failing step", |
| 1316 | { sensitive: "output", retries: { limit: 0, delay: 0 } }, |
| 1317 | async () => { |
| 1318 | throw new NonRetryableError("boom with secret context"); |
| 1319 | }, |
| 1320 | ); |
| 1321 | } catch {} |
| 1322 | }); |
| 1323 | |
| 1324 | await vi.waitUntil( |
| 1325 | async () => { |
| 1326 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1327 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1328 | }, |
| 1329 | { timeout: 5000 }, |
| 1330 | ); |
| 1331 | |
| 1332 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1333 | const attemptFailure = logs.logs.find((val) => val.event === InstanceEvent.ATTEMPT_FAILURE); |
| 1334 | const error = attemptFailure?.metadata.error as { message: string } | undefined; |
| 1335 | expect(error?.message).toContain("boom with secret context"); |
| 1336 | }); |
| 1337 | |
| 1338 | it("should redact a sensitive streaming step output in readLogs", async ({ expect }) => { |
| 1339 | const payload = "streamed secret"; |
| 1340 | const payloadBytes = encodeUtf8(payload); |
| 1341 | |
| 1342 | const engineStub = await runWorkflowAndAwait( |
| 1343 | "SENSITIVE-STREAM-OUTPUT", |
| 1344 | async (_event, step) => { |
| 1345 | const stream = await step.do("sensitive stream step", { sensitive: "output" }, async () => { |
| 1346 | return new ReadableStream<Uint8Array>({ |
| 1347 | start(controller) { |
| 1348 | controller.enqueue(payloadBytes); |
| 1349 | controller.close(); |
| 1350 | }, |
| 1351 | }); |
| 1352 | }); |
| 1353 | const bytes = await readStreamBytes(stream as ReadableStream<Uint8Array>); |
| 1354 | return decodeUtf8(bytes); |
| 1355 | }, |
| 1356 | ); |
| 1357 | |
| 1358 | await vi.waitUntil( |
| 1359 | async () => { |
| 1360 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1361 | return logs.logs.some((val) => val.event === InstanceEvent.WORKFLOW_SUCCESS); |
| 1362 | }, |
| 1363 | { timeout: 5000 }, |
| 1364 | ); |
| 1365 | |
| 1366 | const logs = (await engineStub.readLogs()) as EngineLogs; |
| 1367 | const stepLog = logs.logs.find( |
| 1368 | (val) => val.event === InstanceEvent.STEP_SUCCESS && val.target === "sensitive stream step-1", |
| 1369 | ); |
| 1370 | expect(stepLog?.metadata.result).toBe(REDACTED_STEP_OUTPUT); |
| 1371 | }); |
| 1372 | }); |