The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as Cloudflare from "@alchemy.run/cloudflare"; |
| 2 | import { describe, expect } from "@alchemy.run/test"; |
| 3 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 4 | import * as Alchemy from "alchemy"; |
| 5 | import * as Data from "effect/Data"; |
| 6 | import * as Effect from "effect/Effect"; |
| 7 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 8 | import * as Schedule from "effect/Schedule"; |
| 9 | import { requestWorker, waitUntilStable } from "../Utils/WorkerRequest.ts"; |
| 10 | import LifecycleWorker, { type Scenario } from "./fixtures/workflow-lifecycle/worker.ts"; |
| 11 | |
| 12 | interface Status { |
| 13 | status: string; |
| 14 | output?: string[]; |
| 15 | error?: { name?: string; message?: string } | null; |
| 16 | rollback?: { outcome: string } | null; |
| 17 | entries: string[]; |
| 18 | } |
| 19 | |
| 20 | // Tagged rather than a plain `Error`: a plain `Error` in the same union |
| 21 | // absorbs `WorkerNotPropagated` (a structural subtype), hiding it from |
| 22 | // `catchTag`. |
| 23 | class WorkerRequestFailed extends Data.TaggedError("WorkerRequestFailed")<{ |
| 24 | readonly message: string; |
| 25 | }> {} |
| 26 | |
| 27 | const request = Effect.fn(function* (url: string, method: "GET" | "POST" = "GET") { |
| 28 | const response = yield* requestWorker( |
| 29 | method === "GET" ? HttpClientRequest.get(url) : HttpClientRequest.post(url), |
| 30 | ); |
| 31 | const body = yield* response.text; |
| 32 | if (response.status !== 200) { |
| 33 | return yield* Effect.fail( |
| 34 | new WorkerRequestFailed({ message: `${method} ${url}: ${response.status}: ${body}` }), |
| 35 | ); |
| 36 | } |
| 37 | return body; |
| 38 | }); |
| 39 | |
| 40 | const waitForReady = Effect.fn(function* (url: string) { |
| 41 | const readyOnce = Effect.gen(function* () { |
| 42 | const response = yield* requestWorker(HttpClientRequest.get(url), { retryDelay: "3 seconds" }); |
| 43 | const body = yield* response.text; |
| 44 | if ( |
| 45 | response.status === 503 && |
| 46 | response.headers["x-workflow-lifecycle-readiness"] === "journal-rpc-not-ready" && |
| 47 | body === "LifecycleJournal.entries not ready" |
| 48 | ) { |
| 49 | yield* Effect.logInfo(`Workflow readiness: native journal RPC not ready at ${url}`); |
| 50 | return false; |
| 51 | } |
| 52 | if (response.status !== 200) { |
| 53 | return yield* Effect.fail( |
| 54 | new WorkerRequestFailed({ message: `GET ${url}: ${response.status}: ${body}` }), |
| 55 | ); |
| 56 | } |
| 57 | if (body === "Alchemy worker is being deployed...") return false; |
| 58 | expect(body).toBe("ready"); |
| 59 | return true; |
| 60 | }).pipe(Effect.catchTag("WorkerNotPropagated", () => Effect.succeed(false))); |
| 61 | // One ready answer only proves one host has the new version; scenarios |
| 62 | // that land on another host see `entries` missing from the journal RPC. |
| 63 | yield* waitUntilStable(`journal RPC at ${url}`, readyOnce, { timeout: "60 seconds" }); |
| 64 | }); |
| 65 | |
| 66 | const probeWorkflow = Effect.fn(function* (url: string, className = "LifecycleWorkflow") { |
| 67 | const probe = Effect.gen(function* () { |
| 68 | const started = yield* request(`${url}/probe`, "POST").pipe( |
| 69 | // Workflows' control plane still failing after the fixture's own |
| 70 | // retries: not ready yet, so let the next round try again. |
| 71 | Effect.catchTag("WorkerRequestFailed", (error) => |
| 72 | error.message.includes("WorkflowControlUnavailable") |
| 73 | ? Effect.succeed(undefined) |
| 74 | : Effect.fail(error), |
| 75 | ), |
| 76 | ); |
| 77 | if (started === undefined) return false; |
| 78 | const { id } = yield* Effect.try(() => JSON.parse(started) as { id: string }); |
| 79 | const status = yield* request(`${url}/probe/${id}`).pipe( |
| 80 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Omit<Status, "entries">)), |
| 81 | Effect.repeat({ |
| 82 | schedule: Schedule.spaced("1 second"), |
| 83 | times: 10, |
| 84 | until: (status) => status.status === "complete" || status.status === "errored", |
| 85 | }), |
| 86 | ); |
| 87 | yield* Effect.logInfo(`Workflow readiness probe ${id}: ${JSON.stringify(status)}`); |
| 88 | // A brand-new script reaches Workflow hosts one at a time, and Cloudflare |
| 89 | // exposes no propagation API. Only a probe may observe the script (or its |
| 90 | // entrypoint) as absent; scenario instances never retry. |
| 91 | // An instance still queued after polling has not been picked up yet, and |
| 92 | // one that hit a Workflows "internal error; reference = <id>" says nothing |
| 93 | // about the script: both just mean "not ready yet". |
| 94 | if (status.status === "queued") return false; |
| 95 | if (status.status === "errored" && status.error?.message?.startsWith("internal error")) { |
| 96 | return false; |
| 97 | } |
| 98 | if ( |
| 99 | status.status === "errored" && |
| 100 | ((status.error?.name === "Error" && status.error.message === "Worker not found.") || |
| 101 | (status.error?.name === "TypeError" && |
| 102 | status.error.message === |
| 103 | `The entrypoint name ${className} was not found in this worker. Ensure the worker exports an entrypoint with that name.`)) |
| 104 | ) { |
| 105 | return false; |
| 106 | } |
| 107 | expect(status, JSON.stringify(status)).toMatchObject({ |
| 108 | status: "complete", |
| 109 | output: ["workflow-ready"], |
| 110 | }); |
| 111 | return true; |
| 112 | }).pipe(Effect.catchTag("WorkerNotPropagated", () => Effect.succeed(false))); |
| 113 | // Each probe is a fresh instance that may land on a different host, so a |
| 114 | // round passes only when every probe completes, and three rounds in a row |
| 115 | // must pass before the ~30 scenarios fan out across hosts. |
| 116 | const round = Effect.all( |
| 117 | Array.from({ length: 6 }, () => probe), |
| 118 | { concurrency: "unbounded" }, |
| 119 | ).pipe(Effect.map((results) => results.every(Boolean))); |
| 120 | yield* waitUntilStable(`Workflow ${className} at ${url}`, round, { |
| 121 | consecutive: 3, |
| 122 | timeout: "90 seconds", |
| 123 | }); |
| 124 | }); |
| 125 | |
| 126 | const cases: Array<{ scenario: Scenario; entries: string[] }> = [ |
| 127 | ...( |
| 128 | [ |
| 129 | "unsupported-function", |
| 130 | "unsupported-symbol", |
| 131 | "unsupported-cycle", |
| 132 | "unsupported-size", |
| 133 | "unsupported-array", |
| 134 | "unsupported-accessor", |
| 135 | "unsupported-tag-accessor", |
| 136 | "unsupported-alias", |
| 137 | ] as const |
| 138 | ).map((scenario) => ({ scenario, entries: ["open:1:captured:true", "close:1"] })), |
| 139 | { scenario: "success", entries: ["open:1:captured:true", "close:1", "after-task"] }, |
| 140 | { |
| 141 | scenario: "retry", |
| 142 | entries: ["open:1:captured:true", "close:1", "open:2:captured:true", "close:2", "after-task"], |
| 143 | }, |
| 144 | { |
| 145 | scenario: "exhaustion", |
| 146 | entries: [ |
| 147 | "open:1:captured:true", |
| 148 | "close:1", |
| 149 | "open:2:captured:true", |
| 150 | "close:2", |
| 151 | "caught:application:2:true", |
| 152 | "after-task", |
| 153 | ], |
| 154 | }, |
| 155 | { |
| 156 | scenario: "uncaught", |
| 157 | entries: ["open:1:captured:true", "close:1", "open:2:captured:true", "close:2"], |
| 158 | }, |
| 159 | { scenario: "die", entries: ["open:1:captured:true", "close:1"] }, |
| 160 | { scenario: "orDie", entries: ["open:1:captured:true", "close:1"] }, |
| 161 | { scenario: "interrupt", entries: ["open:1:captured:true", "close:1", "joined", "after-task"] }, |
| 162 | { |
| 163 | scenario: "interrupt-retry", |
| 164 | entries: ["open:1:captured:true", "close:1", "joined", "after-task"], |
| 165 | }, |
| 166 | { |
| 167 | scenario: "rollback-retry", |
| 168 | entries: [ |
| 169 | "open:1:captured:true", |
| 170 | "close:1", |
| 171 | "after-task", |
| 172 | "rollback-open:captured:true:1", |
| 173 | "rollback-body:1", |
| 174 | "rollback-close:1", |
| 175 | "rollback-open:captured:true:2", |
| 176 | "rollback-body:2", |
| 177 | "rollback-close:2", |
| 178 | ], |
| 179 | }, |
| 180 | ...(["rollback-die", "rollback-orDie"] as const).map((scenario) => ({ |
| 181 | scenario, |
| 182 | entries: [ |
| 183 | "open:1:captured:true", |
| 184 | "close:1", |
| 185 | "after-task", |
| 186 | "rollback-open:captured:true:1", |
| 187 | "rollback-body:1", |
| 188 | "rollback-close:1", |
| 189 | ], |
| 190 | })), |
| 191 | { |
| 192 | scenario: "rollback", |
| 193 | entries: [ |
| 194 | "open:1:captured:true", |
| 195 | "close:1", |
| 196 | "after-task", |
| 197 | "rollback-open:captured:true", |
| 198 | "rollback-body", |
| 199 | "rollback-close", |
| 200 | ], |
| 201 | }, |
| 202 | ]; |
| 203 | |
| 204 | describe.concurrent.each([ |
| 205 | { dev: true, stage: `${Test.defaultStage()}-workflow-lifecycle-local` }, |
| 206 | { dev: false, stage: `${Test.defaultStage()}-workflow-lifecycle-live` }, |
| 207 | ])( |
| 208 | "Workflow lifecycle (dev: $dev)", |
| 209 | ({ dev, stage }) => { |
| 210 | const state = dev ? Alchemy.inMemoryState() : Cloudflare.state(); |
| 211 | const Stack = Alchemy.Stack( |
| 212 | "WorkflowLifecycleStack", |
| 213 | { providers: Cloudflare.providers(), state }, |
| 214 | Effect.gen(function* () { |
| 215 | const worker = yield* LifecycleWorker; |
| 216 | return { url: worker.url.as<string>() }; |
| 217 | }), |
| 218 | ); |
| 219 | const { test, beforeAll, afterAll, deploy, destroy } = Test.make({ |
| 220 | providers: Cloudflare.providers(), |
| 221 | state, |
| 222 | dev, |
| 223 | stage, |
| 224 | }); |
| 225 | const stack = beforeAll( |
| 226 | Effect.gen(function* () { |
| 227 | yield* destroy(Stack); |
| 228 | const output = yield* deploy(Stack); |
| 229 | yield* waitForReady(`${output.url}/ready`); |
| 230 | yield* probeWorkflow(output.url); |
| 231 | return output; |
| 232 | }), |
| 233 | { timeout: 240_000 }, |
| 234 | ); |
| 235 | afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack), { timeout: 30_000 }); |
| 236 | |
| 237 | for (const [kind, message] of [ |
| 238 | ["missing", 'The RPC receiver does not implement the method "entries".'], |
| 239 | ["internal", "internal error; reference = application"], |
| 240 | ]) { |
| 241 | test( |
| 242 | `does not mask an application ${kind} failure as readiness`, |
| 243 | Effect.gen(function* () { |
| 244 | const { url } = yield* stack; |
| 245 | const failure = yield* waitForReady(`${url}/ready?application-error=${kind}`).pipe( |
| 246 | Effect.flip, |
| 247 | ); |
| 248 | expect(failure.message).toContain(": 500:"); |
| 249 | expect(failure.message).toContain(message); |
| 250 | expect(failure.message).not.toContain("LifecycleJournal.entries not ready"); |
| 251 | }), |
| 252 | { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 }, |
| 253 | ); |
| 254 | } |
| 255 | |
| 256 | test( |
| 257 | "repeated starts keep the same instance and execute the scenario once", |
| 258 | Effect.gen(function* () { |
| 259 | const { url } = yield* stack; |
| 260 | const startUrl = `${url}/start/success?id=repeated-start`; |
| 261 | const first = yield* request(startUrl, "POST"); |
| 262 | expect(yield* request(startUrl, "POST")).toBe(first); |
| 263 | const { id } = yield* Effect.try(() => JSON.parse(first) as { id: string }); |
| 264 | expect(id).toBe("repeated-start"); |
| 265 | const status = yield* request(`${url}/status/${id}`).pipe( |
| 266 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)), |
| 267 | Effect.repeat({ |
| 268 | schedule: Schedule.spaced("1 second"), |
| 269 | times: 10, |
| 270 | until: (status) => status.status === "complete", |
| 271 | }), |
| 272 | ); |
| 273 | expect(status).toMatchObject({ |
| 274 | status: "complete", |
| 275 | entries: ["open:1:captured:true", "close:1", "after-task"], |
| 276 | }); |
| 277 | }), |
| 278 | { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 }, |
| 279 | ); |
| 280 | |
| 281 | test( |
| 282 | "preserves native validation failures when starting an instance", |
| 283 | Effect.gen(function* () { |
| 284 | const { url } = yield* stack; |
| 285 | const response = yield* requestWorker( |
| 286 | HttpClientRequest.post(`${url}/start/success?id=invalid%2Fid`), |
| 287 | ); |
| 288 | expect(response.status).toBe(500); |
| 289 | const body = yield* response.text; |
| 290 | expect(body).not.toContain("WorkflowControlUnavailable"); |
| 291 | expect(body.toLowerCase()).toContain("invalid"); |
| 292 | }), |
| 293 | { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 }, |
| 294 | ); |
| 295 | |
| 296 | for (const scenario of [ |
| 297 | "replay", |
| 298 | "replay-inherited", |
| 299 | "replay-terminal-die", |
| 300 | "replay-terminal-orDie", |
| 301 | ] as const) { |
| 302 | test( |
| 303 | `preserves ${scenario} error semantics after checkpoint replay`, |
| 304 | Effect.gen(function* () { |
| 305 | const { url } = yield* stack; |
| 306 | const terminal = scenario.startsWith("replay-terminal"); |
| 307 | const caught = terminal ? "caught:terminal" : "caught:application:2"; |
| 308 | const { id } = yield* request( |
| 309 | `${url}/start/${scenario}?stage=${encodeURIComponent(stage)}`, |
| 310 | "POST", |
| 311 | ).pipe(Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as { id: string }))); |
| 312 | const recovered = yield* request(`${url}/journal/${id}`).pipe( |
| 313 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])), |
| 314 | Effect.repeat({ |
| 315 | schedule: Schedule.spaced("1 second"), |
| 316 | times: 10, |
| 317 | until: (entries) => entries.includes("checkpoint"), |
| 318 | }), |
| 319 | ); |
| 320 | if (!recovered.includes(`${caught}:true`)) { |
| 321 | yield* Effect.logInfo( |
| 322 | "Workflow replay recovery status", |
| 323 | yield* request(`${url}/status/${id}`), |
| 324 | ); |
| 325 | } |
| 326 | expect(recovered).toContain(`${caught}:true`); |
| 327 | // A journal write does not acknowledge native run completion. |
| 328 | const completed = yield* request(`${url}/status/${id}`).pipe( |
| 329 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)), |
| 330 | Effect.repeat({ |
| 331 | schedule: Schedule.spaced("1 second"), |
| 332 | times: 10, |
| 333 | until: (status) => status.status === "complete" || status.status === "errored", |
| 334 | }), |
| 335 | ); |
| 336 | expect(completed, JSON.stringify(completed)).toMatchObject({ |
| 337 | status: "complete", |
| 338 | output: recovered, |
| 339 | }); |
| 340 | yield* request(`${url}/restart/${id}`, "POST"); |
| 341 | const journal = yield* request(`${url}/journal/${id}`).pipe( |
| 342 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])), |
| 343 | Effect.repeat({ |
| 344 | schedule: Schedule.spaced("1 second"), |
| 345 | times: 10, |
| 346 | until: (entries) => entries.includes("after-task"), |
| 347 | }), |
| 348 | ); |
| 349 | yield* Effect.logInfo(`Replay journal ${id}: ${JSON.stringify(journal)}`); |
| 350 | if (!journal.includes("after-task")) { |
| 351 | yield* Effect.logInfo( |
| 352 | "Workflow checkpoint restart status", |
| 353 | yield* request(`${url}/status/${id}`), |
| 354 | ); |
| 355 | } |
| 356 | expect(journal).toContain("after-task"); |
| 357 | // Native terminal failures may rerun on an explicit checkpoint restart. |
| 358 | const reranTerminal = terminal && journal.includes("open:3:captured:true"); |
| 359 | expect(journal).toContain(`${caught}:${reranTerminal}`); |
| 360 | const replayed = yield* request(`${url}/status/${id}`).pipe( |
| 361 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)), |
| 362 | Effect.repeat({ |
| 363 | schedule: Schedule.spaced("1 second"), |
| 364 | times: 10, |
| 365 | until: (status) => status.status === "complete" || status.status === "errored", |
| 366 | }), |
| 367 | ); |
| 368 | expect(replayed, JSON.stringify(replayed)).toMatchObject({ |
| 369 | status: "complete", |
| 370 | entries: [ |
| 371 | "run", |
| 372 | "open:1:captured:true", |
| 373 | "close:1", |
| 374 | "open:2:captured:true", |
| 375 | "close:2", |
| 376 | `${caught}:true`, |
| 377 | "checkpoint", |
| 378 | "run", |
| 379 | ...(reranTerminal ? ["open:3:captured:true", "close:3"] : []), |
| 380 | `${caught}:${reranTerminal}`, |
| 381 | "after-task", |
| 382 | ], |
| 383 | }); |
| 384 | }), |
| 385 | { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 }, |
| 386 | ); |
| 387 | } |
| 388 | |
| 389 | for (const { scenario, entries } of cases) { |
| 390 | test( |
| 391 | `closes and owns ${scenario} attempt resources`, |
| 392 | Effect.gen(function* () { |
| 393 | const { url } = yield* stack; |
| 394 | const started = yield* request(`${url}/start/${scenario}`, "POST"); |
| 395 | const { id } = yield* Effect.try(() => JSON.parse(started) as { id: string }); |
| 396 | if (scenario.startsWith("rollback")) { |
| 397 | // Native status() can reject during compensation; observe cleanup first. |
| 398 | const journal = yield* request(`${url}/journal/${id}`).pipe( |
| 399 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])), |
| 400 | // Workflows can take a while to pick up a new instance. |
| 401 | Effect.repeat({ |
| 402 | schedule: Schedule.spaced("2 seconds"), |
| 403 | times: 30, |
| 404 | until: (actual) => actual.includes(entries[entries.length - 1]), |
| 405 | }), |
| 406 | ); |
| 407 | expect(journal).toEqual(entries); |
| 408 | } |
| 409 | const status = yield* request(`${url}/status/${id}`).pipe( |
| 410 | Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)), |
| 411 | Effect.repeat({ |
| 412 | schedule: Schedule.spaced("2 seconds"), |
| 413 | times: 30, |
| 414 | until: (status) => status.status === "complete" || status.status === "errored", |
| 415 | }), |
| 416 | ); |
| 417 | const failed = |
| 418 | scenario.startsWith("rollback") || |
| 419 | scenario.startsWith("unsupported") || |
| 420 | ["uncaught", "die", "orDie"].includes(scenario); |
| 421 | expect(status, JSON.stringify(status)).toMatchObject({ |
| 422 | status: failed ? "errored" : "complete", |
| 423 | entries, |
| 424 | }); |
| 425 | if (scenario.startsWith("rollback")) { |
| 426 | if (!dev) { |
| 427 | expect(status.rollback).toMatchObject({ |
| 428 | outcome: |
| 429 | scenario === "rollback-die" || scenario === "rollback-orDie" |
| 430 | ? "failed" |
| 431 | : "complete", |
| 432 | }); |
| 433 | } |
| 434 | expect(status.error).toMatchObject({ message: "trigger compensation" }); |
| 435 | } else if (scenario.startsWith("unsupported")) { |
| 436 | expect(status.error?.message).toContain( |
| 437 | "Workflow application failure is not serializable", |
| 438 | ); |
| 439 | if (scenario === "unsupported-array") |
| 440 | expect(status.error?.message).toContain("sparse arrays or custom array properties"); |
| 441 | if (scenario === "unsupported-accessor" || scenario === "unsupported-tag-accessor") { |
| 442 | expect(status.error?.message).toContain("accessor error data"); |
| 443 | expect(status.error?.message).not.toContain("getter was invoked"); |
| 444 | } |
| 445 | if (scenario === "unsupported-alias") |
| 446 | expect(status.error?.message).toContain("shared references in error data"); |
| 447 | } else if (failed) { |
| 448 | expect(status.error?.message).toContain("application failure"); |
| 449 | } else { |
| 450 | expect(status.output).toEqual(entries); |
| 451 | } |
| 452 | }), |
| 453 | { tags: [...(dev ? ["local"] : ["live"])], timeout: 120_000 }, |
| 454 | ); |
| 455 | } |
| 456 | }, |
| 457 | { tags: ["provider:cloudflare", "provider:cloudflare:worker", "provider:cloudflare:workflow"] }, |
| 458 | ); |