@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 458 lines 17.8 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Alchemy from "alchemy";
5 import * as Data from "effect/Data";
6 import * as Effect from "effect/Effect";
7 import * as HttpClientRequest from "effect/http/HttpClientRequest";
8 import * as Schedule from "effect/Schedule";
9 import { requestWorker, waitUntilStable } from "../Utils/WorkerRequest.ts";
10 import LifecycleWorker, { type Scenario } from "./fixtures/workflow-lifecycle/worker.ts";
11
12 interface Status {
13 status: string;
14 output?: string[];
15 error?: { name?: string; message?: string } | null;
16 rollback?: { outcome: string } | null;
17 entries: string[];
18 }
19
20 // Tagged rather than a plain `Error`: a plain `Error` in the same union
21 // absorbs `WorkerNotPropagated` (a structural subtype), hiding it from
22 // `catchTag`.
23 class WorkerRequestFailed extends Data.TaggedError("WorkerRequestFailed")<{
24 readonly message: string;
25 }> {}
26
27 const request = Effect.fn(function* (url: string, method: "GET" | "POST" = "GET") {
28 const response = yield* requestWorker(
29 method === "GET" ? HttpClientRequest.get(url) : HttpClientRequest.post(url),
30 );
31 const body = yield* response.text;
32 if (response.status !== 200) {
33 return yield* Effect.fail(
34 new WorkerRequestFailed({ message: `${method} ${url}: ${response.status}: ${body}` }),
35 );
36 }
37 return body;
38 });
39
40 const waitForReady = Effect.fn(function* (url: string) {
41 const readyOnce = Effect.gen(function* () {
42 const response = yield* requestWorker(HttpClientRequest.get(url), { retryDelay: "3 seconds" });
43 const body = yield* response.text;
44 if (
45 response.status === 503 &&
46 response.headers["x-workflow-lifecycle-readiness"] === "journal-rpc-not-ready" &&
47 body === "LifecycleJournal.entries not ready"
48 ) {
49 yield* Effect.logInfo(`Workflow readiness: native journal RPC not ready at ${url}`);
50 return false;
51 }
52 if (response.status !== 200) {
53 return yield* Effect.fail(
54 new WorkerRequestFailed({ message: `GET ${url}: ${response.status}: ${body}` }),
55 );
56 }
57 if (body === "Alchemy worker is being deployed...") return false;
58 expect(body).toBe("ready");
59 return true;
60 }).pipe(Effect.catchTag("WorkerNotPropagated", () => Effect.succeed(false)));
61 // One ready answer only proves one host has the new version; scenarios
62 // that land on another host see `entries` missing from the journal RPC.
63 yield* waitUntilStable(`journal RPC at ${url}`, readyOnce, { timeout: "60 seconds" });
64 });
65
66 const probeWorkflow = Effect.fn(function* (url: string, className = "LifecycleWorkflow") {
67 const probe = Effect.gen(function* () {
68 const started = yield* request(`${url}/probe`, "POST").pipe(
69 // Workflows' control plane still failing after the fixture's own
70 // retries: not ready yet, so let the next round try again.
71 Effect.catchTag("WorkerRequestFailed", (error) =>
72 error.message.includes("WorkflowControlUnavailable")
73 ? Effect.succeed(undefined)
74 : Effect.fail(error),
75 ),
76 );
77 if (started === undefined) return false;
78 const { id } = yield* Effect.try(() => JSON.parse(started) as { id: string });
79 const status = yield* request(`${url}/probe/${id}`).pipe(
80 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Omit<Status, "entries">)),
81 Effect.repeat({
82 schedule: Schedule.spaced("1 second"),
83 times: 10,
84 until: (status) => status.status === "complete" || status.status === "errored",
85 }),
86 );
87 yield* Effect.logInfo(`Workflow readiness probe ${id}: ${JSON.stringify(status)}`);
88 // A brand-new script reaches Workflow hosts one at a time, and Cloudflare
89 // exposes no propagation API. Only a probe may observe the script (or its
90 // entrypoint) as absent; scenario instances never retry.
91 // An instance still queued after polling has not been picked up yet, and
92 // one that hit a Workflows "internal error; reference = <id>" says nothing
93 // about the script: both just mean "not ready yet".
94 if (status.status === "queued") return false;
95 if (status.status === "errored" && status.error?.message?.startsWith("internal error")) {
96 return false;
97 }
98 if (
99 status.status === "errored" &&
100 ((status.error?.name === "Error" && status.error.message === "Worker not found.") ||
101 (status.error?.name === "TypeError" &&
102 status.error.message ===
103 `The entrypoint name ${className} was not found in this worker. Ensure the worker exports an entrypoint with that name.`))
104 ) {
105 return false;
106 }
107 expect(status, JSON.stringify(status)).toMatchObject({
108 status: "complete",
109 output: ["workflow-ready"],
110 });
111 return true;
112 }).pipe(Effect.catchTag("WorkerNotPropagated", () => Effect.succeed(false)));
113 // Each probe is a fresh instance that may land on a different host, so a
114 // round passes only when every probe completes, and three rounds in a row
115 // must pass before the ~30 scenarios fan out across hosts.
116 const round = Effect.all(
117 Array.from({ length: 6 }, () => probe),
118 { concurrency: "unbounded" },
119 ).pipe(Effect.map((results) => results.every(Boolean)));
120 yield* waitUntilStable(`Workflow ${className} at ${url}`, round, {
121 consecutive: 3,
122 timeout: "90 seconds",
123 });
124 });
125
126 const cases: Array<{ scenario: Scenario; entries: string[] }> = [
127 ...(
128 [
129 "unsupported-function",
130 "unsupported-symbol",
131 "unsupported-cycle",
132 "unsupported-size",
133 "unsupported-array",
134 "unsupported-accessor",
135 "unsupported-tag-accessor",
136 "unsupported-alias",
137 ] as const
138 ).map((scenario) => ({ scenario, entries: ["open:1:captured:true", "close:1"] })),
139 { scenario: "success", entries: ["open:1:captured:true", "close:1", "after-task"] },
140 {
141 scenario: "retry",
142 entries: ["open:1:captured:true", "close:1", "open:2:captured:true", "close:2", "after-task"],
143 },
144 {
145 scenario: "exhaustion",
146 entries: [
147 "open:1:captured:true",
148 "close:1",
149 "open:2:captured:true",
150 "close:2",
151 "caught:application:2:true",
152 "after-task",
153 ],
154 },
155 {
156 scenario: "uncaught",
157 entries: ["open:1:captured:true", "close:1", "open:2:captured:true", "close:2"],
158 },
159 { scenario: "die", entries: ["open:1:captured:true", "close:1"] },
160 { scenario: "orDie", entries: ["open:1:captured:true", "close:1"] },
161 { scenario: "interrupt", entries: ["open:1:captured:true", "close:1", "joined", "after-task"] },
162 {
163 scenario: "interrupt-retry",
164 entries: ["open:1:captured:true", "close:1", "joined", "after-task"],
165 },
166 {
167 scenario: "rollback-retry",
168 entries: [
169 "open:1:captured:true",
170 "close:1",
171 "after-task",
172 "rollback-open:captured:true:1",
173 "rollback-body:1",
174 "rollback-close:1",
175 "rollback-open:captured:true:2",
176 "rollback-body:2",
177 "rollback-close:2",
178 ],
179 },
180 ...(["rollback-die", "rollback-orDie"] as const).map((scenario) => ({
181 scenario,
182 entries: [
183 "open:1:captured:true",
184 "close:1",
185 "after-task",
186 "rollback-open:captured:true:1",
187 "rollback-body:1",
188 "rollback-close:1",
189 ],
190 })),
191 {
192 scenario: "rollback",
193 entries: [
194 "open:1:captured:true",
195 "close:1",
196 "after-task",
197 "rollback-open:captured:true",
198 "rollback-body",
199 "rollback-close",
200 ],
201 },
202 ];
203
204 describe.concurrent.each([
205 { dev: true, stage: `${Test.defaultStage()}-workflow-lifecycle-local` },
206 { dev: false, stage: `${Test.defaultStage()}-workflow-lifecycle-live` },
207 ])(
208 "Workflow lifecycle (dev: $dev)",
209 ({ dev, stage }) => {
210 const state = dev ? Alchemy.inMemoryState() : Cloudflare.state();
211 const Stack = Alchemy.Stack(
212 "WorkflowLifecycleStack",
213 { providers: Cloudflare.providers(), state },
214 Effect.gen(function* () {
215 const worker = yield* LifecycleWorker;
216 return { url: worker.url.as<string>() };
217 }),
218 );
219 const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
220 providers: Cloudflare.providers(),
221 state,
222 dev,
223 stage,
224 });
225 const stack = beforeAll(
226 Effect.gen(function* () {
227 yield* destroy(Stack);
228 const output = yield* deploy(Stack);
229 yield* waitForReady(`${output.url}/ready`);
230 yield* probeWorkflow(output.url);
231 return output;
232 }),
233 { timeout: 240_000 },
234 );
235 afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack), { timeout: 30_000 });
236
237 for (const [kind, message] of [
238 ["missing", 'The RPC receiver does not implement the method "entries".'],
239 ["internal", "internal error; reference = application"],
240 ]) {
241 test(
242 `does not mask an application ${kind} failure as readiness`,
243 Effect.gen(function* () {
244 const { url } = yield* stack;
245 const failure = yield* waitForReady(`${url}/ready?application-error=${kind}`).pipe(
246 Effect.flip,
247 );
248 expect(failure.message).toContain(": 500:");
249 expect(failure.message).toContain(message);
250 expect(failure.message).not.toContain("LifecycleJournal.entries not ready");
251 }),
252 { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 },
253 );
254 }
255
256 test(
257 "repeated starts keep the same instance and execute the scenario once",
258 Effect.gen(function* () {
259 const { url } = yield* stack;
260 const startUrl = `${url}/start/success?id=repeated-start`;
261 const first = yield* request(startUrl, "POST");
262 expect(yield* request(startUrl, "POST")).toBe(first);
263 const { id } = yield* Effect.try(() => JSON.parse(first) as { id: string });
264 expect(id).toBe("repeated-start");
265 const status = yield* request(`${url}/status/${id}`).pipe(
266 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)),
267 Effect.repeat({
268 schedule: Schedule.spaced("1 second"),
269 times: 10,
270 until: (status) => status.status === "complete",
271 }),
272 );
273 expect(status).toMatchObject({
274 status: "complete",
275 entries: ["open:1:captured:true", "close:1", "after-task"],
276 });
277 }),
278 { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 },
279 );
280
281 test(
282 "preserves native validation failures when starting an instance",
283 Effect.gen(function* () {
284 const { url } = yield* stack;
285 const response = yield* requestWorker(
286 HttpClientRequest.post(`${url}/start/success?id=invalid%2Fid`),
287 );
288 expect(response.status).toBe(500);
289 const body = yield* response.text;
290 expect(body).not.toContain("WorkflowControlUnavailable");
291 expect(body.toLowerCase()).toContain("invalid");
292 }),
293 { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 },
294 );
295
296 for (const scenario of [
297 "replay",
298 "replay-inherited",
299 "replay-terminal-die",
300 "replay-terminal-orDie",
301 ] as const) {
302 test(
303 `preserves ${scenario} error semantics after checkpoint replay`,
304 Effect.gen(function* () {
305 const { url } = yield* stack;
306 const terminal = scenario.startsWith("replay-terminal");
307 const caught = terminal ? "caught:terminal" : "caught:application:2";
308 const { id } = yield* request(
309 `${url}/start/${scenario}?stage=${encodeURIComponent(stage)}`,
310 "POST",
311 ).pipe(Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as { id: string })));
312 const recovered = yield* request(`${url}/journal/${id}`).pipe(
313 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])),
314 Effect.repeat({
315 schedule: Schedule.spaced("1 second"),
316 times: 10,
317 until: (entries) => entries.includes("checkpoint"),
318 }),
319 );
320 if (!recovered.includes(`${caught}:true`)) {
321 yield* Effect.logInfo(
322 "Workflow replay recovery status",
323 yield* request(`${url}/status/${id}`),
324 );
325 }
326 expect(recovered).toContain(`${caught}:true`);
327 // A journal write does not acknowledge native run completion.
328 const completed = yield* request(`${url}/status/${id}`).pipe(
329 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)),
330 Effect.repeat({
331 schedule: Schedule.spaced("1 second"),
332 times: 10,
333 until: (status) => status.status === "complete" || status.status === "errored",
334 }),
335 );
336 expect(completed, JSON.stringify(completed)).toMatchObject({
337 status: "complete",
338 output: recovered,
339 });
340 yield* request(`${url}/restart/${id}`, "POST");
341 const journal = yield* request(`${url}/journal/${id}`).pipe(
342 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])),
343 Effect.repeat({
344 schedule: Schedule.spaced("1 second"),
345 times: 10,
346 until: (entries) => entries.includes("after-task"),
347 }),
348 );
349 yield* Effect.logInfo(`Replay journal ${id}: ${JSON.stringify(journal)}`);
350 if (!journal.includes("after-task")) {
351 yield* Effect.logInfo(
352 "Workflow checkpoint restart status",
353 yield* request(`${url}/status/${id}`),
354 );
355 }
356 expect(journal).toContain("after-task");
357 // Native terminal failures may rerun on an explicit checkpoint restart.
358 const reranTerminal = terminal && journal.includes("open:3:captured:true");
359 expect(journal).toContain(`${caught}:${reranTerminal}`);
360 const replayed = yield* request(`${url}/status/${id}`).pipe(
361 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)),
362 Effect.repeat({
363 schedule: Schedule.spaced("1 second"),
364 times: 10,
365 until: (status) => status.status === "complete" || status.status === "errored",
366 }),
367 );
368 expect(replayed, JSON.stringify(replayed)).toMatchObject({
369 status: "complete",
370 entries: [
371 "run",
372 "open:1:captured:true",
373 "close:1",
374 "open:2:captured:true",
375 "close:2",
376 `${caught}:true`,
377 "checkpoint",
378 "run",
379 ...(reranTerminal ? ["open:3:captured:true", "close:3"] : []),
380 `${caught}:${reranTerminal}`,
381 "after-task",
382 ],
383 });
384 }),
385 { tags: [...(dev ? ["local"] : ["live"])], timeout: 60_000 },
386 );
387 }
388
389 for (const { scenario, entries } of cases) {
390 test(
391 `closes and owns ${scenario} attempt resources`,
392 Effect.gen(function* () {
393 const { url } = yield* stack;
394 const started = yield* request(`${url}/start/${scenario}`, "POST");
395 const { id } = yield* Effect.try(() => JSON.parse(started) as { id: string });
396 if (scenario.startsWith("rollback")) {
397 // Native status() can reject during compensation; observe cleanup first.
398 const journal = yield* request(`${url}/journal/${id}`).pipe(
399 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as string[])),
400 // Workflows can take a while to pick up a new instance.
401 Effect.repeat({
402 schedule: Schedule.spaced("2 seconds"),
403 times: 30,
404 until: (actual) => actual.includes(entries[entries.length - 1]),
405 }),
406 );
407 expect(journal).toEqual(entries);
408 }
409 const status = yield* request(`${url}/status/${id}`).pipe(
410 Effect.flatMap((body) => Effect.try(() => JSON.parse(body) as Status)),
411 Effect.repeat({
412 schedule: Schedule.spaced("2 seconds"),
413 times: 30,
414 until: (status) => status.status === "complete" || status.status === "errored",
415 }),
416 );
417 const failed =
418 scenario.startsWith("rollback") ||
419 scenario.startsWith("unsupported") ||
420 ["uncaught", "die", "orDie"].includes(scenario);
421 expect(status, JSON.stringify(status)).toMatchObject({
422 status: failed ? "errored" : "complete",
423 entries,
424 });
425 if (scenario.startsWith("rollback")) {
426 if (!dev) {
427 expect(status.rollback).toMatchObject({
428 outcome:
429 scenario === "rollback-die" || scenario === "rollback-orDie"
430 ? "failed"
431 : "complete",
432 });
433 }
434 expect(status.error).toMatchObject({ message: "trigger compensation" });
435 } else if (scenario.startsWith("unsupported")) {
436 expect(status.error?.message).toContain(
437 "Workflow application failure is not serializable",
438 );
439 if (scenario === "unsupported-array")
440 expect(status.error?.message).toContain("sparse arrays or custom array properties");
441 if (scenario === "unsupported-accessor" || scenario === "unsupported-tag-accessor") {
442 expect(status.error?.message).toContain("accessor error data");
443 expect(status.error?.message).not.toContain("getter was invoked");
444 }
445 if (scenario === "unsupported-alias")
446 expect(status.error?.message).toContain("shared references in error data");
447 } else if (failed) {
448 expect(status.error?.message).toContain("application failure");
449 } else {
450 expect(status.output).toEqual(entries);
451 }
452 }),
453 { tags: [...(dev ? ["local"] : ["live"])], timeout: 120_000 },
454 );
455 }
456 },
457 { tags: ["provider:cloudflare", "provider:cloudflare:worker", "provider:cloudflare:workflow"] },
458 );