The best way to control Claude Code, Codex and more from any computer or phone

main
ts 504 lines 18.3 KB
Raw
1 import { createHash } from "node:crypto";
2 import * as Cloudflare from "@alchemy.run/cloudflare";
3 import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
4 import { expect } from "@alchemy.run/test";
5 import * as Test from "@alchemy.run/test/Alchemy";
6 import * as accounts from "@distilled.cloud/cloudflare/accounts";
7 import * as queues from "@distilled.cloud/cloudflare/queues";
8 import * as r2 from "@distilled.cloud/cloudflare/r2";
9 import * as Retry from "@distilled.cloud/cloudflare/Retry";
10 import * as Output from "alchemy/Output";
11 import * as Effect from "effect/Effect";
12 import * as Redacted from "effect/Redacted";
13 import * as Schedule from "effect/Schedule";
14
15 const { test } = Test.make({ providers: Cloudflare.providers() });
16 const missingJobId = "0000000000000000000000000000000000000000000000000000000000000000";
17
18 const program = (options: {
19 job: boolean;
20 paused?: boolean;
21 overwrite?: boolean;
22 reference?: boolean;
23 sourcePrefix?: string;
24 targetName?: string;
25 otherJob?: boolean;
26 subscription?: "direct" | "ref";
27 }) =>
28 Effect.gen(function* () {
29 const { accountId } = yield* yield* CloudflareEnvironment;
30 const source = yield* Cloudflare.R2.Bucket("Source", {
31 forceDestroy: true,
32 });
33 const target = yield* Cloudflare.R2.Bucket("Target", {
34 name: options.targetName,
35 forceDestroy: true,
36 });
37 const token = yield* Cloudflare.ApiToken.AccountApiToken("Credentials", {
38 policies: [
39 {
40 effect: "allow",
41 permissionGroups: ["Workers R2 Storage Read", "Workers R2 Storage Write"],
42 resources: { [`com.cloudflare.api.account.${accountId}`]: "*" },
43 },
44 ],
45 });
46 const secret = {
47 accessKeyId: token.tokenId.pipe(Output.map(Redacted.make)),
48 secretAccessKey: token.value.pipe(
49 Output.mapEffect((value) =>
50 Effect.sync(() =>
51 Redacted.make(createHash("sha256").update(Redacted.value(value)).digest("hex")),
52 ),
53 ),
54 ),
55 };
56 const job = options.job
57 ? yield* Cloudflare.R2.SuperSlurperJob("Migration", {
58 source: {
59 vendor: "r2",
60 bucket: source.bucketName,
61 secret,
62 pathPrefix: options.sourcePrefix,
63 },
64 target: { vendor: "r2", bucket: target.bucketName, secret },
65 paused: options.paused,
66 overwrite: options.overwrite,
67 })
68 : undefined;
69 const otherJob = options.otherJob
70 ? yield* Cloudflare.R2.SuperSlurperJob("OtherMigration", {
71 source: { vendor: "r2", bucket: source.bucketName, secret },
72 target: { vendor: "r2", bucket: target.bucketName, secret },
73 paused: true,
74 })
75 : undefined;
76 const reference = options.reference
77 ? yield* Cloudflare.R2.SuperSlurperJob.ref("Migration")
78 : undefined;
79 const queue = options.subscription
80 ? yield* Cloudflare.Queues.Queue("JobEventsQueue")
81 : undefined;
82 const subscription =
83 options.subscription && job && queue
84 ? yield* Cloudflare.Queues.Subscription("JobEvents", {
85 source:
86 options.subscription === "ref"
87 ? yield* Cloudflare.R2.SuperSlurperJob.ref("Migration")
88 : job,
89 events: ["job.resumed", "job.completed", "job.aborted"],
90 queueId: queue.queueId,
91 })
92 : undefined;
93 return {
94 source,
95 target,
96 token,
97 job,
98 otherJob,
99 reference,
100 queue,
101 subscription,
102 };
103 });
104
105 const getJob = (identity: { accountId: string; jobId: string }) =>
106 r2
107 .getSuperSlurperJob({
108 accountId: identity.accountId,
109 jobId: identity.jobId,
110 })
111 .pipe(Retry.none);
112
113 const terminal = (status: r2.GetSuperSlurperJobResponse["status"]) =>
114 status === "aborted" || status === "completed";
115
116 test.provider(
117 "observes absence and types the API's absent-job mutation failures",
118 (stack) =>
119 Effect.gen(function* () {
120 yield* stack.destroy();
121 const { accountId } = yield* yield* CloudflareEnvironment;
122 const identity = { accountId, jobId: missingJobId };
123 const missing = yield* getJob(identity).pipe(Effect.flip);
124 expect(missing._tag).toBe("SuperSlurperJobOperationFailed");
125 expect(missing.message).toBe("Internal Server Error");
126 const invalidSecret = {
127 accessKeyId: "00000000000000000000000000000000",
128 secretAccessKey: "0000000000000000000000000000000000000000000000000000000000000000",
129 };
130 const rejected = yield* r2
131 .createSuperSlurperJob({
132 accountId,
133 source: {
134 vendor: "r2",
135 bucket: "alchemy-slurper-missing-source",
136 secret: invalidSecret,
137 },
138 target: {
139 vendor: "r2",
140 bucket: "alchemy-slurper-missing-target",
141 secret: invalidSecret,
142 },
143 })
144 .pipe(Retry.none, Effect.flip);
145 expect(rejected._tag).toBe("SuperSlurperPreconnectivityFailed");
146 expect(rejected.message).toBe("Preconnectivity failed, please verify tokens and try again");
147 for (const operation of [
148 r2.abortSuperSlurperJob,
149 r2.pauseSuperSlurperJob,
150 r2.resumeSuperSlurperJob,
151 ]) {
152 const error = yield* operation(identity).pipe(Retry.none, Effect.flip);
153 expect(error._tag).toBe("SuperSlurperJobOperationFailed");
154 expect(error.message).toBe("Internal Server Error");
155 }
156 yield* stack.destroy();
157 }).pipe(Effect.ensuring(stack.destroy().pipe(Effect.orDie))),
158 {
159 tags: ["provider:cloudflare", "provider:cloudflare:r2", "live"],
160 timeout: 120_000,
161 exclusive: "cloudflare:event-subscriptions",
162 },
163 );
164
165 test.provider(
166 "creates, pauses, resumes, preserves terminal identity, replaces and cancels without deleting objects",
167 (stack) =>
168 Effect.gen(function* () {
169 yield* stack.destroy();
170 const { accountId } = yield* yield* CloudflareEnvironment;
171 const buckets = yield* stack.deploy(program({ job: false }));
172 yield* r2.uploadBucketObject({
173 accountId,
174 bucketName: buckets.source.bucketName,
175 objectName: "migration.txt",
176 body: "super-slurper-source",
177 });
178 yield* r2.uploadBucketObject({
179 accountId,
180 bucketName: buckets.target.bucketName,
181 objectName: "sentinel.txt",
182 body: "keep-existing-target-data",
183 });
184 const created = yield* stack.deploy(program({ job: true, paused: true }));
185 const job = created.job!;
186 expect(job.accountId).toBe(accountId);
187 expect(job.jobId).toBeTruthy();
188 const paused = yield* getJob(job);
189 expect(paused.id).toBe(job.jobId);
190 expect(paused.source?.bucket).toBe(buckets.source.bucketName);
191 expect(paused.target?.bucket).toBe(buckets.target.bucketName);
192 expect(paused.status === "paused" || terminal(paused.status)).toBe(true);
193
194 const noop = yield* stack.deploy(program({ job: true, paused: true, reference: true }));
195 expect(noop.job!.jobId).toBe(job.jobId);
196 expect(noop.reference!.jobId).toBe(job.jobId);
197 expect(noop.reference!.accountId).toBe(accountId);
198
199 const resumed = yield* stack.deploy(program({ job: true }));
200 expect(resumed.job!.jobId).toBe(job.jobId);
201 const running = yield* getJob(job);
202 expect(running.status === "running" || terminal(running.status)).toBe(true);
203 if (!terminal(running.status)) {
204 yield* r2.abortSuperSlurperJob({ accountId, jobId: job.jobId }).pipe(Retry.none);
205 }
206 const finished = yield* getJob(job).pipe(
207 Effect.repeat({
208 schedule: Schedule.spaced("2 seconds"),
209 until: (value) => terminal(value.status),
210 times: 8,
211 }),
212 );
213 expect(terminal(finished.status)).toBe(true);
214
215 const terminalRedeploy = yield* stack.deploy(program({ job: true, paused: true }));
216 expect(terminalRedeploy.job!.jobId).toBe(job.jobId);
217 expect((yield* getJob(job)).status).toBe(finished.status);
218
219 const replaced = yield* stack.deploy(program({ job: true, paused: true, overwrite: true }));
220 const replacement = replaced.job!;
221 expect(replacement.jobId).not.toBe(job.jobId);
222 expect((yield* getJob(job)).status).toBe(finished.status);
223 const changedSource = yield* stack.deploy(
224 program({
225 job: true,
226 paused: true,
227 overwrite: true,
228 sourcePrefix: "migration",
229 otherJob: true,
230 }),
231 );
232 const filtered = changedSource.job!;
233 const otherJob = changedSource.otherJob!;
234 expect(filtered.jobId).not.toBe(replacement.jobId);
235 expect(otherJob.jobId).not.toBe(job.jobId);
236 expect(otherJob.jobId).not.toBe(filtered.jobId);
237 const firstPage = yield* r2.listSuperSlurperJobs({
238 accountId,
239 limit: 1,
240 offset: 0,
241 });
242 const secondPage = yield* r2.listSuperSlurperJobs({
243 accountId,
244 limit: 1,
245 offset: 1,
246 });
247 expect(firstPage.result).toHaveLength(1);
248 expect(secondPage.result).toHaveLength(1);
249 expect(firstPage.result[0]!.id).not.toBe(secondPage.result[0]!.id);
250 expect((yield* getJob(filtered)).source?.pathPrefix).toBe("migration");
251 const otherBeforeRemoval = yield* getJob(otherJob);
252 const removed = yield* stack.deploy(program({ job: false, otherJob: true }));
253 expect(removed.otherJob!.jobId).toBe(otherJob.jobId);
254 expect((yield* getJob(otherJob)).status).toBe(otherBeforeRemoval.status);
255 expect(terminal((yield* getJob(filtered)).status)).toBe(true);
256 expect(removed.source.bucketName).toBe(buckets.source.bucketName);
257 expect(removed.target.bucketName).toBe(buckets.target.bucketName);
258 const cancelled = yield* getJob(replacement).pipe(
259 Effect.repeat({
260 schedule: Schedule.spaced("2 seconds"),
261 until: (value) => terminal(value.status),
262 times: 8,
263 }),
264 );
265 expect(terminal(cancelled.status)).toBe(true);
266 const objects = yield* r2.listBucketObjects({
267 accountId,
268 bucketName: buckets.target.bucketName,
269 });
270 expect(objects.result.some((object) => object.key === "sentinel.txt")).toBe(true);
271 const sourceObjects = yield* r2.listBucketObjects({
272 accountId,
273 bucketName: buckets.source.bucketName,
274 });
275 expect(sourceObjects.result.some((object) => object.key === "migration.txt")).toBe(true);
276
277 yield* stack.destroy();
278 for (const bucketName of [buckets.source.bucketName, buckets.target.bucketName]) {
279 const error = yield* r2.getBucket({ accountId, bucketName }).pipe(Effect.flip);
280 expect(error._tag).toBe("NoSuchBucket");
281 }
282 expect((yield* getJob(job)).status).toBe(finished.status);
283 expect(terminal((yield* getJob(replacement)).status)).toBe(true);
284 expect(terminal((yield* getJob(otherJob)).status)).toBe(true);
285 const tokenGone = yield* accounts
286 .getToken({
287 accountId,
288 tokenId: buckets.token.tokenId,
289 })
290 .pipe(Effect.flip);
291 expect(["TokenNotFound", "InvalidRoute"]).toContain(tokenGone._tag);
292 yield* Effect.log({
293 jobs: [job.jobId, replacement.jobId, filtered.jobId, otherJob.jobId],
294 status: "all terminal; buckets and token destroyed",
295 });
296 yield* stack.destroy();
297 }).pipe(Effect.ensuring(stack.destroy().pipe(Effect.orDie))),
298 {
299 tags: [
300 "provider:cloudflare",
301 "provider:cloudflare:account",
302 "provider:cloudflare:apitoken",
303 "provider:cloudflare:queue",
304 "provider:cloudflare:r2",
305 "live",
306 ],
307 timeout: 120_000,
308 exclusive: "cloudflare:event-subscriptions",
309 },
310 );
311
312 test.provider(
313 "an unresolved target bucket identity replaces the migration",
314 (stack) =>
315 Effect.gen(function* () {
316 yield* stack.destroy();
317 const initial = yield* stack.deploy(
318 program({
319 job: true,
320 paused: true,
321 targetName: "alchemy-slurper-target-replacement-a",
322 }),
323 );
324 const replaced = yield* stack.deploy(
325 program({
326 job: true,
327 paused: true,
328 targetName: "alchemy-slurper-target-replacement-b",
329 }),
330 );
331 expect(replaced.job!.jobId).not.toBe(initial.job!.jobId);
332 expect((yield* getJob(replaced.job!)).target?.bucket).toBe(replaced.target.bucketName);
333 expect(terminal((yield* getJob(initial.job!)).status)).toBe(true);
334 yield* stack.destroy();
335 expect(terminal((yield* getJob(replaced.job!)).status)).toBe(true);
336 }).pipe(Effect.ensuring(stack.destroy().pipe(Effect.orDie))),
337 {
338 tags: [
339 "provider:cloudflare",
340 "provider:cloudflare:apitoken",
341 "provider:cloudflare:queue",
342 "provider:cloudflare:r2",
343 "live",
344 ],
345 timeout: 120_000,
346 exclusive: "cloudflare:event-subscriptions",
347 },
348 );
349
350 test.provider(
351 "job resources and refs deliver account-wide subscription events without taking ownership",
352 (stack) =>
353 Effect.gen(function* () {
354 yield* stack.destroy();
355 const { accountId } = yield* yield* CloudflareEnvironment;
356 const buckets = yield* stack.deploy(program({ job: false }));
357 yield* r2.uploadBucketObject({
358 accountId,
359 bucketName: buckets.source.bucketName,
360 objectName: "event.txt",
361 body: "migration event",
362 });
363 const created = yield* stack.deploy(
364 program({ job: true, paused: true, subscription: "direct" }),
365 );
366 const job = created.job!;
367 const subscription = created.subscription!;
368 expect(subscription.source).toEqual({ type: "superSlurper" });
369 const live = yield* queues.getSubscription({
370 accountId,
371 subscriptionId: subscription.subscriptionId,
372 });
373 expect(live.source).toEqual(expect.objectContaining({ type: "superSlurper" }));
374 yield* queues.createConsumer({
375 accountId,
376 queueId: created.queue!.queueId,
377 type: "http_pull",
378 });
379 const referenced = yield* stack.deploy(
380 program({ job: true, paused: true, subscription: "ref" }),
381 );
382 expect(referenced.subscription!.subscriptionId).toBe(subscription.subscriptionId);
383 const resumed = yield* stack.deploy(program({ job: true, subscription: "ref" }));
384 expect(resumed.job!.jobId).toBe(job.jobId);
385 const bodies: string[] = [];
386 const delivered = () =>
387 bodies.some(
388 (body) =>
389 body.includes("cf.superSlurper.job.") &&
390 body.includes(job.jobId) &&
391 body.includes(subscription.subscriptionId),
392 );
393 yield* Effect.gen(function* () {
394 const batch = yield* queues
395 .pullMessage({
396 accountId,
397 queueId: created.queue!.queueId,
398 batchSize: 100,
399 visibilityTimeoutMs: 1000,
400 })
401 .pipe(
402 Effect.retry({
403 while: (error) => error._tag === "QueueHttpPullNotEnabled",
404 schedule: Schedule.spaced("2 seconds"),
405 times: 8,
406 }),
407 );
408 for (const message of batch.messages ?? []) if (message.body) bodies.push(message.body);
409 const acks = (batch.messages ?? []).flatMap((message) =>
410 message.leaseId ? [{ leaseId: message.leaseId }] : [],
411 );
412 if (acks.length)
413 yield* queues.ackMessage({
414 accountId,
415 queueId: created.queue!.queueId,
416 acks,
417 });
418 // A `job.resumed` emitted before the new subscription propagates is
419 // lost, so emit a fresh one until an event is delivered.
420 if (!delivered() && (yield* getJob(job)).status === "running") {
421 yield* r2.pauseSuperSlurperJob(job).pipe(Retry.none, Effect.ignore);
422 yield* r2.resumeSuperSlurperJob(job).pipe(Retry.none, Effect.ignore);
423 }
424 }).pipe(
425 Effect.repeat({
426 schedule: Schedule.spaced("5 seconds"),
427 times: 10,
428 until: delivered,
429 }),
430 Effect.timeout("60 seconds"),
431 );
432 expect(delivered()).toBe(true);
433 yield* stack.deploy(program({ job: true, paused: true }));
434 expect((yield* getJob(job)).id).toBe(job.jobId);
435 const gone = yield* queues
436 .getSubscription({
437 accountId,
438 subscriptionId: subscription.subscriptionId,
439 })
440 .pipe(Effect.flip);
441 expect(gone._tag).toBe("SubscriptionNotFound");
442 yield* stack.destroy();
443 expect(terminal((yield* getJob(job)).status)).toBe(true);
444 }).pipe(Effect.ensuring(stack.destroy().pipe(Effect.orDie))),
445 {
446 tags: [
447 "provider:cloudflare",
448 "provider:cloudflare:apitoken",
449 "provider:cloudflare:queue",
450 "provider:cloudflare:r2",
451 "live",
452 ],
453 timeout: 120_000,
454 exclusive: "cloudflare:event-subscriptions",
455 },
456 );
457
458 // A one-object migration remained running after the bounded 24-second probe.
459 test.provider.skipIf(process.env.CLOUDFLARE_TEST_SUPER_SLURPER_COMPLETION !== "1")(
460 "completes a real object transfer and never restarts the completed job",
461 (stack) =>
462 Effect.gen(function* () {
463 yield* stack.destroy();
464 const { accountId } = yield* yield* CloudflareEnvironment;
465 const buckets = yield* stack.deploy(program({ job: false }));
466 yield* r2.uploadBucketObject({
467 accountId,
468 bucketName: buckets.source.bucketName,
469 objectName: "migration.txt",
470 body: "super-slurper-source",
471 });
472 const created = yield* stack.deploy(program({ job: true }));
473 const job = created.job!;
474 const completed = yield* getJob(job).pipe(
475 Effect.repeat({
476 schedule: Schedule.spaced("5 seconds"),
477 until: (value) => terminal(value.status),
478 times: 8,
479 }),
480 );
481 expect(completed.status).toBe("completed");
482 const migrated = yield* r2.listBucketObjects({
483 accountId,
484 bucketName: buckets.target.bucketName,
485 });
486 expect(migrated.result.some((object) => object.key === "migration.txt")).toBe(true);
487 const redeployed = yield* stack.deploy(program({ job: true, paused: true }));
488 expect(redeployed.job!.jobId).toBe(job.jobId);
489 expect((yield* getJob(job)).status).toBe("completed");
490 yield* stack.destroy();
491 expect((yield* getJob(job)).status).toBe("completed");
492 }),
493 {
494 tags: [
495 "provider:cloudflare",
496 "provider:cloudflare:apitoken",
497 "provider:cloudflare:queue",
498 "provider:cloudflare:r2",
499 "live",
500 ],
501 timeout: 120_000,
502 exclusive: "cloudflare:event-subscriptions",
503 },
504 );