The best way to control Claude Code, Codex and more from any computer or phone

main
ts 957 lines 34 KB
Raw
1 import * as Fly from "@alchemy.run/fly";
2 import { resolveOrgSlug } from "@alchemy.run/fly/Environment";
3 import { expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as machines from "@distilled.cloud/fly-io/machines";
6 import * as Provider from "alchemy/Provider";
7 import * as Effect from "effect/Effect";
8 import * as HttpClient from "effect/http/HttpClient";
9 import * as Layer from "effect/Layer";
10 import * as Redacted from "effect/Redacted";
11 import { MinimumLogLevel } from "effect/References";
12 import * as Result from "effect/Result";
13 import * as Schedule from "effect/Schedule";
14 import Api from "./fixtures/api.ts";
15 import ChecksApi, { ChecksSite } from "./fixtures/checks-api.ts";
16 import { ECHO_BODY, Echo } from "./fixtures/echo.ts";
17 import { fetchFrom, fetchOnce, nginx } from "./fixtures/flycast.ts";
18 import RegionalApi from "./fixtures/regional-api.ts";
19 import { Ping, Pong } from "./fixtures/rpc-cycle.ts";
20 import RpcGateway from "./fixtures/rpc-gateway.ts";
21 import RpcOrders, { ORDERS } from "./fixtures/rpc-orders.ts";
22 import PingLive from "./fixtures/rpc-ping.ts";
23 import PongLive from "./fixtures/rpc-pong.ts";
24 import RpcStranger from "./fixtures/rpc-stranger.ts";
25 import RpcUsers, { USERS } from "./fixtures/rpc-users.ts";
26 import SecureGateway from "./fixtures/secure-gateway.ts";
27 import SecureUsers, { SECURE_USERS_BODY } from "./fixtures/secure-users.ts";
28 import { API_PORT, MARKER, Site, VOLUME_PATH } from "./fixtures/shared.ts";
29 import UnhealthyApi, { UnhealthySite } from "./fixtures/unhealthy-api.ts";
30
31 const { test } = Test.make({ providers: Fly.providers() });
32
33 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
34
35 const waitUntilGone = (appName: string, machineId: string) =>
36 machines
37 .getMachine({
38 app_name: appName,
39 machine_id: machineId,
40 })
41 .pipe(
42 Effect.map((machine) =>
43 machine.state === "destroyed" ? ("gone" as const) : ("found" as const),
44 ),
45 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
46 Effect.repeat({
47 schedule: Schedule.spaced("2 seconds"),
48 until: (status) => status === "gone",
49 times: 10,
50 }),
51 );
52
53 test.provider(
54 "deploy token probe is typed",
55 (stack) =>
56 Effect.gen(function* () {
57 yield* stack.destroy();
58
59 const app = yield* stack.deploy(
60 Effect.gen(function* () {
61 return yield* Fly.App("TokenSite");
62 }),
63 );
64
65 const minted = yield* machines.createAppDeployToken({
66 app_name: app.appName,
67 });
68 // fly-io marks the deploy token sensitive, so it decodes as `Redacted`.
69 expect(Redacted.isRedacted(minted.token)).toBe(true);
70 expect(Redacted.value(minted.token as Redacted.Redacted<string>).length).toBeGreaterThan(0);
71
72 yield* stack.destroy();
73 }).pipe(logLevel),
74 {
75 tags: [
76 "provider:fly",
77 "provider:fly:app",
78 "provider:fly:machine",
79 "provider:fly:service",
80 "live",
81 ],
82 timeout: 90_000,
83 },
84 );
85
86 test.provider(
87 "create, serve, mount, and delete a service",
88 (stack) =>
89 Effect.gen(function* () {
90 yield* stack.destroy();
91
92 const deployed = yield* stack.deploy(
93 Effect.gen(function* () {
94 const app = yield* Site;
95 const ip = yield* Fly.IpAssignment("Shared", {
96 app,
97 type: "shared_v4",
98 });
99 const api = yield* Api;
100 return { app, ip, api };
101 }),
102 );
103
104 expect(deployed.api.machineId).toEqual(expect.any(String));
105 expect(deployed.api.machineId.length).toBeGreaterThan(0);
106 expect(deployed.api.machineIds).toEqual([deployed.api.machineId]);
107 expect(deployed.api.count).toEqual(1);
108 expect(deployed.api.appName).toEqual(deployed.app.appName);
109 expect(deployed.api.name).toEqual(expect.any(String));
110 expect(deployed.api.region).toEqual("iad");
111 expect(deployed.api.state).toEqual("started");
112 expect(deployed.api.url).toEqual(`https://${deployed.app.appName}.fly.dev`);
113 expect(deployed.api.code.hash).toEqual(expect.any(String));
114 expect(deployed.api.code.hash.length).toBeGreaterThan(0);
115 expect(deployed.api.mounts[0]?.path).toEqual(VOLUME_PATH);
116 expect(deployed.api.mounts[0]?.volumeId).toEqual(expect.any(String));
117
118 const fetched = yield* machines.getMachine({
119 app_name: deployed.api.appName,
120 machine_id: deployed.api.machineId,
121 });
122 expect(fetched.id).toEqual(deployed.api.machineId);
123 expect(fetched.name).toEqual(deployed.api.name);
124 expect(fetched.region).toEqual("iad");
125 expect(fetched.state).toEqual("started");
126 expect(fetched.config?.metadata?.["alchemy.type"]).toEqual("Fly.Service");
127 expect(fetched.config?.metadata?.["alchemy.stack"]).toEqual(expect.any(String));
128 expect(fetched.config?.image).toEqual(expect.stringContaining("registry.fly.io/"));
129 expect(fetched.config?.image).toEqual(expect.stringContaining(deployed.api.code.hash));
130 expect(fetched.config?.mounts?.[0]?.path).toEqual(VOLUME_PATH);
131 expect(fetched.config?.mounts?.[0]?.volume).toEqual(deployed.api.mounts[0]?.volumeId);
132 expect(fetched.config?.metadata?.["alchemy.replica"]).toEqual("0");
133 expect(fetched.config?.guest?.cpus).toEqual(1);
134 expect(fetched.config?.guest?.memory_mb).toEqual(256);
135 const defaultCheck = fetched.config?.services?.[0]?.checks?.[0];
136 expect(defaultCheck?.type).toEqual("tcp");
137 expect(defaultCheck?.port).toEqual(API_PORT);
138 expect(defaultCheck?.interval).toEqual("10s");
139 expect(defaultCheck?.timeout).toEqual("2s");
140 expect(defaultCheck?.grace_period).toEqual("30s");
141
142 const liveVolume = yield* machines.getVolumeById({
143 app_name: deployed.api.appName,
144 volume_id: deployed.api.mounts[0]!.volumeId,
145 });
146 expect(liveVolume.attached_machine_id).toEqual(deployed.api.machineId);
147
148 const provider = yield* Provider.findProvider(Fly.Service);
149 const all = yield* provider.list();
150 const found = all.find((service) => service.machineId === deployed.api.machineId);
151 expect(found).toBeDefined();
152 expect(found?.appName).toEqual(deployed.api.appName);
153 expect(found?.name).toEqual(deployed.api.name);
154 expect(found?.region).toEqual("iad");
155
156 const body = yield* HttpClient.get(deployed.api.url!).pipe(
157 Effect.flatMap((res) =>
158 res.status === 200 ? res.json : Effect.fail(new Error(`api returned ${res.status}`)),
159 ),
160 Effect.retry({
161 schedule: Schedule.spaced("4 seconds"),
162 times: 10,
163 }),
164 Effect.map((value) => value as { text: string; path: string }),
165 );
166 expect(body.path).toEqual(VOLUME_PATH);
167 expect(body.text).toEqual(MARKER);
168
169 yield* stack.destroy();
170
171 const gone = yield* waitUntilGone(deployed.api.appName, deployed.api.machineId);
172 expect(gone).toEqual("gone");
173 }).pipe(logLevel),
174 {
175 tags: [
176 "provider:fly",
177 "provider:fly:app",
178 "provider:fly:ipassignment",
179 "provider:fly:machine",
180 "provider:fly:service",
181 "provider:fly:volume",
182 "live",
183 ],
184 timeout: 180_000,
185 },
186 );
187
188 test.provider(
189 "destroy recovers a service and its volumes after initial checks fail",
190 (stack) =>
191 Effect.gen(function* () {
192 yield* stack.destroy();
193 const app = yield* stack.deploy(UnhealthySite);
194 const result = yield* stack.deploy(UnhealthyApi).pipe(Effect.result);
195 expect(Result.isFailure(result)).toBe(true);
196 if (Result.isFailure(result)) {
197 expect(result.failure).toMatchObject({
198 _tag: "Fly.ReplicaChecksNotPassing",
199 });
200 }
201 const live = yield* machines.listMachines({ app_name: app.appName });
202 expect(live.filter((machine) => machine.state !== "destroyed")).toHaveLength(1);
203 const volumes = yield* machines.listVolumes({ app_name: app.appName });
204 expect(volumes).toHaveLength(2);
205 const attached = volumes.find((volume) => volume.attached_machine_id === live[0]?.id);
206 expect(attached).toBeDefined();
207 const blocked = yield* machines
208 .deleteVolume({
209 app_name: app.appName,
210 volume_id: attached!.id!,
211 })
212 .pipe(Effect.flip);
213 expect(blocked._tag).toBe("VolumeAttached");
214 yield* stack.destroy();
215 for (const machine of live) {
216 expect(yield* waitUntilGone(app.appName, machine.id!)).toBe("gone");
217 }
218 for (const volume of volumes) {
219 const missing = yield* machines
220 .getVolumeById({
221 app_name: app.appName,
222 volume_id: volume.id!,
223 })
224 .pipe(Effect.flip);
225 expect(missing._tag).toBe("NotFound");
226 }
227 }).pipe(logLevel),
228 {
229 tags: [
230 "provider:fly",
231 "provider:fly:app",
232 "provider:fly:machine",
233 "provider:fly:service",
234 "provider:fly:volume",
235 "live",
236 ],
237 timeout: 180_000,
238 },
239 );
240
241 test.provider(
242 "creates a custom service check and reports it passing",
243 (stack) =>
244 Effect.gen(function* () {
245 yield* stack.destroy();
246
247 const deployed = yield* stack.deploy(
248 Effect.gen(function* () {
249 const app = yield* ChecksSite;
250 const service = yield* ChecksApi;
251 return { app, service };
252 }),
253 );
254
255 const live = yield* machines.getMachine({
256 app_name: deployed.app.appName,
257 machine_id: deployed.service.machineId,
258 });
259 const check = live.config?.services?.[0]?.checks?.[0];
260 expect(check?.type).toEqual("http");
261 expect(check?.port).toEqual(API_PORT);
262 expect(check?.method).toEqual("GET");
263 expect(check?.path).toEqual("/health");
264 expect(check?.protocol).toEqual("http");
265 expect(check?.interval).toEqual("15s");
266 expect(check?.timeout).toEqual("3s");
267 expect(check?.grace_period).toEqual("20s");
268
269 const serviceChecks =
270 live.checks?.filter((check) => check.name?.startsWith("servicecheck-")) ?? [];
271 expect(serviceChecks).toHaveLength(1);
272 expect(serviceChecks[0]?.name).toEqual(`servicecheck-00-http-${API_PORT}`);
273 expect(serviceChecks[0]?.status).toEqual("passing");
274
275 yield* stack.destroy();
276
277 const gone = yield* waitUntilGone(deployed.service.appName, deployed.service.machineId);
278 expect(gone).toEqual("gone");
279 }).pipe(logLevel),
280 {
281 tags: [
282 "provider:fly",
283 "provider:fly:app",
284 "provider:fly:machine",
285 "provider:fly:service",
286 "live",
287 ],
288 timeout: 180_000,
289 },
290 );
291
292 const ownedTags = [
293 "provider:fly",
294 "provider:fly:app",
295 "provider:fly:machine",
296 "provider:fly:service",
297 "live",
298 ];
299
300 /** Observed address kinds on an App. */
301 const addressKinds = (appName: string) =>
302 machines
303 .listAppIPAssignments({ app_name: appName })
304 .pipe(
305 Effect.map((res) =>
306 (res.ips ?? [])
307 .map((ip) =>
308 ip.network !== undefined && ip.network !== null
309 ? "flycast"
310 : ip.shared === true
311 ? "shared_v4"
312 : (ip.ip ?? "").includes(":")
313 ? "v6"
314 : "v4",
315 )
316 .sort(),
317 ),
318 );
319
320 const appGone = (appName: string) =>
321 machines.getApp({ app_name: appName }).pipe(
322 Effect.as(false),
323 Effect.catchTag("NotFound", () => Effect.succeed(true)),
324 Effect.repeat({
325 schedule: Schedule.spaced("2 seconds"),
326 until: (gone) => gone,
327 times: 10,
328 }),
329 );
330
331 const appNames = Effect.gen(function* () {
332 const org_slug = yield* resolveOrgSlug();
333 const { apps } = yield* machines.listApps({ org_slug });
334 return new Set((apps ?? []).flatMap((app) => (app.name ? [app.name] : [])));
335 });
336
337 // An image build sits between a Service creating its own App and starting
338 // its Machines. A deploy interrupted there leaves an App with nothing in it,
339 // which destroy must still find and delete.
340 test.provider(
341 "destroy deletes the App of a Service whose deploy was interrupted before its Machines started",
342 (stack) =>
343 Effect.gen(function* () {
344 yield* stack.destroy();
345 const before = yield* appNames;
346
347 // Interrupt the deploy as soon as the Service's App appears.
348 const created = yield* Effect.raceFirst(
349 stack.deploy(Echo()).pipe(Effect.as(undefined)),
350 appNames.pipe(
351 Effect.map((names) => [...names].find((name) => !before.has(name))),
352 Effect.repeat({
353 schedule: Schedule.spaced("1 second"),
354 until: (name) => name !== undefined,
355 times: 120,
356 }),
357 ),
358 );
359 expect(created).toEqual(expect.any(String));
360
361 yield* stack.destroy();
362 expect(yield* appGone(created!)).toBe(true);
363 }).pipe(logLevel),
364 {
365 tags: ["provider:fly", "provider:fly:app", "provider:fly:service", "live"],
366 timeout: 300_000,
367 },
368 );
369
370 const getText = (url: string) =>
371 HttpClient.get(url).pipe(
372 Effect.flatMap((res) =>
373 res.status === 200
374 ? res.text
375 : res.text.pipe(
376 Effect.flatMap((body) =>
377 Effect.fail(new Error(`${url} returned ${res.status}: ${body}`)),
378 ),
379 ),
380 ),
381 Effect.retry({ schedule: Schedule.spaced("4 seconds"), times: 15 }),
382 );
383
384 test.provider(
385 "a Service owns its App and serves its public url",
386 (stack) =>
387 Effect.gen(function* () {
388 yield* stack.destroy();
389
390 const owned = yield* stack.deploy(Echo());
391 expect(owned.ownsApp).toBe(true);
392 expect(owned.url).toEqual(`https://${owned.appName}.fly.dev`);
393 // Public Services get a plain-HTTP binding port for bound callers.
394 expect(owned.privateUrl).toEqual(`http://${owned.appName}.flycast:7780`);
395 // Port 80 only redirects to HTTPS, so it is not an endpoint.
396 expect(owned.endpoints).toEqual([
397 {
398 host: `${owned.appName}.fly.dev`,
399 port: 443,
400 internalPort: 3000,
401 protocol: "tcp",
402 handlers: ["tls", "http"],
403 url: `https://${owned.appName}.fly.dev`,
404 },
405 ]);
406 expect(yield* addressKinds(owned.appName)).toEqual(["flycast", "shared_v4", "v6"]);
407 expect(yield* getText(owned.url!)).toEqual(ECHO_BODY);
408
409 // Moving into a shared App replaces the Service and deletes its App.
410 const moved = yield* stack.deploy(
411 Effect.gen(function* () {
412 const site = yield* Fly.App("EchoSharedSite");
413 return yield* Echo({ app: site });
414 }),
415 );
416 expect(moved.ownsApp).toBe(false);
417 expect(moved.appName).not.toEqual(owned.appName);
418 expect(moved.url).toEqual(`https://${moved.appName}.fly.dev`);
419 expect(moved.privateUrl).toEqual(`http://${moved.appName}.flycast:7780`);
420 expect(yield* appGone(owned.appName)).toBe(true);
421
422 yield* stack.destroy();
423 expect(yield* appGone(moved.appName)).toBe(true);
424 }).pipe(logLevel),
425 { tags: ownedTags, timeout: 300_000 },
426 );
427
428 test.provider(
429 "a private Service answers over Flycast and turning it public keeps its App",
430 (stack) =>
431 Effect.gen(function* () {
432 yield* stack.destroy();
433 const deploy = (isPublic: boolean) =>
434 stack.deploy(
435 Effect.gen(function* () {
436 const client = yield* Fly.App("EchoCallerSite");
437 const caller = yield* Fly.Machine("Caller", {
438 app: client,
439 ...nginx,
440 });
441 const echo = yield* Echo({ public: isPublic });
442 return { caller, echo };
443 }),
444 );
445
446 const hidden = yield* deploy(false);
447 expect(hidden.echo.ownsApp).toBe(true);
448 expect(hidden.echo.url).toBeUndefined();
449 expect(hidden.echo.privateUrl).toEqual(`http://${hidden.echo.appName}.flycast`);
450 expect(hidden.echo.endpoints).toEqual([
451 {
452 host: `${hidden.echo.appName}.flycast`,
453 port: 80,
454 internalPort: 3000,
455 protocol: "tcp",
456 handlers: ["http"],
457 url: `http://${hidden.echo.appName}.flycast`,
458 },
459 ]);
460 expect(yield* addressKinds(hidden.echo.appName)).toEqual(["flycast"]);
461 expect(yield* fetchFrom(hidden.caller, hidden.echo.privateUrl!, ECHO_BODY)).toContain(
462 ECHO_BODY,
463 );
464
465 const shown = yield* deploy(true);
466 expect(shown.echo.appName).toEqual(hidden.echo.appName);
467 expect(shown.echo.url).toEqual(`https://${shown.echo.appName}.fly.dev`);
468 expect(yield* addressKinds(shown.echo.appName)).toEqual(["flycast", "shared_v4", "v6"]);
469 expect(yield* getText(shown.echo.url!)).toEqual(ECHO_BODY);
470
471 const hiddenAgain = yield* deploy(false);
472 expect(hiddenAgain.echo.appName).toEqual(hidden.echo.appName);
473 expect(hiddenAgain.echo.url).toBeUndefined();
474 expect(yield* addressKinds(hiddenAgain.echo.appName)).toEqual(["flycast"]);
475
476 yield* stack.destroy();
477 expect(yield* appGone(hidden.echo.appName)).toBe(true);
478 expect(yield* appGone(hidden.caller.appName)).toBe(true);
479 }).pipe(logLevel),
480 { tags: ownedTags, timeout: 400_000 },
481 );
482
483 test.provider(
484 "public or network on a Service in a shared App is rejected before anything is created",
485 (stack) =>
486 Effect.gen(function* () {
487 yield* stack.destroy();
488 const app = yield* stack.deploy(Fly.App("EchoInvalidSite"));
489 for (const invalid of [{ public: false }, { network: "any-network" }]) {
490 const failed = yield* stack
491 .deploy(
492 Effect.gen(function* () {
493 const site = yield* Fly.App("EchoInvalidSite");
494 return yield* Echo({ app: site, ...invalid });
495 }),
496 )
497 .pipe(Effect.flip);
498 expect(failed).toMatchObject({ _tag: "Fly.InvalidServiceProps" });
499 }
500 expect(yield* machines.listMachines({ app_name: app.appName })).toEqual([]);
501 yield* stack.destroy();
502 expect(yield* appGone(app.appName)).toBe(true);
503 }).pipe(logLevel),
504 { tags: ownedTags, timeout: 120_000 },
505 );
506
507 test.provider(
508 "Services on the stack network reach each other and nothing else reaches them",
509 (stack) =>
510 Effect.gen(function* () {
511 yield* stack.destroy();
512 const deployed = yield* stack.deploy(
513 Effect.gen(function* () {
514 const users = yield* SecureUsers;
515 const gateway = yield* SecureGateway;
516 const outside = yield* Fly.App("OutsideSite");
517 const outsider = yield* Fly.Machine("Outsider", {
518 app: outside,
519 ...nginx,
520 });
521 return {
522 users,
523 gateway,
524 outsider,
525 network: yield* Fly.stackNetwork,
526 };
527 }),
528 );
529 const { users, gateway, outsider, network } = deployed;
530 expect(users.network).toEqual(network);
531 expect(gateway.network).toEqual(network);
532 expect(users.url).toBeUndefined();
533 expect(users.privateUrl).toEqual(`http://${users.appName}.flycast`);
534 for (const appName of [users.appName, gateway.appName]) {
535 const app = yield* machines.getApp({ app_name: appName });
536 expect(app.network).toEqual(network);
537 }
538 const flycast = (yield* machines.listAppIPAssignments({
539 app_name: users.appName,
540 })).ips;
541 expect(flycast?.map((ip) => ip.network?.name)).toEqual([network]);
542
543 // The public gateway reaches the private Service over the stack network.
544 expect(yield* getText(gateway.url!)).toEqual(SECURE_USERS_BODY);
545
546 // An App on the default network cannot resolve either private name.
547 for (const url of [users.privateUrl!, `http://${users.appName}.internal:3000`]) {
548 const response = yield* fetchOnce(outsider, url);
549 expect(response).not.toContain(SECURE_USERS_BODY);
550 expect(response).toContain("bad address");
551 }
552
553 yield* stack.destroy();
554 for (const appName of [users.appName, gateway.appName, outsider.appName])
555 expect(yield* appGone(appName)).toBe(true);
556 }).pipe(logLevel),
557 { tags: ownedTags, timeout: 400_000 },
558 );
559
560 test.provider(
561 "Services in one App publish separate ports and each url reaches its own Service",
562 (stack) =>
563 Effect.gen(function* () {
564 yield* stack.destroy();
565 const deployed = yield* stack.deploy(
566 Effect.gen(function* () {
567 const site = yield* Fly.App("PortsSite");
568 yield* Fly.IpAssignment("PortsV4", { app: site, type: "shared_v4" });
569 const api = yield* Echo(
570 {
571 app: site,
572 env: { ECHO_BODY: "api" },
573 services: [
574 {
575 protocol: "tcp",
576 internalPort: 3000,
577 autostop: "off",
578 ports: [
579 { port: 80, handlers: ["http"], forceHttps: true },
580 { port: 443, handlers: ["tls", "http"] },
581 ],
582 },
583 ],
584 },
585 "PortsApi",
586 );
587 const admin = yield* Echo(
588 {
589 app: site,
590 env: { ECHO_BODY: "admin" },
591 // The API already uses the default binding port.
592 bindingPort: 7781,
593 services: [
594 {
595 protocol: "tcp",
596 internalPort: 3000,
597 autostop: "off",
598 ports: [{ port: 8443, handlers: ["tls", "http"] }],
599 },
600 {
601 protocol: "tcp",
602 internalPort: 3000,
603 autostop: "off",
604 ports: [{ port: 7000 }],
605 },
606 ],
607 },
608 "PortsAdmin",
609 );
610 return { api, admin };
611 }),
612 );
613 const { api, admin } = deployed;
614 const host = `${api.appName}.fly.dev`;
615 expect(admin.appName).toEqual(api.appName);
616 expect(api.url).toEqual(`https://${host}`);
617 expect(admin.url).toEqual(`https://${host}:8443`);
618 expect(api.privateUrl).toEqual(`http://${api.appName}.flycast:7780`);
619 expect(admin.privateUrl).toEqual(`http://${api.appName}.flycast:7781`);
620 expect(admin.endpoints).toEqual([
621 {
622 host,
623 port: 8443,
624 internalPort: 3000,
625 protocol: "tcp",
626 handlers: ["tls", "http"],
627 url: `https://${host}:8443`,
628 },
629 {
630 host,
631 port: 7000,
632 internalPort: 3000,
633 protocol: "tcp",
634 handlers: [],
635 url: undefined,
636 },
637 ]);
638 // TLS on a non-standard port is served over the shared IPv4.
639 expect(yield* getText(api.url!)).toEqual("api");
640 expect(yield* getText(admin.url!)).toEqual("admin");
641
642 yield* stack.destroy();
643 expect(yield* appGone(api.appName)).toBe(true);
644 }).pipe(logLevel),
645 { tags: ownedTags, timeout: 400_000 },
646 );
647
648 /** POST to an RPC path from inside a Machine, printing the status line. */
649 const postRpc = (caller: { appName: string; machineId: string }, url: string) =>
650 machines
651 .execMachine({
652 app_name: caller.appName,
653 machine_id: caller.machineId,
654 command: [
655 "sh",
656 "-c",
657 `wget -S -qO- -T 5 --post-data='[]' ${url}/__rpc__/listUsers 2>&1 || true`,
658 ],
659 timeout: 15,
660 })
661 .pipe(Effect.map((result) => result.stdout ?? ""));
662
663 test.provider(
664 "bound Services call methods, stream, fetch, and a published endpoint",
665 (stack) =>
666 Effect.gen(function* () {
667 yield* stack.destroy();
668 const deployed = yield* stack.deploy(
669 Effect.gen(function* () {
670 const users = yield* RpcUsers;
671 const orders = yield* RpcOrders;
672 const gateway = yield* RpcGateway;
673 const network = yield* Fly.stackNetwork;
674 // On the stack network, but binds nothing.
675 const snoopApp = yield* Fly.App("SnoopSite", { network });
676 const snoop = yield* Fly.Machine("Snoop", {
677 app: snoopApp,
678 ...nginx,
679 });
680 return { users, orders, gateway, snoop };
681 }),
682 );
683 const { users, orders, gateway, snoop } = deployed;
684 const base = gateway.url!;
685 expect(users.url).toBeUndefined();
686 expect(users.privateUrl).toEqual(`http://${users.appName}.flycast`);
687 expect(orders.privateUrl).toEqual(`http://${orders.appName}.flycast`);
688 expect(gateway.privateUrl).toEqual(`http://${gateway.appName}.flycast:7780`);
689
690 // A method call returns a typed result.
691 expect(JSON.parse(yield* getText(`${base}/users`))).toEqual(USERS);
692 // A private Service calls another through its own binding.
693 expect(JSON.parse(yield* getText(`${base}/orders`))).toEqual(
694 ORDERS.map((order) => ({
695 ...order,
696 user: USERS.find((user) => user.id === order.userId),
697 })),
698 );
699 // A streamed method delivers every element.
700 expect(JSON.parse(yield* getText(`${base}/stream`))).toEqual(USERS);
701 // `fetch` reaches the Service's HTTP routes on its private address.
702 expect(yield* getText(`${base}/http`)).toEqual("users-http 80 /hello");
703 // `bindEndpoint` reaches the admin port.
704 expect(yield* getText(`${base}/admin`)).toEqual(
705 `${users.appName}.flycast:9000 users-http 9000 /stats`,
706 );
707
708 // Same network, no binding, no token: the method is refused.
709 const snooped = yield* postRpc(snoop, users.privateUrl!);
710 expect(snooped).toContain("401");
711 expect(snooped).not.toContain("Ada");
712
713 // A public request never passes the Fly-Src check.
714 const publicCall = yield* HttpClient.post(`${base}/__rpc__/listUsers`).pipe(
715 Effect.map((response) => response.status),
716 );
717 expect(publicCall).toEqual(401);
718
719 yield* stack.destroy();
720 for (const appName of [users.appName, orders.appName, gateway.appName, snoop.appName])
721 expect(yield* appGone(appName)).toBe(true);
722 }).pipe(logLevel),
723 { tags: ownedTags, timeout: 600_000 },
724 );
725
726 test.provider(
727 "binding a Service on another network fails before the caller is created",
728 (stack) =>
729 Effect.gen(function* () {
730 yield* stack.destroy();
731 const users = yield* stack.deploy(RpcUsers);
732 const failed = yield* stack
733 .deploy(
734 Effect.gen(function* () {
735 yield* RpcUsers;
736 return yield* RpcStranger;
737 }),
738 )
739 .pipe(Effect.flip);
740 expect(failed).toMatchObject({
741 _tag: "Fly.ServiceUnreachable",
742 target: "RpcUsers",
743 network: undefined,
744 targetNetwork: users.network,
745 });
746 yield* stack.destroy();
747 expect(yield* appGone(users.appName)).toBe(true);
748 }).pipe(logLevel),
749 { tags: ownedTags, timeout: 400_000 },
750 );
751
752 test.provider(
753 "two Services that bind each other deploy and call each other",
754 (stack) =>
755 Effect.gen(function* () {
756 yield* stack.destroy();
757 const deployed = yield* stack.deploy(
758 Effect.gen(function* () {
759 const ping = yield* Ping;
760 const pong = yield* Pong;
761 return { ping, pong };
762 }).pipe(Effect.provide(Layer.mergeAll(PingLive, PongLive))),
763 );
764 expect(yield* getText(deployed.ping.url!)).toEqual("ping hears pong");
765 expect(yield* getText(deployed.pong.url!)).toEqual("pong hears ping");
766 yield* stack.destroy();
767 expect(yield* appGone(deployed.ping.appName)).toBe(true);
768 expect(yield* appGone(deployed.pong.appName)).toBe(true);
769 }).pipe(logLevel),
770 { tags: ownedTags, timeout: 600_000 },
771 );
772
773 test.provider(
774 "Services on the same port in one App are rejected before any Machine exists",
775 (stack) =>
776 Effect.gen(function* () {
777 yield* stack.destroy();
778 const program = (second: Partial<Parameters<typeof Echo>[0]>) =>
779 Effect.gen(function* () {
780 const site = yield* Fly.App("ConflictSite");
781 const first = yield* Echo({ app: site }, "ConflictFirst");
782 const other = yield* Echo({ app: site, ...second }, "ConflictSecond");
783 return { site, first, other };
784 });
785 const machinesIn = (appName: string) =>
786 machines
787 .listMachines({ app_name: appName })
788 .pipe(Effect.map((listed) => listed.filter((machine) => machine.state !== "destroyed")));
789
790 // Two new Services on the same default ports in one deploy.
791 const both = yield* stack.deploy(program({})).pipe(Effect.flip);
792 expect(both).toMatchObject({
793 _tag: "Fly.ServicePortConflict",
794 port: 80,
795 });
796
797 // One deployed Service, then a second one on its ports.
798 const first = yield* stack.deploy(
799 Effect.gen(function* () {
800 const site = yield* Fly.App("ConflictSite");
801 return yield* Echo({ app: site }, "ConflictFirst");
802 }),
803 );
804 expect(yield* machinesIn(first.appName)).toHaveLength(1);
805 const added = yield* stack.deploy(program({})).pipe(Effect.flip);
806 expect(added).toMatchObject({ _tag: "Fly.ServicePortConflict" });
807 expect(yield* machinesIn(first.appName)).toHaveLength(1);
808
809 // Distinct ports deploy.
810 const fixed = yield* stack.deploy(
811 program({
812 services: [
813 {
814 protocol: "tcp",
815 internalPort: 3000,
816 autostop: "off",
817 ports: [{ port: 8443, handlers: ["tls", "http"] }],
818 },
819 ],
820 bindingPort: 7781,
821 }),
822 );
823 expect(yield* machinesIn(fixed.site.appName)).toHaveLength(2);
824 yield* stack.destroy();
825 expect(yield* appGone(fixed.site.appName)).toBe(true);
826 }).pipe(logLevel),
827 { tags: ownedTags, timeout: 600_000 },
828 );
829
830 /** Region of every running Machine in an App, sorted. */
831 const machineRegions = (appName: string) =>
832 machines.listMachines({ app_name: appName }).pipe(
833 Effect.map((listed) =>
834 listed
835 .filter((machine) => machine.state !== "destroyed")
836 .map((machine) => machine.region ?? "")
837 .sort(),
838 ),
839 );
840
841 const machineIdsIn = (appName: string, region: string) =>
842 machines.listMachines({ app_name: appName }).pipe(
843 Effect.map((listed) =>
844 listed
845 .filter((machine) => machine.state !== "destroyed" && machine.region === region)
846 .map((machine) => machine.id ?? "")
847 .sort(),
848 ),
849 );
850
851 test.provider(
852 "a Service runs in several regions and adds, scales, and drops regions in place",
853 (stack) =>
854 Effect.gen(function* () {
855 yield* stack.destroy();
856 const deploy = (options: Parameters<typeof Echo>[0]) =>
857 stack.deploy(Echo(options, "Regional"));
858
859 const two = yield* deploy({ region: ["iad", "lhr"] });
860 expect(two.region).toEqual("iad");
861 expect(two.regions).toEqual(["iad", "lhr"]);
862 expect(two.count).toEqual(1);
863 expect(yield* machineRegions(two.appName)).toEqual(["iad", "lhr"]);
864 expect(yield* getText(two.url!)).toEqual(ECHO_BODY);
865 const iad = yield* machineIdsIn(two.appName, "iad");
866 const lhr = yield* machineIdsIn(two.appName, "lhr");
867
868 // Adding a region keeps the App and the existing Machines.
869 const three = yield* deploy({ region: ["iad", "lhr", "sin"] });
870 expect(three.appName).toEqual(two.appName);
871 expect(three.regions).toEqual(["iad", "lhr", "sin"]);
872 expect(yield* machineRegions(three.appName)).toEqual(["iad", "lhr", "sin"]);
873 expect(yield* machineIdsIn(three.appName, "iad")).toEqual(iad);
874 expect(yield* machineIdsIn(three.appName, "lhr")).toEqual(lhr);
875
876 // `count` is per region.
877 const scaled = yield* deploy({ region: ["iad", "lhr", "sin"], count: 2 });
878 expect(scaled.count).toEqual(2);
879 expect(scaled.machineIds).toHaveLength(6);
880 expect(yield* machineRegions(scaled.appName)).toEqual([
881 "iad",
882 "iad",
883 "lhr",
884 "lhr",
885 "sin",
886 "sin",
887 ]);
888
889 // Dropping a region deletes its Machines and keeps the hostname.
890 const dropped = yield* deploy({ region: ["iad", "sin"], count: 2 });
891 expect(dropped.appName).toEqual(two.appName);
892 expect(dropped.url).toEqual(two.url);
893 expect(yield* machineRegions(dropped.appName)).toEqual(["iad", "iad", "sin", "sin"]);
894 expect(yield* getText(dropped.url!)).toEqual(ECHO_BODY);
895
896 // A single region string still works.
897 const single = yield* deploy({ region: "sin" });
898 expect(single.regions).toEqual(["sin"]);
899 expect(yield* machineRegions(single.appName)).toEqual(["sin"]);
900
901 yield* stack.destroy();
902 expect(yield* appGone(two.appName)).toBe(true);
903 }).pipe(logLevel),
904 { tags: ownedTags, timeout: 600_000 },
905 );
906
907 test.provider(
908 "a multi-region Service mounts a Volume in each Machine's region",
909 (stack) =>
910 Effect.gen(function* () {
911 yield* stack.destroy();
912 const api = yield* stack.deploy(RegionalApi);
913 expect(api.regions).toEqual(["iad", "lhr"]);
914 expect(["iad", "lhr"]).toContain(yield* getText(api.url!));
915 const volumes = yield* machines.listVolumes({ app_name: api.appName });
916 const live = (yield* machines.listMachines({
917 app_name: api.appName,
918 })).filter((machine) => machine.state !== "destroyed");
919 expect(volumes.map((volume) => volume.region).sort()).toEqual(["iad", "lhr"]);
920 for (const volume of volumes) {
921 const attached = live.find((machine) => machine.id === volume.attached_machine_id);
922 expect(attached?.region).toEqual(volume.region);
923 }
924 yield* stack.destroy();
925 expect(yield* appGone(api.appName)).toBe(true);
926 }).pipe(logLevel),
927 { tags: ownedTags, timeout: 400_000 },
928 );
929
930 test.provider(
931 "a blue/green Service replaces its generation in every region",
932 (stack) =>
933 Effect.gen(function* () {
934 yield* stack.destroy();
935 const deploy = (body: string) =>
936 stack.deploy(
937 Echo(
938 {
939 region: ["iad", "lhr"],
940 env: { ECHO_BODY: body },
941 deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
942 },
943 "RegionalBlueGreen",
944 ),
945 );
946 const first = yield* deploy("one");
947 expect(yield* machineRegions(first.appName)).toEqual(["iad", "lhr"]);
948 const second = yield* deploy("two");
949 expect(second.appName).toEqual(first.appName);
950 expect(second.machineIds.some((id) => first.machineIds.includes(id))).toBe(false);
951 expect(yield* machineRegions(second.appName)).toEqual(["iad", "lhr"]);
952 expect(yield* getText(second.url!)).toEqual("two");
953 yield* stack.destroy();
954 expect(yield* appGone(first.appName)).toBe(true);
955 }).pipe(logLevel),
956 { tags: ownedTags, timeout: 500_000 },
957 );