The best way to control Claude Code, Codex and more from any computer or phone

main
ts 394 lines 14.5 KB
Raw
1 import { describe, expect } from "bun:test";
2 import * as GCP from "@alchemy.run/gcp";
3 import * as Test from "@alchemy.run/test/Bun";
4 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
5 import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
6 import * as run from "@distilled.cloud/gcp/run_v2";
7 import * as storage from "@distilled.cloud/gcp/storage_v1";
8 import * as Alchemy from "alchemy";
9 import * as Effect from "effect/Effect";
10 import * as FetchHttpClient from "effect/http/FetchHttpClient";
11 import * as HttpClient from "effect/http/HttpClient";
12 import * as HttpClientRequest from "effect/http/HttpClientRequest";
13 import * as Layer from "effect/Layer";
14 import * as Schedule from "effect/Schedule";
15 import Stack from "../alchemy.run.ts";
16 import { MAX_DELIVERY_ATTEMPTS } from "../src/Email.ts";
17 import { emailObjectFor, type OrderEvent } from "../src/resources.ts";
18
19 const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
20 providers: GCP.providers(),
21 state: Alchemy.localState(),
22 });
23
24 const { getWhenReady } = Test;
25
26 // Out-of-band calls to the Google APIs resolve the same stored credentials
27 // the deploy uses, so the test runs against the configured profile.
28 const GcpHttp = Layer.mergeAll(GCP.GcpAuth, GCP.fromAuthProvider(), FetchHttpClient.layer);
29
30 // Deploy, tests and destroy all sit behind the same Docker guard.
31 describe("gcp-pubsub-fanout", () => {
32 const stack = beforeAll(deploy(Stack), { timeout: 900_000 });
33
34 const baseUrlOf = (url: string | undefined) => {
35 if (url === undefined) throw new Error("the service has no URL");
36 return url.replace(/\/+$/, "");
37 };
38
39 /** `projects/{p}/datasets/{d}/tables/{t}` → its parts. */
40 const tableRefOf = (tableName: string) => {
41 const [, projectId, , datasetId, , tableId] = tableName.split("/");
42 return { projectId: projectId!, datasetId: datasetId!, tableId: tableId! };
43 };
44
45 /** The push subscriptions the two consumers' event sources created. */
46 // Pub/Sub reads are eventually consistent: a just-created subscription can
47 // briefly read as missing, so re-list until every listed one resolves.
48 const subscriptionsOf = (topic: string) =>
49 pubsub.listProjectsTopicsSubscriptions({ topic }).pipe(
50 Effect.map((page) => page.subscriptions ?? []),
51 Effect.flatMap((names) =>
52 Effect.forEach(names, (subscription) => pubsub.getProjectsSubscriptions({ subscription })),
53 ),
54 Effect.retry({
55 while: (error) => error._tag === "NotFound",
56 schedule: Schedule.spaced("2 seconds"),
57 times: 15,
58 }),
59 Effect.provide(GcpHttp),
60 );
61
62 const placeOrder = (baseUrl: string, email: string, total: number) =>
63 Effect.gen(function* () {
64 const res = yield* HttpClient.execute(
65 HttpClientRequest.post(`${baseUrl}/orders`).pipe(
66 HttpClientRequest.bodyJsonUnsafe({ email, total }),
67 ),
68 );
69 expect(res.status).toBe(202);
70 return (yield* res.json) as { orderId: string; eventId: string };
71 });
72
73 const cancelOrder = (baseUrl: string, orderId: string, email: string) =>
74 Effect.gen(function* () {
75 const res = yield* HttpClient.execute(
76 HttpClientRequest.post(`${baseUrl}/orders/${orderId}/cancel`).pipe(
77 HttpClientRequest.bodyJsonUnsafe({ email }),
78 ),
79 );
80 expect(res.status).toBe(202);
81 return (yield* res.json) as { orderId: string; eventId: string };
82 });
83
84 /** The outbox object for an event, or `undefined` while it is missing. */
85 const emailFor = (bucket: string, eventId: string) =>
86 storage.getObjects({ bucket, object: emailObjectFor(eventId) }).pipe(
87 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
88 Effect.provide(GcpHttp),
89 );
90
91 /** Poll until the email consumer has written the event's confirmation. */
92 const awaitEmail = (bucket: string, eventId: string) =>
93 emailFor(bucket, eventId).pipe(
94 Effect.repeat({
95 schedule: Schedule.spaced("5 seconds"),
96 until: (object) => object !== undefined,
97 times: 60,
98 }),
99 );
100
101 interface AnalyticsRow {
102 eventId: string;
103 type: string;
104 orderId: string;
105 }
106
107 /** Rows the analytics consumer recorded for these events. */
108 const analyticsRowsFor = (tableName: string, eventIds: string[]) =>
109 Effect.gen(function* () {
110 const { projectId, datasetId, tableId } = tableRefOf(tableName);
111 const result = yield* bigquery.queryJobs({
112 projectId,
113 body: {
114 query: `SELECT eventId, type, orderId FROM \`${datasetId}.${tableId}\` WHERE eventId IN UNNEST(@ids)`,
115 useLegacySql: false,
116 parameterMode: "NAMED",
117 queryParameters: [
118 {
119 name: "ids",
120 parameterType: { type: "ARRAY", arrayType: { type: "STRING" } },
121 parameterValue: {
122 arrayValues: eventIds.map((value) => ({ value })),
123 },
124 },
125 ],
126 timeoutMs: 20_000,
127 },
128 });
129 return (result.rows ?? []).map((row): AnalyticsRow => ({
130 eventId: String(row.f?.[0]?.v),
131 type: String(row.f?.[1]?.v),
132 orderId: String(row.f?.[2]?.v),
133 }));
134 }).pipe(Effect.provide(GcpHttp));
135
136 /** Poll until every event has a row (streaming inserts are queryable at once). */
137 const awaitAnalytics = (tableName: string, eventIds: string[]) =>
138 analyticsRowsFor(tableName, eventIds).pipe(
139 Effect.repeat({
140 schedule: Schedule.spaced("5 seconds"),
141 until: (rows) => eventIds.every((id) => rows.some((row) => row.eventId === id)),
142 times: 60,
143 }),
144 );
145
146 test(
147 "each consumer gets its own push subscription on the one topic",
148 Effect.gen(function* () {
149 const { topicName, deadLetterTopicName, deadLetterSubscription, project } = yield* stack;
150
151 const subscriptions = yield* subscriptionsOf(topicName);
152 expect(subscriptions.length).toBe(2);
153 for (const subscription of subscriptions) {
154 expect(subscription.pushConfig?.pushEndpoint).toMatch(
155 /^https:\/\/.+\/__alchemy\/pubsub\/orderevents$/,
156 );
157 expect(subscription.pushConfig?.oidcToken?.serviceAccountEmail).toMatch(
158 /@.+\.iam\.gserviceaccount\.com$/,
159 );
160 }
161
162 const filtered = subscriptions.filter((s) => s.filter);
163 expect(filtered.length).toBe(1);
164 const email = filtered[0]!;
165 expect(email.filter).toBe('attributes.type = "order.created"');
166 expect(email.deadLetterPolicy).toEqual({
167 deadLetterTopic: deadLetterTopicName,
168 maxDeliveryAttempts: MAX_DELIVERY_ATTEMPTS,
169 });
170 // Backoff keeps the attempts from being spent on a transient failure.
171 expect(email.retryPolicy).toEqual({
172 minimumBackoff: "10s",
173 maximumBackoff: "600s",
174 });
175
176 const analytics = subscriptions.find((s) => !s.filter)!;
177 expect(analytics.deadLetterPolicy).toBeUndefined();
178
179 // Dead letters are forwarded by the project's Pub/Sub service agent,
180 // which needs to publish to the dead-letter topic and ack the source.
181 const topicPolicy = yield* pubsub
182 .getIamPolicyProjectsTopics({ resource: deadLetterTopicName })
183 .pipe(Effect.provide(GcpHttp));
184 const agent = (topicPolicy.bindings ?? [])
185 .find((binding) => binding.role === "roles/pubsub.publisher")
186 ?.members?.find((member) => member.includes("@gcp-sa-pubsub."));
187 expect(agent).toBeDefined();
188 const subscriptionPolicy = yield* pubsub
189 .getIamPolicyProjectsSubscriptions({ resource: email.name! })
190 .pipe(Effect.provide(GcpHttp));
191 expect(
192 (subscriptionPolicy.bindings ?? []).some(
193 (binding) =>
194 binding.role === "roles/pubsub.subscriber" && binding.members?.includes(agent!),
195 ),
196 ).toBe(true);
197
198 expect(deadLetterSubscription).toStartWith(`projects/${project}/`);
199 }),
200 { timeout: 120_000 },
201 );
202
203 test(
204 "rejects an order without an email or total",
205 Effect.gen(function* () {
206 const { url } = yield* stack;
207 const baseUrl = baseUrlOf(url);
208 yield* getWhenReady(`${baseUrl}/`);
209
210 for (const body of [{ total: 10 }, { email: "a@example.com" }]) {
211 const res = yield* HttpClient.execute(
212 HttpClientRequest.post(`${baseUrl}/orders`).pipe(HttpClientRequest.bodyJsonUnsafe(body)),
213 );
214 expect(res.status).toBe(400);
215 }
216 }),
217 { timeout: 120_000 },
218 );
219
220 test(
221 "every order.created reaches both the email and the analytics consumer",
222 Effect.gen(function* () {
223 const { url, bucketName, tableName } = yield* stack;
224 const baseUrl = baseUrlOf(url);
225 yield* getWhenReady(`${baseUrl}/`);
226
227 const orders = [];
228 for (const [i, total] of [12.5, 99, 7.25].entries()) {
229 orders.push(yield* placeOrder(baseUrl, `buyer${i}@example.com`, total));
230 }
231
232 // A fresh deploy's first pushes can wait on IAM propagation for a few
233 // minutes; Pub/Sub retries them until they land.
234 for (const [i, order] of orders.entries()) {
235 const email = yield* awaitEmail(bucketName, order.eventId);
236 expect(email).toBeDefined();
237 expect(email!.metadata).toEqual({
238 orderId: order.orderId,
239 to: `buyer${i}@example.com`,
240 });
241 }
242
243 const rows = yield* awaitAnalytics(
244 tableName,
245 orders.map((order) => order.eventId),
246 );
247 for (const order of orders) {
248 expect(rows).toContainEqual({
249 eventId: order.eventId,
250 type: "order.created" satisfies OrderEvent["type"],
251 orderId: order.orderId,
252 });
253 }
254 }),
255 { timeout: 900_000 },
256 );
257
258 test(
259 "order.cancelled only reaches analytics",
260 Effect.gen(function* () {
261 const { url, bucketName, tableName } = yield* stack;
262 const baseUrl = baseUrlOf(url);
263 yield* getWhenReady(`${baseUrl}/`);
264
265 const created = yield* placeOrder(baseUrl, "changed@example.com", 40);
266 const cancelled = yield* cancelOrder(baseUrl, created.orderId, "changed@example.com");
267
268 const rows = yield* awaitAnalytics(tableName, [created.eventId, cancelled.eventId]);
269 expect(rows).toContainEqual({
270 eventId: cancelled.eventId,
271 type: "order.cancelled",
272 orderId: created.orderId,
273 });
274
275 // The confirmation for the created event lands; the cancellation is
276 // filtered out by Pub/Sub before it is ever pushed to the email service.
277 expect(yield* awaitEmail(bucketName, created.eventId)).toBeDefined();
278 expect(yield* emailFor(bucketName, cancelled.eventId)).toBeUndefined();
279 }),
280 { timeout: 900_000 },
281 );
282
283 test(
284 "an email that can never be sent is dead-lettered after max attempts",
285 Effect.gen(function* () {
286 const { url, bucketName, tableName, deadLetterSubscription } = yield* stack;
287 const baseUrl = baseUrlOf(url);
288 yield* getWhenReady(`${baseUrl}/`);
289
290 const order = yield* placeOrder(baseUrl, "nobody@example.invalid", 5);
291
292 // Analytics is a separate subscription, so the email failures don't
293 // hold it back.
294 yield* awaitAnalytics(tableName, [order.eventId]);
295
296 const pullDeadLetters = pubsub
297 .pullProjectsSubscriptions({
298 subscription: deadLetterSubscription,
299 body: { maxMessages: 10, returnImmediately: true },
300 })
301 .pipe(Effect.provide(GcpHttp));
302
303 const dead: pubsub.ReceivedMessage[] = [];
304 yield* pullDeadLetters.pipe(
305 Effect.tap(({ receivedMessages = [] }) =>
306 Effect.gen(function* () {
307 dead.push(...receivedMessages);
308 const ackIds = receivedMessages.flatMap((m) => (m.ackId ? [m.ackId] : []));
309 if (ackIds.length > 0) {
310 yield* pubsub
311 .acknowledgeProjectsSubscriptions({
312 subscription: deadLetterSubscription,
313 body: { ackIds },
314 })
315 .pipe(Effect.provide(GcpHttp));
316 }
317 }),
318 ),
319 Effect.repeat({
320 schedule: Schedule.spaced("10 seconds"),
321 until: () => dead.some((m) => decode(m).eventId === order.eventId),
322 times: 48,
323 }),
324 );
325
326 const letter = dead.find((m) => decode(m).eventId === order.eventId)!;
327 expect(letter.message?.attributes?.type).toBe("order.created");
328 // Pub/Sub stamps where the dead letter came from.
329 expect(letter.message?.attributes?.CloudPubSubDeadLetterSourceSubscription).toBeDefined();
330
331 expect(yield* emailFor(bucketName, order.eventId)).toBeUndefined();
332 }),
333 { timeout: 900_000 },
334 );
335
336 const decode = (received: pubsub.ReceivedMessage) =>
337 JSON.parse(
338 Buffer.from(received.message?.data ?? "e30=", "base64").toString("utf8"),
339 ) as OrderEvent;
340
341 /** Poll until a deleted resource reads as `NotFound`. */
342 const expectGone = <E, R>(what: string, status: Effect.Effect<"found" | "gone", E, R>) =>
343 status.pipe(
344 Effect.repeat({
345 schedule: Schedule.spaced("5 seconds"),
346 until: (status) => status === "gone",
347 times: 12,
348 }),
349 Effect.tap((status) => Effect.sync(() => expect(`${what}: ${status}`).toBe(`${what}: gone`))),
350 );
351
352 const statusOf = <A, E extends { _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
353 effect.pipe(
354 Effect.as("found" as const),
355 Effect.catchIf(
356 (error) => error._tag === "NotFound",
357 () => Effect.succeed("gone" as const),
358 ),
359 Effect.provide(GcpHttp),
360 );
361
362 afterAll.skipIf(!!process.env.NO_DESTROY)(
363 Effect.gen(function* () {
364 const outputs = yield* stack;
365 const subscriptions =
366 outputs === undefined
367 ? []
368 : (yield* subscriptionsOf(outputs.topicName)).map((s) => s.name!);
369
370 yield* destroy(Stack);
371 if (outputs === undefined) return;
372
373 for (const [what, name] of [
374 ["email service", outputs.emailService],
375 ["analytics service", outputs.analyticsService],
376 ] as const) {
377 yield* expectGone(what, statusOf(run.getProjectsLocationsServices({ name })));
378 }
379 for (const subscription of [...subscriptions, outputs.deadLetterSubscription]) {
380 yield* expectGone(
381 subscription,
382 statusOf(pubsub.getProjectsSubscriptions({ subscription })),
383 );
384 }
385 for (const topic of [outputs.topicName, outputs.deadLetterTopicName]) {
386 yield* expectGone(topic, statusOf(pubsub.getProjectsTopics({ topic })));
387 }
388 yield* expectGone("bucket", statusOf(storage.getBuckets({ bucket: outputs.bucketName })));
389 const { projectId, datasetId } = tableRefOf(outputs.tableName);
390 yield* expectGone("dataset", statusOf(bigquery.getDatasets({ projectId, datasetId })));
391 }),
392 { timeout: 600_000 },
393 );
394 });