@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 1,048 lines 37.2 KB
Raw
1 import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
2 import * as iam from "@distilled.cloud/gcp/unstable/iam_v1";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { tagRecord } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import { GcpEnvironment } from "../Environment.ts";
14 import {
15 deleteHostServiceAccount,
16 ensureHostServiceAccount,
17 hostServiceAccountEmail,
18 hostServiceAccountId,
19 projectNumber,
20 } from "../Host.ts";
21 import {
22 type GcpIamResourceKind,
23 hasIamMembership,
24 revokeIamMembership,
25 updateIamMembership,
26 } from "../IamPolicy.ts";
27 import {
28 createInternalLabels,
29 diffLabels,
30 hasAlchemyLabels,
31 stripInternalLabels,
32 toLabels,
33 } from "../Labels.ts";
34 import type { Providers } from "../Providers.ts";
35
36 export type SubscriptionRetryPolicy = {
37 /**
38 * Minimum delay between consecutive deliveries of a given message
39 * (e.g. `"10s"`). Must be between 0 and 600 seconds.
40 */
41 minimumBackoff?: string;
42 /**
43 * Maximum delay between consecutive deliveries of a given message
44 * (e.g. `"600s"`). Must be between 0 and 600 seconds.
45 */
46 maximumBackoff?: string;
47 };
48
49 export type SubscriptionDeadLetterPolicy = {
50 /**
51 * Topic that receives dead-lettered messages, as
52 * `projects/{project}/topics/{topic}`.
53 */
54 deadLetterTopic?: string;
55 /**
56 * Maximum delivery attempts before dead-lettering. Must be between 5 and
57 * 100. `0` is treated as `5`.
58 */
59 maxDeliveryAttempts?: number;
60 };
61
62 export type SubscriptionExpirationPolicy = {
63 /**
64 * Idle time-to-live (e.g. `"2678400s"` for 31 days). Minimum is 1 day.
65 * Omit `ttl` to never expire. If the whole policy is omitted, GCP
66 * applies a 31-day default.
67 */
68 ttl?: string;
69 };
70
71 export type SubscriptionOidcToken = {
72 /** Service account email Pub/Sub uses to sign the JWT. */
73 serviceAccountEmail?: string;
74 /** JWT audience. Defaults to the push endpoint. */
75 audience?: string;
76 };
77
78 export type SubscriptionPushConfig = {
79 /**
80 * HTTPS endpoint that receives pushed messages
81 * (e.g. `"https://example.com/push"`).
82 */
83 pushEndpoint?: string;
84 /**
85 * Endpoint attributes. The only currently supported key is
86 * `x-goog-version` (`v1` / `v1beta1` / `v1beta2`).
87 */
88 attributes?: Record<string, string>;
89 /**
90 * Attach an OIDC JWT as `Authorization` on every push.
91 */
92 oidcToken?: SubscriptionOidcToken;
93 };
94
95 export type SubscriptionBigQueryConfig = {
96 /**
97 * Table that receives the messages. Accepts a `GCP.BigQuery.Table`'s
98 * `name` (`projects/{project}/datasets/{dataset}/tables/{table}`),
99 * `{project}.{dataset}.{table}`, `{project}:{dataset}.{table}`, or
100 * `{dataset}.{table}` in the deploying project.
101 *
102 * Pub/Sub writes as a service account Alchemy creates for this
103 * subscription and grants `roles/bigquery.dataEditor` on the table.
104 * The account and its grant are removed with the subscription.
105 */
106 table: string;
107 /**
108 * Write message fields to the table's columns using the topic's schema.
109 * Cannot be combined with `useTableSchema`.
110 * @default false
111 */
112 useTopicSchema?: boolean;
113 /**
114 * Write JSON message fields to the table's columns using the table's
115 * schema. Cannot be combined with `useTopicSchema`.
116 * @default false
117 */
118 useTableSchema?: boolean;
119 /**
120 * Also write `subscription_name`, `message_id`, `publish_time`, and
121 * `attributes` to columns of the same names.
122 * @default false
123 */
124 writeMetadata?: boolean;
125 /**
126 * With `useTopicSchema` or `useTableSchema`, drop message fields that
127 * are not in the table schema instead of failing the write.
128 * @default false
129 */
130 dropUnknownFields?: boolean;
131 /**
132 * Service account Pub/Sub writes to BigQuery as: an `IAM.ServiceAccount`'s
133 * `email`, a referenced account's, or a literal email. Alchemy grants it
134 * `roles/bigquery.dataEditor` on the table and lets the Pub/Sub service
135 * agent act as it, then revokes only those grants when they are no
136 * longer needed; the account and any roles it already held are left
137 * alone. The deployer needs permission to edit the account's and the
138 * table's IAM policies.
139 *
140 * Omit it and Alchemy creates an account for this subscription, deleted
141 * with it.
142 */
143 serviceAccountEmail?: string;
144 };
145
146 /** One IAM grant a subscription made, revoked once it is no longer needed. */
147 export type SubscriptionIamGrant = {
148 /** Kind of resource whose policy holds the grant. */
149 kind: GcpIamResourceKind;
150 /** Resource name of that resource. */
151 name: string;
152 /** Granted role. */
153 role: string;
154 /** Principal the role was granted to. */
155 member: string;
156 };
157
158 export type SubscriptionBigQueryAttributes = {
159 /** Destination table, as `{project}.{dataset}.{table}`. */
160 table: string;
161 /** Whether the topic's schema maps message fields to columns. */
162 useTopicSchema: boolean;
163 /** Whether the table's schema maps message fields to columns. */
164 useTableSchema: boolean;
165 /** Whether message metadata columns are written. */
166 writeMetadata: boolean;
167 /** Whether fields missing from the table schema are dropped. */
168 dropUnknownFields: boolean;
169 /** Service account Pub/Sub writes to BigQuery as. */
170 serviceAccountEmail: string | undefined;
171 /** Server-reported delivery state (`ACTIVE`, `PERMISSION_DENIED`, …). */
172 state: string | undefined;
173 };
174
175 export type SubscriptionProps = {
176 /**
177 * Subscription id (the `{subscription}` segment of
178 * `projects/{project}/subscriptions/{subscription}`). If omitted, a
179 * unique name is generated from the stack, stage, and logical id.
180 * Changing it replaces the subscription.
181 */
182 subscriptionId?: string;
183 /**
184 * Topic this subscription receives from. Accepts a full resource name
185 * (`projects/{project}/topics/{topic}`) or a topic id. Immutable —
186 * changing it replaces the subscription.
187 */
188 topic: string;
189 /**
190 * User labels. Alchemy ownership labels are merged in automatically.
191 */
192 labels?: Record<string, string>;
193 /**
194 * Approximate time Pub/Sub waits for an ack before redelivering, in
195 * seconds. Minimum 10, maximum 600. `0` is treated as 10.
196 * @default 10
197 */
198 ackDeadlineSeconds?: number;
199 /**
200 * Retain acknowledged messages so they can be seeked until they fall
201 * out of `messageRetentionDuration`.
202 * @default false
203 */
204 retainAckedMessages?: boolean;
205 /**
206 * How long to retain unacknowledged messages (e.g. `"86400s"`). Cannot
207 * be more than 31 days or less than 10 minutes. Defaults to 7 days.
208 */
209 messageRetentionDuration?: string;
210 /**
211 * Deliver messages with the same ordering key in publish order.
212 * Immutable — changing it replaces the subscription.
213 * @default false
214 */
215 enableMessageOrdering?: boolean;
216 /**
217 * Guarantee that an acknowledged message is not redelivered (best
218 * effort; distinct publishes of the same payload still get distinct
219 * `message_id`s).
220 * @default false
221 */
222 enableExactlyOnceDelivery?: boolean;
223 /**
224 * Pub/Sub filter language expression. Only matching messages are
225 * delivered. Immutable — changing it replaces the subscription.
226 */
227 filter?: string;
228 /**
229 * When the subscription expires if idle. Omit for the 31-day default.
230 */
231 expirationPolicy?: SubscriptionExpirationPolicy;
232 /**
233 * Retry backoff after a NACK or ack-deadline exceed.
234 */
235 retryPolicy?: SubscriptionRetryPolicy;
236 /**
237 * Dead-letter topic and max delivery attempts. Omit to disable.
238 *
239 * Pub/Sub forwards dead letters as the project's Pub/Sub service agent
240 * (`service-{projectNumber}@gcp-sa-pubsub.iam.gserviceaccount.com`), so
241 * reconcile grants it `roles/pubsub.publisher` on the dead-letter topic
242 * and `roles/pubsub.subscriber` on this subscription. The topic grant is
243 * left in place on delete: the agent is shared by every subscription in
244 * the project that dead-letters to that topic.
245 */
246 deadLetterPolicy?: SubscriptionDeadLetterPolicy;
247 /**
248 * Push delivery endpoint. Omit (or leave empty) for a pull
249 * subscription.
250 */
251 pushConfig?: SubscriptionPushConfig;
252 /**
253 * Write messages straight to a BigQuery table. Cannot be combined with
254 * `pushConfig`.
255 */
256 bigqueryConfig?: SubscriptionBigQueryConfig;
257 };
258
259 export type Subscription = Resource<
260 "GCP.PubSub.Subscription",
261 SubscriptionProps,
262 {
263 /** Full resource name `projects/{project}/subscriptions/{subscription}`. */
264 name: string;
265 /** Subscription id (last path segment). */
266 subscriptionId: string;
267 /** Project id. */
268 project: string;
269 /** Topic resource name this subscription is attached to. */
270 topic: string;
271 /** User labels (Alchemy ownership labels stripped). */
272 labels: Record<string, string>;
273 /** Ack deadline in seconds. */
274 ackDeadlineSeconds: number;
275 /** Whether acknowledged messages are retained. */
276 retainAckedMessages: boolean;
277 /** Message retention duration, if set. */
278 messageRetentionDuration: string | undefined;
279 /** Whether message ordering is enabled. */
280 enableMessageOrdering: boolean;
281 /** Whether exactly-once delivery is enabled. */
282 enableExactlyOnceDelivery: boolean;
283 /** Filter expression, if set. */
284 filter: string | undefined;
285 /** Server-reported subscription state. */
286 state: string | undefined;
287 /** Idle expiration policy, if set. */
288 expirationPolicy: SubscriptionExpirationPolicy | undefined;
289 /** Retry policy, if set. */
290 retryPolicy: SubscriptionRetryPolicy | undefined;
291 /** Dead-letter policy, if set. */
292 deadLetterPolicy: SubscriptionDeadLetterPolicy | undefined;
293 /** Push config, if this is a push subscription. */
294 pushConfig: SubscriptionPushConfig | undefined;
295 /** BigQuery delivery, if this is a BigQuery subscription. */
296 bigqueryConfig: SubscriptionBigQueryAttributes | undefined;
297 /** IAM grants this subscription made for its BigQuery writer. */
298 iamGrants?: SubscriptionIamGrant[];
299 },
300 never,
301 Providers
302 >;
303
304 /**
305 * A Google Cloud Pub/Sub subscription.
306 *
307 * ### Creating a Subscription
308 * **Example:** Pull subscription on a topic
309 * ```typescript
310 * const topic = yield* GCP.PubSub.Topic("events", {});
311 * const subscription = yield* GCP.PubSub.Subscription("orders", {
312 * topic: topic.name,
313 * });
314 * ```
315 *
316 * **Example:** Explicit id, labels, and ack deadline
317 * ```typescript
318 * const topic = yield* GCP.PubSub.Topic("events", {});
319 * const subscription = yield* GCP.PubSub.Subscription("orders", {
320 * subscriptionId: "order-events",
321 * topic: topic.name,
322 * ackDeadlineSeconds: 30,
323 * labels: { env: "prod" },
324 * });
325 * ```
326 *
327 * ### Pulling Messages
328 * **Example:** Pull and acknowledge
329 * ```typescript
330 * const pull = yield* GCP.PubSub.Pull(subscription);
331 * const { receivedMessages } = yield* pull({
332 * body: { maxMessages: 1 },
333 * });
334 * const ackIds = (receivedMessages ?? [])
335 * .map((message) => message.ackId)
336 * .filter((ackId): ackId is string => ackId !== undefined);
337 * if (ackIds.length > 0) {
338 * const acknowledge = yield* GCP.PubSub.Acknowledge(subscription);
339 * yield* acknowledge({ body: { ackIds } });
340 * }
341 * ```
342 *
343 * ### Writing to BigQuery
344 * Pub/Sub writes each message as a row. Alchemy creates a service account
345 * for the subscription and grants it write access to the table.
346 *
347 * **Example:** Write messages and their metadata to a table
348 * ```typescript
349 * const topic = yield* GCP.PubSub.Topic("impressions", {});
350 * const dataset = yield* GCP.BigQuery.Dataset("raw", {});
351 * const table = yield* GCP.BigQuery.Table("impression_event", {
352 * datasetId: dataset.datasetId,
353 * schema: [
354 * { name: "subscription_name", type: "STRING" },
355 * { name: "message_id", type: "STRING" },
356 * { name: "publish_time", type: "TIMESTAMP" },
357 * { name: "data", type: "STRING" },
358 * { name: "attributes", type: "STRING" },
359 * ],
360 * });
361 * const subscription = yield* GCP.PubSub.Subscription("impressions-bigquery", {
362 * topic: topic.name,
363 * bigqueryConfig: {
364 * table: table.name,
365 * writeMetadata: true,
366 * },
367 * });
368 * ```
369 *
370 * **Example:** Write as a service account you provide
371 * ```typescript
372 * const writer = yield* GCP.IAM.ServiceAccount("impressions-writer", {});
373 * const subscription = yield* GCP.PubSub.Subscription("impressions-bigquery", {
374 * topic: topic.name,
375 * bigqueryConfig: {
376 * table: table.name,
377 * // also a referenced account's email, or a literal one
378 * serviceAccountEmail: writer.email,
379 * },
380 * });
381 * ```
382 *
383 * @resource
384 * @category PubSub
385 */
386 export const Subscription = Resource<Subscription>("GCP.PubSub.Subscription");
387
388 export class SubscriptionNotResolved extends Data.TaggedError(
389 "GCP.PubSub.SubscriptionNotResolved",
390 )<{
391 resourceName: string;
392 }> {}
393
394 const DEFAULT_ACK_DEADLINE_SECONDS = 10;
395
396 const subscriptionIdOf = (name: string) => name.split("/").pop() ?? name;
397
398 const resourceName = (project: string, subscriptionId: string) =>
399 `projects/${project}/subscriptions/${subscriptionId}`;
400
401 const topicNameOf = (project: string, topic: string) =>
402 topic.startsWith("projects/") ? topic : `projects/${project}/topics/${topic}`;
403
404 const topicKey = (topic: string | undefined) =>
405 topic === undefined ? undefined : subscriptionIdOf(topic);
406
407 const userLabels = (
408 labels: Record<string, string | undefined> | null | undefined,
409 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
410
411 const toId = (id: string, subscriptionId: string | undefined, existing?: string) =>
412 Effect.gen(function* () {
413 return (
414 subscriptionId ??
415 existing ??
416 (yield* createPhysicalName({
417 id,
418 maxLength: 255,
419 lowercase: true,
420 forbiddenPrefixes: ["goog"],
421 }))
422 );
423 });
424
425 const toPushConfig = (
426 config: pubsub.PushConfig | undefined,
427 ): SubscriptionPushConfig | undefined => {
428 if (config === undefined) return undefined;
429 const attributes = tagRecord(config.attributes);
430 const pushEndpoint = config.pushEndpoint;
431 if (pushEndpoint === undefined && Object.keys(attributes).length === 0) {
432 return undefined;
433 }
434 return {
435 pushEndpoint,
436 attributes: Object.keys(attributes).length > 0 ? attributes : undefined,
437 oidcToken: config.oidcToken,
438 };
439 };
440
441 const toBigQueryAttributes = (
442 config: pubsub.BigQueryConfig | undefined,
443 ): SubscriptionBigQueryAttributes | undefined => {
444 if (config?.table === undefined || config.table === "") return undefined;
445 return {
446 table: config.table,
447 useTopicSchema: config.useTopicSchema === true,
448 useTableSchema: config.useTableSchema === true,
449 writeMetadata: config.writeMetadata === true,
450 dropUnknownFields: config.dropUnknownFields === true,
451 serviceAccountEmail: config.serviceAccountEmail,
452 state: config.state,
453 };
454 };
455
456 type BigQueryTableRef = { project: string; datasetId: string; tableId: string };
457
458 /**
459 * Parse every table form `bigqueryConfig.table` accepts. The project
460 * group is greedy so domain-scoped ids (`example.com:proj`) keep their
461 * dots; dataset ids never contain one.
462 */
463 const parseBigQueryTable = (table: string, project: string): BigQueryTableRef | undefined => {
464 const name = table.match(/^projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/);
465 if (name) return { project: name[1]!, datasetId: name[2]!, tableId: name[3]! };
466 const qualified = table.match(/^(.+)[.:](\w+)\.([^.]+)$/);
467 if (qualified) {
468 return { project: qualified[1]!, datasetId: qualified[2]!, tableId: qualified[3]! };
469 }
470 const local = table.match(/^(\w+)\.([^.]+)$/);
471 if (local) return { project, datasetId: local[1]!, tableId: local[2]! };
472 return undefined;
473 };
474
475 const bigqueryTableId = (table: BigQueryTableRef) =>
476 `${table.project}.${table.datasetId}.${table.tableId}`;
477
478 const bigqueryTableName = (table: BigQueryTableRef) =>
479 `projects/${table.project}/datasets/${table.datasetId}/tables/${table.tableId}`;
480
481 const BIGQUERY_WRITER_ROLE = "roles/bigquery.dataEditor";
482
483 export class SubscriptionInvalidBigQueryTable extends Data.TaggedError(
484 "GCP.PubSub.SubscriptionInvalidBigQueryTable",
485 )<{ table: string }> {}
486
487 const toAttrs = (subscription: pubsub.Subscription, project: string) => {
488 const name = subscription.name ?? "";
489 return {
490 name,
491 subscriptionId: subscriptionIdOf(name),
492 project,
493 topic: subscription.topic ?? "",
494 labels: userLabels(subscription.labels),
495 ackDeadlineSeconds: subscription.ackDeadlineSeconds ?? DEFAULT_ACK_DEADLINE_SECONDS,
496 retainAckedMessages: subscription.retainAckedMessages === true,
497 messageRetentionDuration: subscription.messageRetentionDuration,
498 enableMessageOrdering: subscription.enableMessageOrdering === true,
499 enableExactlyOnceDelivery: subscription.enableExactlyOnceDelivery === true,
500 filter: subscription.filter,
501 state: subscription.state,
502 expirationPolicy: subscription.expirationPolicy,
503 retryPolicy: subscription.retryPolicy,
504 deadLetterPolicy: subscription.deadLetterPolicy,
505 pushConfig: toPushConfig(subscription.pushConfig),
506 bigqueryConfig: toBigQueryAttributes(subscription.bigqueryConfig),
507 };
508 };
509
510 const getByName = (name: string) =>
511 pubsub
512 .getProjectsSubscriptions({ subscription: name })
513 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
514
515 const waitUntilGone = (name: string) =>
516 getByName(name).pipe(
517 Effect.filterOrFail(
518 (existing) => existing === undefined,
519 () => new SubscriptionStillExists({ resourceName: name }),
520 ),
521 Effect.retry({
522 while: (error) => error._tag === "GCP.PubSub.SubscriptionStillExists",
523 schedule: Schedule.spaced("2 seconds"),
524 times: 30,
525 }),
526 Effect.asVoid,
527 );
528
529 export class SubscriptionStillExists extends Data.TaggedError(
530 "GCP.PubSub.SubscriptionStillExists",
531 )<{ resourceName: string }> {}
532
533 // Pub/Sub serves reads from replicas that lag writes: one successful GET
534 // does not mean the next one (or a publish routed to it) sees the
535 // resource. Require consecutive successful reads before reporting ready.
536 const CONSISTENT_READS = 3;
537
538 const waitUntilPresent = (name: string) =>
539 Effect.gen(function* () {
540 let found: pubsub.Subscription | undefined;
541 for (let read = 0; read < CONSISTENT_READS; read++) {
542 if (read > 0) yield* Effect.sleep("1 second");
543 found = yield* getByName(name);
544 if (found === undefined) {
545 return yield* new SubscriptionNotResolved({ resourceName: name });
546 }
547 }
548 return found!;
549 }).pipe(
550 Effect.retry({
551 while: (error) => error._tag === "GCP.PubSub.SubscriptionNotResolved",
552 schedule: Schedule.spaced("1 second"),
553 times: 60,
554 }),
555 );
556
557 /**
558 * Let the project's Pub/Sub service agent forward dead letters: publish to
559 * the dead-letter topic and ack the original on the subscription.
560 */
561 const pubsubServiceAgent = (project: string) =>
562 projectNumber(project).pipe(
563 Effect.map(
564 (number) => `serviceAccount:service-${number}@gcp-sa-pubsub.iam.gserviceaccount.com`,
565 ),
566 );
567
568 const grantDeadLetterAgent = (project: string, subscription: string, deadLetterTopic: string) =>
569 Effect.gen(function* () {
570 const member = yield* pubsubServiceAgent(project);
571 yield* updateIamMembership({
572 kind: "pubsub.topic",
573 name: topicNameOf(project, deadLetterTopic),
574 member,
575 add: ["roles/pubsub.publisher"],
576 });
577 yield* updateIamMembership({
578 kind: "pubsub.subscription",
579 name: subscription,
580 member,
581 add: ["roles/pubsub.subscriber"],
582 });
583 });
584
585 const sameOptionalString = (left: string | undefined, right: string | undefined) =>
586 (left ?? "") === (right ?? "");
587
588 const retryChanged = (
589 desired: SubscriptionRetryPolicy | undefined,
590 observed: pubsub.RetryPolicy | undefined,
591 ) =>
592 desired !== undefined &&
593 !(
594 sameOptionalString(desired.minimumBackoff, observed?.minimumBackoff) &&
595 sameOptionalString(desired.maximumBackoff, observed?.maximumBackoff)
596 );
597
598 const deadLetterChanged = (
599 desired: SubscriptionDeadLetterPolicy | undefined,
600 observed: pubsub.DeadLetterPolicy | undefined,
601 ) =>
602 desired !== undefined &&
603 !(
604 sameOptionalString(desired.deadLetterTopic, observed?.deadLetterTopic) &&
605 (desired.maxDeliveryAttempts ?? 0) === (observed?.maxDeliveryAttempts ?? 0)
606 );
607
608 const expirationChanged = (
609 desired: SubscriptionExpirationPolicy | undefined,
610 observed: pubsub.ExpirationPolicy | undefined,
611 ) => desired !== undefined && !sameOptionalString(desired.ttl, observed?.ttl);
612
613 const pushChanged = (
614 desired: SubscriptionPushConfig | undefined,
615 observed: pubsub.PushConfig | undefined,
616 ) => {
617 if (desired === undefined) return false;
618 const observedAttributes = tagRecord(observed?.attributes);
619 const desiredAttributes = tagRecord(desired.attributes);
620 return (
621 (desired.pushEndpoint ?? "") !== (observed?.pushEndpoint ?? "") ||
622 JSON.stringify(desiredAttributes) !== JSON.stringify(observedAttributes) ||
623 (desired.oidcToken?.serviceAccountEmail ?? "") !==
624 (observed?.oidcToken?.serviceAccountEmail ?? "") ||
625 (desired.oidcToken?.audience ?? "") !== (observed?.oidcToken?.audience ?? "")
626 );
627 };
628
629 const sameFlag = (left: boolean | undefined, right: boolean | undefined) =>
630 (left ?? false) === (right ?? false);
631
632 const bigqueryChanged = (
633 desired: pubsub.BigQueryConfig | undefined,
634 observed: pubsub.BigQueryConfig | undefined,
635 project: string,
636 ) => {
637 const observedTable =
638 observed?.table === undefined || observed.table === ""
639 ? undefined
640 : parseBigQueryTable(observed.table, project);
641 if (desired === undefined) return observedTable !== undefined;
642 return !(
643 observedTable !== undefined &&
644 bigqueryTableId(observedTable) === desired.table &&
645 sameFlag(desired.useTopicSchema, observed?.useTopicSchema) &&
646 sameFlag(desired.useTableSchema, observed?.useTableSchema) &&
647 sameFlag(desired.writeMetadata, observed?.writeMetadata) &&
648 sameFlag(desired.dropUnknownFields, observed?.dropUnknownFields) &&
649 sameOptionalString(desired.serviceAccountEmail, observed?.serviceAccountEmail)
650 );
651 };
652
653 const toCreateBody = (
654 topicName: string,
655 news: SubscriptionProps,
656 labels: Record<string, string>,
657 bigqueryConfig: pubsub.BigQueryConfig | undefined,
658 ): pubsub.Subscription => {
659 const body: pubsub.Subscription = {
660 topic: topicName,
661 labels,
662 };
663 if (news.ackDeadlineSeconds !== undefined) {
664 body.ackDeadlineSeconds = news.ackDeadlineSeconds;
665 }
666 if (news.retainAckedMessages !== undefined) {
667 body.retainAckedMessages = news.retainAckedMessages;
668 }
669 if (news.messageRetentionDuration !== undefined) {
670 body.messageRetentionDuration = news.messageRetentionDuration;
671 }
672 if (news.enableMessageOrdering !== undefined) {
673 body.enableMessageOrdering = news.enableMessageOrdering;
674 }
675 if (news.enableExactlyOnceDelivery !== undefined) {
676 body.enableExactlyOnceDelivery = news.enableExactlyOnceDelivery;
677 }
678 if (news.filter !== undefined) {
679 body.filter = news.filter;
680 }
681 if (news.expirationPolicy !== undefined) {
682 body.expirationPolicy = news.expirationPolicy;
683 }
684 if (news.retryPolicy !== undefined) {
685 body.retryPolicy = news.retryPolicy;
686 }
687 if (news.deadLetterPolicy !== undefined) {
688 body.deadLetterPolicy = news.deadLetterPolicy;
689 }
690 if (news.pushConfig !== undefined) {
691 body.pushConfig = news.pushConfig;
692 }
693 if (bigqueryConfig !== undefined) {
694 body.bigqueryConfig = bigqueryConfig;
695 }
696 return body;
697 };
698
699 const SUBSCRIPTION_TYPE = "GCP.PubSub.Subscription";
700
701 /**
702 * The service account Alchemy mints for one subscription's BigQuery
703 * writes. Keyed on the instance id, not just the name: IAM keeps serving a
704 * deleted account's identity for minutes, so a destroy → redeploy that
705 * re-created the same email would be rejected by Pub/Sub.
706 */
707 const bigqueryWriterKey = (subscription: string, instanceId: string) =>
708 `${subscription}#${instanceId}`;
709
710 const bigqueryWriterEmail = (project: string, subscription: string, instanceId: string) =>
711 hostServiceAccountEmail(
712 project,
713 hostServiceAccountId(SUBSCRIPTION_TYPE, bigqueryWriterKey(subscription, instanceId)),
714 );
715
716 const grantKey = (grant: SubscriptionIamGrant) =>
717 `${grant.kind}\u0000${grant.name}\u0000${grant.role}\u0000${grant.member}`;
718
719 /**
720 * Converge the BigQuery writer: mint the subscription's own account unless
721 * one is given, grant it write access to the table, and let Pub/Sub's
722 * service agent act as it. Returns the config Pub/Sub should run with and
723 * the grants this subscription owns: everything for a minted account, and
724 * for a given one only the grants it did not already hold.
725 */
726 const ensureBigQueryWriter = Effect.fn(function* (options: {
727 project: string;
728 subscription: string;
729 instanceId: string;
730 config: SubscriptionBigQueryConfig;
731 recorded: readonly SubscriptionIamGrant[];
732 }) {
733 const { project, config } = options;
734 const table = parseBigQueryTable(config.table, project);
735 if (table === undefined) {
736 return yield* new SubscriptionInvalidBigQueryTable({ table: config.table });
737 }
738 const given = config.serviceAccountEmail;
739 const email =
740 given ??
741 (yield* ensureHostServiceAccount({
742 project,
743 hostType: SUBSCRIPTION_TYPE,
744 resourceName: bigqueryWriterKey(options.subscription, options.instanceId),
745 actAs: false,
746 })).email;
747 const desired: SubscriptionIamGrant[] = [
748 {
749 kind: "bigquery.table",
750 name: bigqueryTableName(table),
751 role: BIGQUERY_WRITER_ROLE,
752 member: `serviceAccount:${email}`,
753 },
754 // Pub/Sub's service agent mints the account's tokens to write as it.
755 {
756 kind: "iam.serviceAccount",
757 name: `projects/${given === undefined ? project : "-"}/serviceAccounts/${email}`,
758 role: "roles/iam.serviceAccountTokenCreator",
759 member: yield* pubsubServiceAgent(project),
760 },
761 ];
762 const recorded = new Set(options.recorded.map(grantKey));
763 const owned: SubscriptionIamGrant[] = [];
764 for (const grant of desired) {
765 // A given account may already hold the role for other reasons: use
766 // it, but never record it, so it is never revoked.
767 if (given !== undefined && !recorded.has(grantKey(grant)) && (yield* hasIamMembership(grant))) {
768 continue;
769 }
770 yield* updateIamMembership({ ...grant, add: [grant.role] });
771 owned.push(grant);
772 }
773 return {
774 grants: owned,
775 config: {
776 table: bigqueryTableId(table),
777 useTopicSchema: config.useTopicSchema ?? false,
778 useTableSchema: config.useTableSchema ?? false,
779 writeMetadata: config.writeMetadata ?? false,
780 dropUnknownFields: config.dropUnknownFields ?? false,
781 serviceAccountEmail: email,
782 } satisfies pubsub.BigQueryConfig,
783 };
784 });
785
786 /** Revoke recorded grants that `keep` no longer contains. */
787 const revokeStaleGrants = Effect.fn(function* (
788 recorded: readonly SubscriptionIamGrant[],
789 keep: readonly SubscriptionIamGrant[],
790 ) {
791 const kept = new Set(keep.map(grantKey));
792 for (const grant of recorded) {
793 if (kept.has(grantKey(grant))) continue;
794 yield* revokeIamMembership({ ...grant, roles: [grant.role] });
795 }
796 });
797
798 /**
799 * Pub/Sub rejects a just-minted writer (`BigQueryServiceAccountNotReady`)
800 * and fresh grants on it (`Forbidden`) until IAM propagates them.
801 */
802 const retryWriterGrants =
803 (bigqueryConfig: pubsub.BigQueryConfig | undefined) =>
804 <A, E extends { _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
805 bigqueryConfig === undefined
806 ? effect
807 : effect.pipe(
808 Effect.retry({
809 while: (error) =>
810 error._tag === "BigQueryServiceAccountNotReady" || error._tag === "Forbidden",
811 times: 10,
812 schedule: Schedule.spaced("2 seconds"),
813 }),
814 );
815
816 export const SubscriptionProvider = () =>
817 Provider.succeed(Subscription, {
818 stables: ["name", "subscriptionId", "project", "topic"],
819
820 diff: Effect.fn(function* ({ news, olds, output }) {
821 if (!isResolved(news)) return undefined;
822 const previousId = olds?.subscriptionId ?? output?.subscriptionId;
823 const nextId = news.subscriptionId ?? previousId;
824 const nameChanged =
825 news.subscriptionId !== undefined &&
826 previousId !== undefined &&
827 news.subscriptionId !== previousId;
828 const previousTopic = olds?.topic ?? output?.topic;
829 const topicChanged =
830 previousTopic !== undefined && topicKey(news.topic) !== topicKey(previousTopic);
831 const previousFilter = olds?.filter ?? output?.filter ?? "";
832 const nextFilter = news.filter ?? "";
833 const previousOrdering =
834 olds?.enableMessageOrdering ?? output?.enableMessageOrdering ?? false;
835 const nextOrdering = news.enableMessageOrdering ?? false;
836 if (
837 !nameChanged &&
838 !topicChanged &&
839 previousFilter === nextFilter &&
840 previousOrdering === nextOrdering
841 ) {
842 return undefined;
843 }
844 return {
845 action: "replace" as const,
846 deleteFirst:
847 !nameChanged && nextId !== undefined && previousId !== undefined && nextId === previousId,
848 };
849 }),
850
851 read: Effect.fn(function* ({ id, olds, output }) {
852 const env = yield* GcpEnvironment.current;
853 const subscriptionId = yield* toId(id, olds?.subscriptionId, output?.subscriptionId);
854 const name = output?.name ?? resourceName(env.project, subscriptionId);
855 const existing = yield* getByName(name);
856 if (existing === undefined) return undefined;
857 const attrs = { ...toAttrs(existing, env.project), iamGrants: output?.iamGrants };
858 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
859 }),
860
861 list: () =>
862 Effect.gen(function* () {
863 const env = yield* GcpEnvironment.current;
864 return yield* pubsub.listProjectsSubscriptions
865 .pages({
866 project: `projects/${env.project}`,
867 pageSize: 1000,
868 })
869 .pipe(
870 Stream.flatMap((page) => Stream.fromIterable(page.subscriptions ?? [])),
871 Stream.filter((subscription) =>
872 Object.keys(subscription.labels ?? {}).some((key) => key.startsWith("alchemy-")),
873 ),
874 Stream.map((subscription) => toAttrs(subscription, env.project)),
875 Stream.runCollect,
876 Effect.map((chunk) => Array.from(chunk)),
877 );
878 }),
879
880 reconcile: Effect.fn(function* ({ id, instanceId, news, output }) {
881 const env = yield* GcpEnvironment.current;
882 const subscriptionId = yield* toId(id, news.subscriptionId, output?.subscriptionId);
883 const name = resourceName(env.project, subscriptionId);
884 const topicName = topicNameOf(env.project, news.topic);
885 const desiredLabels = {
886 ...toLabels(news.labels),
887 ...(yield* createInternalLabels(id)),
888 };
889 const previousGrants = output?.iamGrants ?? [];
890 const writer =
891 news.bigqueryConfig === undefined
892 ? undefined
893 : yield* ensureBigQueryWriter({
894 project: env.project,
895 subscription: name,
896 instanceId,
897 config: news.bigqueryConfig,
898 recorded: previousGrants,
899 });
900 const bigqueryConfig = writer?.config;
901
902 let current =
903 output !== undefined
904 ? yield* waitUntilPresent(name).pipe(
905 Effect.catchTag("GCP.PubSub.SubscriptionNotResolved", () => Effect.undefined),
906 )
907 : yield* getByName(name);
908
909 if (current === undefined) {
910 const created = yield* pubsub
911 .createProjectsSubscriptions({
912 name,
913 body: toCreateBody(topicName, news, desiredLabels, bigqueryConfig),
914 })
915 .pipe(
916 retryWriterGrants(bigqueryConfig),
917 Effect.catchTag("Conflict", () => waitUntilPresent(name)),
918 );
919 // Block until Pub/Sub serves the subscription: the create response
920 // alone does not mean a GET (or a publish routed to it) sees it yet.
921 current = created === undefined ? undefined : yield* waitUntilPresent(name);
922 }
923
924 if (current === undefined) {
925 return yield* new SubscriptionNotResolved({ resourceName: name });
926 }
927
928 const observedLabels = tagRecord(current.labels);
929 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
930 const labelsChanged = upsert.length > 0 || removed.length > 0;
931 const desiredAck =
932 news.ackDeadlineSeconds ?? current.ackDeadlineSeconds ?? DEFAULT_ACK_DEADLINE_SECONDS;
933 const ackChanged =
934 desiredAck !== (current.ackDeadlineSeconds ?? DEFAULT_ACK_DEADLINE_SECONDS);
935 const desiredRetain = news.retainAckedMessages ?? current.retainAckedMessages ?? false;
936 const retainChanged = desiredRetain !== (current.retainAckedMessages === true);
937 const desiredExactlyOnce =
938 news.enableExactlyOnceDelivery ?? current.enableExactlyOnceDelivery ?? false;
939 const exactlyOnceChanged =
940 desiredExactlyOnce !== (current.enableExactlyOnceDelivery === true);
941 const retentionChanged =
942 news.messageRetentionDuration !== undefined &&
943 !sameOptionalString(news.messageRetentionDuration, current.messageRetentionDuration);
944 const expChanged = expirationChanged(news.expirationPolicy, current.expirationPolicy);
945 const retriesChanged = retryChanged(news.retryPolicy, current.retryPolicy);
946 const deadLettersChanged = deadLetterChanged(news.deadLetterPolicy, current.deadLetterPolicy);
947 const pushConfigChanged = pushChanged(news.pushConfig, current.pushConfig);
948 const bigqueryConfigChanged = bigqueryChanged(
949 bigqueryConfig,
950 current.bigqueryConfig,
951 env.project,
952 );
953
954 if (
955 labelsChanged ||
956 ackChanged ||
957 retainChanged ||
958 exactlyOnceChanged ||
959 retentionChanged ||
960 expChanged ||
961 retriesChanged ||
962 deadLettersChanged ||
963 pushConfigChanged ||
964 bigqueryConfigChanged
965 ) {
966 current = yield* pubsub
967 .patchProjectsSubscriptions({
968 name,
969 body: {
970 subscription: {
971 name,
972 topic: topicName,
973 labels: desiredLabels,
974 ackDeadlineSeconds: desiredAck,
975 retainAckedMessages: desiredRetain,
976 enableExactlyOnceDelivery: desiredExactlyOnce,
977 messageRetentionDuration:
978 news.messageRetentionDuration ?? current.messageRetentionDuration,
979 expirationPolicy: news.expirationPolicy ?? current.expirationPolicy,
980 retryPolicy: news.retryPolicy ?? current.retryPolicy,
981 deadLetterPolicy: news.deadLetterPolicy ?? current.deadLetterPolicy,
982 pushConfig: news.pushConfig ?? current.pushConfig,
983 // Masked but omitted clears it: back to a pull subscription.
984 bigqueryConfig,
985 },
986 updateMask: [
987 labelsChanged ? "labels" : undefined,
988 ackChanged ? "ackDeadlineSeconds" : undefined,
989 retainChanged ? "retainAckedMessages" : undefined,
990 exactlyOnceChanged ? "enableExactlyOnceDelivery" : undefined,
991 retentionChanged ? "messageRetentionDuration" : undefined,
992 expChanged ? "expirationPolicy" : undefined,
993 retriesChanged ? "retryPolicy" : undefined,
994 deadLettersChanged ? "deadLetterPolicy" : undefined,
995 pushConfigChanged ? "pushConfig" : undefined,
996 bigqueryConfigChanged ? "bigqueryConfig" : undefined,
997 ]
998 .filter((field): field is string => field !== undefined)
999 .join(","),
1000 },
1001 })
1002 .pipe(
1003 Effect.retry({
1004 while: (error) => error._tag === "NotFound",
1005 times: 8,
1006 schedule: Schedule.spaced("1 second"),
1007 }),
1008 retryWriterGrants(bigqueryConfig),
1009 );
1010 }
1011
1012 // Only after Pub/Sub switched over: revoke what the previous writer
1013 // or table no longer needs. A minted writer itself lives until the
1014 // subscription is deleted, so turning BigQuery back on reuses it.
1015 yield* revokeStaleGrants(previousGrants, writer?.grants ?? []);
1016
1017 const deadLetterTopic = current.deadLetterPolicy?.deadLetterTopic;
1018 if (deadLetterTopic !== undefined && deadLetterTopic !== "") {
1019 yield* grantDeadLetterAgent(env.project, name, deadLetterTopic);
1020 }
1021
1022 return { ...toAttrs(current, env.project), iamGrants: writer?.grants ?? [] };
1023 }),
1024
1025 delete: Effect.fn(function* ({ instanceId, output }) {
1026 yield* pubsub
1027 .deleteProjectsSubscriptions({ subscription: output.name })
1028 .pipe(Effect.catchTag("NotFound", () => Effect.void));
1029 // Pub/Sub reads lag deletes; block until it stops serving it.
1030 yield* waitUntilGone(output.name);
1031 // Then revoke the grants it made, and drop the writer Alchemy minted
1032 // for it. Look the writer up even without recorded grants: a create
1033 // that failed after minting it never recorded any.
1034 yield* revokeStaleGrants(output.iamGrants ?? [], []);
1035 const writer = bigqueryWriterEmail(output.project, output.name, instanceId);
1036 const minted = yield* iam
1037 .getProjectsServiceAccounts({
1038 name: `projects/${output.project}/serviceAccounts/${writer}`,
1039 })
1040 .pipe(
1041 Effect.as(true),
1042 Effect.catchTag("NotFound", () => Effect.succeed(false)),
1043 );
1044 if (minted) {
1045 yield* deleteHostServiceAccount({ project: output.project, email: writer });
1046 }
1047 }),
1048 });