@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 501 lines 17.5 KB
Raw
1 import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
2 import * as storage from "@distilled.cloud/gcp/storage_v1";
3 import * as serviceusage from "@distilled.cloud/gcp/unstable/serviceusage_v1beta1";
4 import { unownedUnlessPrevious } from "alchemy/AdoptPolicy";
5 import { isResolved } from "alchemy/Diff";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { tagRecord } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import { GcpEnvironment } from "../Environment.ts";
14 import { DeleteNotConfirmed } from "../Errors.ts";
15 import {
16 ALCHEMY_LABEL_PREFIX,
17 hasAlchemyLabels,
18 stripInternalLabels,
19 toLabels,
20 } from "../Labels.ts";
21 import { waitForOperation } from "../Operation.ts";
22 import type { Providers } from "../Providers.ts";
23
24 const DEFAULT_PAYLOAD_FORMAT = "JSON_API_V1";
25 const CANONICAL_TOPIC_PREFIX = "//pubsub.googleapis.com/";
26 const PUBLISHER_ROLE = "roles/pubsub.publisher";
27
28 export type NotificationPayloadFormat = "JSON_API_V1" | "NONE";
29
30 export type NotificationProps = {
31 /**
32 * Name of the Cloud Storage bucket this notification is attached to.
33 * Immutable — changing it replaces the notification.
34 */
35 bucketName: string;
36 /**
37 * Cloud Pub/Sub topic to publish events to. Accepts a topic id, a
38 * resource name (`projects/{project}/topics/{topic}`), or the canonical
39 * URI (`//pubsub.googleapis.com/projects/{project}/topics/{topic}`).
40 * Immutable — changing it replaces the notification.
41 */
42 topic: string;
43 /**
44 * Payload included in each Pub/Sub message.
45 * @default "JSON_API_V1"
46 */
47 payloadFormat?: NotificationPayloadFormat;
48 /**
49 * Event types to publish. Omit (or pass an empty list) to receive every
50 * event (`OBJECT_FINALIZE`, `OBJECT_METADATA_UPDATE`, `OBJECT_DELETE`,
51 * `OBJECT_ARCHIVE`). Immutable — changing it replaces the notification.
52 */
53 eventTypes?: string[];
54 /**
55 * If set, only objects whose names start with this prefix generate
56 * notifications. Immutable — changing it replaces the notification.
57 */
58 objectNamePrefix?: string;
59 /**
60 * Extra attributes attached to every published Pub/Sub message. Alchemy
61 * writes no ownership markers here (they would reach every subscriber);
62 * the config is tracked by its server-assigned id. Immutable — changing
63 * attributes replaces the notification.
64 */
65 customAttributes?: Record<string, string>;
66 };
67
68 export type Notification = Resource<
69 "GCP.Storage.Notification",
70 NotificationProps,
71 {
72 /** Parent bucket name. */
73 bucketName: string;
74 /** Server-assigned notification id. */
75 notificationId: string;
76 /** Canonical Pub/Sub topic URI. */
77 topic: string;
78 /** Topic id (last path segment). */
79 topicId: string;
80 /** Project id used when normalizing the topic. */
81 project: string;
82 /** Payload format (`JSON_API_V1` or `NONE`). */
83 payloadFormat: string;
84 /** Event types this notification is subscribed to (empty = all). */
85 eventTypes: string[];
86 /** Object-name prefix filter, if any. */
87 objectNamePrefix: string | undefined;
88 /** User custom attributes (Alchemy ownership keys stripped). */
89 customAttributes: Record<string, string>;
90 /** GCS self-link. */
91 selfLink: string | undefined;
92 /** HTTP etag. */
93 etag: string | undefined;
94 },
95 never,
96 Providers
97 >;
98
99 /**
100 * A Cloud Storage Pub/Sub notification configuration on a bucket.
101 *
102 * GCS has no update API for notifications — every user-facing field is
103 * immutable and changing it replaces the config. The Cloud Storage
104 * service account is granted `roles/pubsub.publisher` on the topic
105 * during reconcile so events can actually be published.
106 *
107 * Notifications carry no labels, and `customAttributes` are delivered on
108 * every message, so Alchemy does not stamp ownership markers. `read` finds
109 * the config by its recorded id; without one, an identical config on the
110 * bucket is reported as unowned.
111 *
112 * ### Creating a Notification
113 * **Example:** Notify on every object event
114 * ```typescript
115 * const bucket = yield* GCP.Storage.Bucket("assets", {
116 * forceDestroy: true,
117 * });
118 * const topic = yield* GCP.PubSub.Topic("events", {});
119 * const notification = yield* GCP.Storage.Notification("object-events", {
120 * bucketName: bucket.bucketName,
121 * topic: topic.name,
122 * });
123 * ```
124 *
125 * **Example:** Filtered events, prefix, and custom attributes
126 * ```typescript
127 * const notification = yield* GCP.Storage.Notification("uploads", {
128 * bucketName: bucket.bucketName,
129 * topic: topic.name,
130 * payloadFormat: "JSON_API_V1",
131 * eventTypes: ["OBJECT_FINALIZE"],
132 * objectNamePrefix: "uploads/",
133 * customAttributes: { env: "prod" },
134 * });
135 * ```
136 *
137 * @resource
138 * @category Storage
139 */
140 export const Notification = Resource<Notification>("GCP.Storage.Notification");
141
142 export class NotificationNotResolved extends Data.TaggedError(
143 "GCP.Storage.NotificationNotResolved",
144 )<{
145 bucketName: string;
146 notificationId?: string;
147 }> {}
148
149 export class NotificationServiceAccountMissing extends Data.TaggedError(
150 "GCP.Storage.NotificationServiceAccountMissing",
151 )<{
152 project: string;
153 }> {}
154
155 const topicIdOf = (topic: string) => topic.split("/").pop() ?? topic;
156
157 const toCanonicalTopic = (topic: string, project: string) => {
158 if (topic.startsWith(CANONICAL_TOPIC_PREFIX)) return topic;
159 if (topic.startsWith("projects/")) {
160 return `${CANONICAL_TOPIC_PREFIX}${topic}`;
161 }
162 return `${CANONICAL_TOPIC_PREFIX}projects/${project}/topics/${topic}`;
163 };
164
165 const toPubsubName = (topic: string, project: string) => {
166 const canonical = toCanonicalTopic(topic, project);
167 return canonical.startsWith(CANONICAL_TOPIC_PREFIX)
168 ? canonical.slice(CANONICAL_TOPIC_PREFIX.length)
169 : canonical;
170 };
171
172 const topicKey = (topic: string, project: string) => toPubsubName(topic, project);
173
174 const userAttributes = (
175 attributes: Record<string, string | undefined> | null | undefined,
176 ): Record<string, string> => stripInternalLabels(tagRecord(attributes));
177
178 const sameAttributes = (
179 left: Record<string, string> | undefined,
180 right: Record<string, string> | undefined,
181 ) => {
182 const a = left ?? {};
183 const b = right ?? {};
184 const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
185 for (const key of keys) {
186 if (a[key] !== b[key]) return false;
187 }
188 return true;
189 };
190
191 const normalizeEventTypes = (eventTypes: string[] | undefined) =>
192 [...(eventTypes ?? [])].map((event) => event.toUpperCase()).sort();
193
194 const sameEventTypes = (left: string[] | undefined, right: string[] | undefined) =>
195 normalizeEventTypes(left).join("\0") === normalizeEventTypes(right).join("\0");
196
197 const toAttrs = (notification: storage.Notification, bucketName: string, project: string) => {
198 const topic = notification.topic ?? "";
199 return {
200 bucketName,
201 notificationId: notification.id ?? "",
202 topic,
203 topicId: topicIdOf(topic),
204 project,
205 payloadFormat: notification.payload_format ?? DEFAULT_PAYLOAD_FORMAT,
206 eventTypes: notification.event_types ?? [],
207 objectNamePrefix: notification.object_name_prefix,
208 customAttributes: userAttributes(notification.custom_attributes),
209 selfLink: notification.selfLink,
210 etag: notification.etag,
211 };
212 };
213
214 const matchesDesired = (
215 notification: storage.Notification,
216 news: NotificationProps,
217 project: string,
218 ) => {
219 const desiredTopic = toCanonicalTopic(news.topic, project);
220 const observedTopic = notification.topic ?? "";
221 const payloadFormat = news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT;
222 return (
223 topicKey(observedTopic, project) === topicKey(desiredTopic, project) &&
224 (notification.payload_format ?? DEFAULT_PAYLOAD_FORMAT) === payloadFormat &&
225 sameEventTypes(notification.event_types, news.eventTypes) &&
226 (notification.object_name_prefix ?? "") === (news.objectNamePrefix ?? "") &&
227 sameAttributes(userAttributes(notification.custom_attributes), toLabels(news.customAttributes))
228 );
229 };
230
231 const immutableChanged = (
232 news: NotificationProps,
233 olds: Partial<NotificationProps> | undefined,
234 output: Notification["Attributes"] | undefined,
235 project: string,
236 ) => {
237 const previousBucket = olds?.bucketName ?? output?.bucketName;
238 if (previousBucket !== undefined && news.bucketName !== previousBucket) {
239 return true;
240 }
241 const previousTopic = olds?.topic ?? output?.topic;
242 if (
243 previousTopic !== undefined &&
244 topicKey(news.topic, project) !== topicKey(previousTopic, project)
245 ) {
246 return true;
247 }
248 const previousPayload = olds?.payloadFormat ?? output?.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT;
249 if ((news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT) !== previousPayload) {
250 return true;
251 }
252 if (!sameEventTypes(news.eventTypes, olds?.eventTypes ?? output?.eventTypes)) {
253 return true;
254 }
255 const previousPrefix = olds?.objectNamePrefix ?? output?.objectNamePrefix ?? "";
256 if ((news.objectNamePrefix ?? "") !== previousPrefix) {
257 return true;
258 }
259 const previousAttrs = olds?.customAttributes ?? output?.customAttributes;
260 if (!sameAttributes(toLabels(news.customAttributes), previousAttrs)) {
261 return true;
262 }
263 return false;
264 };
265
266 const getById = (bucketName: string, notificationId: string) =>
267 storage
268 .getNotifications({
269 bucket: bucketName,
270 notification: notificationId,
271 })
272 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
273
274 const listOnBucket = (bucketName: string) =>
275 storage.listNotifications({ bucket: bucketName }).pipe(
276 Effect.map((page) => page.items ?? []),
277 Effect.catchTag("NotFound", () => Effect.succeed([] as Array<storage.Notification>)),
278 );
279
280 /** A config on the bucket identical to the desired one (props match). */
281 const findMatching = (bucketName: string, props: NotificationProps, project: string) =>
282 listOnBucket(bucketName).pipe(
283 Effect.map((items) => items.find((item) => matchesDesired(item, props, project))),
284 );
285
286 const waitUntilGone = (bucketName: string, notificationId: string) =>
287 getById(bucketName, notificationId).pipe(
288 Effect.map((existing) => (existing === undefined ? ("gone" as const) : ("found" as const))),
289 Effect.repeat({
290 schedule: Schedule.spaced("1 second"),
291 until: (status) => status === "gone",
292 times: 30,
293 }),
294 Effect.flatMap((status): Effect.Effect<void, DeleteNotConfirmed> =>
295 status === "gone"
296 ? Effect.void
297 : Effect.fail(
298 new DeleteNotConfirmed({
299 resource: `${bucketName}/notificationConfigs/${notificationId}`,
300 }),
301 ),
302 ),
303 );
304
305 const deleteById = (bucketName: string, notificationId: string) =>
306 storage
307 .deleteNotifications({
308 bucket: bucketName,
309 notification: notificationId,
310 })
311 .pipe(Effect.catchTag("NotFound", () => Effect.void));
312
313 const ensureGcsServiceIdentity = (project: string, email: string) =>
314 Effect.gen(function* () {
315 const projectNumber = email.match(/^service-(\d+)@/)?.[1] ?? project;
316 const operation = yield* serviceusage
317 .generateServiceIdentityServices({
318 parent: `projects/${projectNumber}/services/storage.googleapis.com`,
319 })
320 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
321 if (operation === undefined) return;
322 yield* waitForOperation(operation, (name) => serviceusage.getOperations({ name }), {
323 budget: "2 minutes",
324 interval: "2 seconds",
325 });
326 });
327
328 const ensureTopicPublisher = (project: string, topic: string) =>
329 Effect.gen(function* () {
330 const serviceAccount = yield* storage.getProjectsServiceAccount({
331 projectId: project,
332 });
333 const email = serviceAccount.email_address;
334 if (!email) {
335 return yield* new NotificationServiceAccountMissing({ project });
336 }
337 yield* ensureGcsServiceIdentity(project, email);
338 const resource = toPubsubName(topic, project);
339 const member = `serviceAccount:${email}`;
340 const grant = Effect.gen(function* () {
341 const policy = yield* pubsub.getIamPolicyProjectsTopics({
342 resource,
343 });
344 const bindings = (policy.bindings ?? []).map((binding) => ({
345 ...binding,
346 members: [...(binding.members ?? [])],
347 }));
348 const publisher = bindings.find((binding) => binding.role === PUBLISHER_ROLE);
349 if (publisher?.members?.includes(member)) {
350 return;
351 }
352 if (publisher) {
353 publisher.members = [...(publisher.members ?? []), member];
354 } else {
355 bindings.push({ role: PUBLISHER_ROLE, members: [member] });
356 }
357 yield* pubsub.setIamPolicyProjectsTopics({
358 resource,
359 body: {
360 policy: {
361 ...policy,
362 bindings,
363 },
364 },
365 });
366 });
367 yield* grant.pipe(
368 Effect.retry({
369 while: (error) => error._tag === "Conflict" || error._tag === "IamMemberNotFound",
370 times: 8,
371 schedule: Schedule.spaced("2 seconds"),
372 }),
373 );
374 });
375
376 export const NotificationProvider = () =>
377 Provider.succeed(Notification, {
378 stables: ["notificationId", "bucketName", "project", "selfLink"],
379
380 diff: Effect.fn(function* ({ news, olds, output }) {
381 if (!isResolved(news)) return undefined;
382 if (output === undefined && olds === undefined) return undefined;
383 const env = yield* GcpEnvironment.current;
384 if (immutableChanged(news, olds, output, env.project)) {
385 return { action: "replace" as const, deleteFirst: true };
386 }
387 return undefined;
388 }),
389
390 read: Effect.fn(function* ({ id, olds, output, previous }) {
391 const env = yield* GcpEnvironment.current;
392 const bucketName = olds?.bucketName ?? output?.bucketName;
393 if (!bucketName) return undefined;
394 let existing =
395 output?.notificationId !== undefined
396 ? yield* getById(bucketName, output.notificationId)
397 : undefined;
398 if (existing === undefined && olds !== undefined) {
399 existing = yield* findMatching(bucketName, olds, env.project);
400 }
401 if (existing === undefined) return undefined;
402 const attrs = toAttrs(existing, bucketName, env.project);
403 // Configs created before markers were dropped still carry them.
404 const owned =
405 (existing.id !== undefined && existing.id === output?.notificationId) ||
406 (yield* hasAlchemyLabels(id, tagRecord(existing.custom_attributes)));
407 return owned ? attrs : unownedUnlessPrevious(attrs, previous, "notificationId");
408 }),
409
410 list: () =>
411 Effect.gen(function* () {
412 const env = yield* GcpEnvironment.current;
413 const buckets = yield* storage.listBuckets
414 .items({ project: env.project, maxResults: 1000 })
415 .pipe(
416 Stream.runCollect,
417 Effect.map((chunk) => Array.from(chunk)),
418 );
419 const pages = yield* Effect.forEach(
420 buckets,
421 (bucket) => {
422 const bucketName = bucket.name;
423 if (!bucketName) {
424 return Effect.succeed([] as Array<Notification["Attributes"]>);
425 }
426 // Only configs from before ownership markers were dropped are
427 // identifiable; the rest go away with their bucket.
428 return listOnBucket(bucketName).pipe(
429 Effect.map((items) =>
430 items
431 .filter((item) =>
432 Object.keys(item.custom_attributes ?? {}).some((key) =>
433 key.startsWith(ALCHEMY_LABEL_PREFIX),
434 ),
435 )
436 .map((item) => toAttrs(item, bucketName, env.project)),
437 ),
438 );
439 },
440 { concurrency: 8 },
441 );
442 return pages.flat();
443 }),
444
445 reconcile: Effect.fn(function* ({ news, output }) {
446 const env = yield* GcpEnvironment.current;
447 const bucketName = news.bucketName;
448 const payloadFormat = news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT;
449 const canonicalTopic = toCanonicalTopic(news.topic, env.project);
450 const desiredAttributes = toLabels(news.customAttributes);
451
452 let current: storage.Notification | undefined;
453 if (output?.notificationId) {
454 current = yield* getById(output.bucketName ?? bucketName, output.notificationId);
455 }
456 if (current === undefined) {
457 current = yield* findMatching(bucketName, news, env.project);
458 }
459
460 if (current === undefined) {
461 yield* ensureTopicPublisher(env.project, canonicalTopic);
462 const created = yield* storage
463 .insertNotifications({
464 bucket: bucketName,
465 body: {
466 topic: canonicalTopic,
467 payload_format: payloadFormat,
468 event_types:
469 news.eventTypes && news.eventTypes.length > 0 ? news.eventTypes : undefined,
470 object_name_prefix: news.objectNamePrefix,
471 custom_attributes: desiredAttributes,
472 },
473 })
474 .pipe(
475 Effect.retry({
476 while: (error) => error._tag === "NotificationTopicUnavailable",
477 times: 8,
478 schedule: Schedule.spaced("2 seconds"),
479 }),
480 );
481 current = created;
482 }
483
484 if (current === undefined || !current.id) {
485 return yield* new NotificationNotResolved({
486 bucketName,
487 notificationId: output?.notificationId,
488 });
489 }
490
491 return toAttrs(current, bucketName, env.project);
492 }),
493
494 delete: Effect.fn(function* ({ output }) {
495 const bucketName = output.bucketName;
496 const notificationId = output.notificationId;
497 if (!bucketName || !notificationId) return;
498 yield* deleteById(bucketName, notificationId);
499 yield* waitUntilGone(bucketName, notificationId);
500 }),
501 });