The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as pubsub from "@distilled.cloud/gcp/pubsub_v1"; |
| 2 | import * as storage from "@distilled.cloud/gcp/storage_v1"; |
| 3 | import * as serviceusage from "@distilled.cloud/gcp/unstable/serviceusage_v1beta1"; |
| 4 | import { unownedUnlessPrevious } from "alchemy/AdoptPolicy"; |
| 5 | import { isResolved } from "alchemy/Diff"; |
| 6 | import * as Provider from "alchemy/Provider"; |
| 7 | import { Resource } from "alchemy/Resource"; |
| 8 | import { tagRecord } from "alchemy/Tags"; |
| 9 | import * as Data from "effect/Data"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Schedule from "effect/Schedule"; |
| 12 | import * as Stream from "effect/Stream"; |
| 13 | import { GcpEnvironment } from "../Environment.ts"; |
| 14 | import { DeleteNotConfirmed } from "../Errors.ts"; |
| 15 | import { |
| 16 | ALCHEMY_LABEL_PREFIX, |
| 17 | hasAlchemyLabels, |
| 18 | stripInternalLabels, |
| 19 | toLabels, |
| 20 | } from "../Labels.ts"; |
| 21 | import { waitForOperation } from "../Operation.ts"; |
| 22 | import type { Providers } from "../Providers.ts"; |
| 23 | |
| 24 | const DEFAULT_PAYLOAD_FORMAT = "JSON_API_V1"; |
| 25 | const CANONICAL_TOPIC_PREFIX = "//pubsub.googleapis.com/"; |
| 26 | const PUBLISHER_ROLE = "roles/pubsub.publisher"; |
| 27 | |
| 28 | export type NotificationPayloadFormat = "JSON_API_V1" | "NONE"; |
| 29 | |
| 30 | export type NotificationProps = { |
| 31 | /** |
| 32 | * Name of the Cloud Storage bucket this notification is attached to. |
| 33 | * Immutable — changing it replaces the notification. |
| 34 | */ |
| 35 | bucketName: string; |
| 36 | /** |
| 37 | * Cloud Pub/Sub topic to publish events to. Accepts a topic id, a |
| 38 | * resource name (`projects/{project}/topics/{topic}`), or the canonical |
| 39 | * URI (`//pubsub.googleapis.com/projects/{project}/topics/{topic}`). |
| 40 | * Immutable — changing it replaces the notification. |
| 41 | */ |
| 42 | topic: string; |
| 43 | /** |
| 44 | * Payload included in each Pub/Sub message. |
| 45 | * @default "JSON_API_V1" |
| 46 | */ |
| 47 | payloadFormat?: NotificationPayloadFormat; |
| 48 | /** |
| 49 | * Event types to publish. Omit (or pass an empty list) to receive every |
| 50 | * event (`OBJECT_FINALIZE`, `OBJECT_METADATA_UPDATE`, `OBJECT_DELETE`, |
| 51 | * `OBJECT_ARCHIVE`). Immutable — changing it replaces the notification. |
| 52 | */ |
| 53 | eventTypes?: string[]; |
| 54 | /** |
| 55 | * If set, only objects whose names start with this prefix generate |
| 56 | * notifications. Immutable — changing it replaces the notification. |
| 57 | */ |
| 58 | objectNamePrefix?: string; |
| 59 | /** |
| 60 | * Extra attributes attached to every published Pub/Sub message. Alchemy |
| 61 | * writes no ownership markers here (they would reach every subscriber); |
| 62 | * the config is tracked by its server-assigned id. Immutable — changing |
| 63 | * attributes replaces the notification. |
| 64 | */ |
| 65 | customAttributes?: Record<string, string>; |
| 66 | }; |
| 67 | |
| 68 | export type Notification = Resource< |
| 69 | "GCP.Storage.Notification", |
| 70 | NotificationProps, |
| 71 | { |
| 72 | /** Parent bucket name. */ |
| 73 | bucketName: string; |
| 74 | /** Server-assigned notification id. */ |
| 75 | notificationId: string; |
| 76 | /** Canonical Pub/Sub topic URI. */ |
| 77 | topic: string; |
| 78 | /** Topic id (last path segment). */ |
| 79 | topicId: string; |
| 80 | /** Project id used when normalizing the topic. */ |
| 81 | project: string; |
| 82 | /** Payload format (`JSON_API_V1` or `NONE`). */ |
| 83 | payloadFormat: string; |
| 84 | /** Event types this notification is subscribed to (empty = all). */ |
| 85 | eventTypes: string[]; |
| 86 | /** Object-name prefix filter, if any. */ |
| 87 | objectNamePrefix: string | undefined; |
| 88 | /** User custom attributes (Alchemy ownership keys stripped). */ |
| 89 | customAttributes: Record<string, string>; |
| 90 | /** GCS self-link. */ |
| 91 | selfLink: string | undefined; |
| 92 | /** HTTP etag. */ |
| 93 | etag: string | undefined; |
| 94 | }, |
| 95 | never, |
| 96 | Providers |
| 97 | >; |
| 98 | |
| 99 | /** |
| 100 | * A Cloud Storage Pub/Sub notification configuration on a bucket. |
| 101 | * |
| 102 | * GCS has no update API for notifications — every user-facing field is |
| 103 | * immutable and changing it replaces the config. The Cloud Storage |
| 104 | * service account is granted `roles/pubsub.publisher` on the topic |
| 105 | * during reconcile so events can actually be published. |
| 106 | * |
| 107 | * Notifications carry no labels, and `customAttributes` are delivered on |
| 108 | * every message, so Alchemy does not stamp ownership markers. `read` finds |
| 109 | * the config by its recorded id; without one, an identical config on the |
| 110 | * bucket is reported as unowned. |
| 111 | * |
| 112 | * ### Creating a Notification |
| 113 | * **Example:** Notify on every object event |
| 114 | * ```typescript |
| 115 | * const bucket = yield* GCP.Storage.Bucket("assets", { |
| 116 | * forceDestroy: true, |
| 117 | * }); |
| 118 | * const topic = yield* GCP.PubSub.Topic("events", {}); |
| 119 | * const notification = yield* GCP.Storage.Notification("object-events", { |
| 120 | * bucketName: bucket.bucketName, |
| 121 | * topic: topic.name, |
| 122 | * }); |
| 123 | * ``` |
| 124 | * |
| 125 | * **Example:** Filtered events, prefix, and custom attributes |
| 126 | * ```typescript |
| 127 | * const notification = yield* GCP.Storage.Notification("uploads", { |
| 128 | * bucketName: bucket.bucketName, |
| 129 | * topic: topic.name, |
| 130 | * payloadFormat: "JSON_API_V1", |
| 131 | * eventTypes: ["OBJECT_FINALIZE"], |
| 132 | * objectNamePrefix: "uploads/", |
| 133 | * customAttributes: { env: "prod" }, |
| 134 | * }); |
| 135 | * ``` |
| 136 | * |
| 137 | * @resource |
| 138 | * @category Storage |
| 139 | */ |
| 140 | export const Notification = Resource<Notification>("GCP.Storage.Notification"); |
| 141 | |
| 142 | export class NotificationNotResolved extends Data.TaggedError( |
| 143 | "GCP.Storage.NotificationNotResolved", |
| 144 | )<{ |
| 145 | bucketName: string; |
| 146 | notificationId?: string; |
| 147 | }> {} |
| 148 | |
| 149 | export class NotificationServiceAccountMissing extends Data.TaggedError( |
| 150 | "GCP.Storage.NotificationServiceAccountMissing", |
| 151 | )<{ |
| 152 | project: string; |
| 153 | }> {} |
| 154 | |
| 155 | const topicIdOf = (topic: string) => topic.split("/").pop() ?? topic; |
| 156 | |
| 157 | const toCanonicalTopic = (topic: string, project: string) => { |
| 158 | if (topic.startsWith(CANONICAL_TOPIC_PREFIX)) return topic; |
| 159 | if (topic.startsWith("projects/")) { |
| 160 | return `${CANONICAL_TOPIC_PREFIX}${topic}`; |
| 161 | } |
| 162 | return `${CANONICAL_TOPIC_PREFIX}projects/${project}/topics/${topic}`; |
| 163 | }; |
| 164 | |
| 165 | const toPubsubName = (topic: string, project: string) => { |
| 166 | const canonical = toCanonicalTopic(topic, project); |
| 167 | return canonical.startsWith(CANONICAL_TOPIC_PREFIX) |
| 168 | ? canonical.slice(CANONICAL_TOPIC_PREFIX.length) |
| 169 | : canonical; |
| 170 | }; |
| 171 | |
| 172 | const topicKey = (topic: string, project: string) => toPubsubName(topic, project); |
| 173 | |
| 174 | const userAttributes = ( |
| 175 | attributes: Record<string, string | undefined> | null | undefined, |
| 176 | ): Record<string, string> => stripInternalLabels(tagRecord(attributes)); |
| 177 | |
| 178 | const sameAttributes = ( |
| 179 | left: Record<string, string> | undefined, |
| 180 | right: Record<string, string> | undefined, |
| 181 | ) => { |
| 182 | const a = left ?? {}; |
| 183 | const b = right ?? {}; |
| 184 | const keys = new Set([...Object.keys(a), ...Object.keys(b)]); |
| 185 | for (const key of keys) { |
| 186 | if (a[key] !== b[key]) return false; |
| 187 | } |
| 188 | return true; |
| 189 | }; |
| 190 | |
| 191 | const normalizeEventTypes = (eventTypes: string[] | undefined) => |
| 192 | [...(eventTypes ?? [])].map((event) => event.toUpperCase()).sort(); |
| 193 | |
| 194 | const sameEventTypes = (left: string[] | undefined, right: string[] | undefined) => |
| 195 | normalizeEventTypes(left).join("\0") === normalizeEventTypes(right).join("\0"); |
| 196 | |
| 197 | const toAttrs = (notification: storage.Notification, bucketName: string, project: string) => { |
| 198 | const topic = notification.topic ?? ""; |
| 199 | return { |
| 200 | bucketName, |
| 201 | notificationId: notification.id ?? "", |
| 202 | topic, |
| 203 | topicId: topicIdOf(topic), |
| 204 | project, |
| 205 | payloadFormat: notification.payload_format ?? DEFAULT_PAYLOAD_FORMAT, |
| 206 | eventTypes: notification.event_types ?? [], |
| 207 | objectNamePrefix: notification.object_name_prefix, |
| 208 | customAttributes: userAttributes(notification.custom_attributes), |
| 209 | selfLink: notification.selfLink, |
| 210 | etag: notification.etag, |
| 211 | }; |
| 212 | }; |
| 213 | |
| 214 | const matchesDesired = ( |
| 215 | notification: storage.Notification, |
| 216 | news: NotificationProps, |
| 217 | project: string, |
| 218 | ) => { |
| 219 | const desiredTopic = toCanonicalTopic(news.topic, project); |
| 220 | const observedTopic = notification.topic ?? ""; |
| 221 | const payloadFormat = news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT; |
| 222 | return ( |
| 223 | topicKey(observedTopic, project) === topicKey(desiredTopic, project) && |
| 224 | (notification.payload_format ?? DEFAULT_PAYLOAD_FORMAT) === payloadFormat && |
| 225 | sameEventTypes(notification.event_types, news.eventTypes) && |
| 226 | (notification.object_name_prefix ?? "") === (news.objectNamePrefix ?? "") && |
| 227 | sameAttributes(userAttributes(notification.custom_attributes), toLabels(news.customAttributes)) |
| 228 | ); |
| 229 | }; |
| 230 | |
| 231 | const immutableChanged = ( |
| 232 | news: NotificationProps, |
| 233 | olds: Partial<NotificationProps> | undefined, |
| 234 | output: Notification["Attributes"] | undefined, |
| 235 | project: string, |
| 236 | ) => { |
| 237 | const previousBucket = olds?.bucketName ?? output?.bucketName; |
| 238 | if (previousBucket !== undefined && news.bucketName !== previousBucket) { |
| 239 | return true; |
| 240 | } |
| 241 | const previousTopic = olds?.topic ?? output?.topic; |
| 242 | if ( |
| 243 | previousTopic !== undefined && |
| 244 | topicKey(news.topic, project) !== topicKey(previousTopic, project) |
| 245 | ) { |
| 246 | return true; |
| 247 | } |
| 248 | const previousPayload = olds?.payloadFormat ?? output?.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT; |
| 249 | if ((news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT) !== previousPayload) { |
| 250 | return true; |
| 251 | } |
| 252 | if (!sameEventTypes(news.eventTypes, olds?.eventTypes ?? output?.eventTypes)) { |
| 253 | return true; |
| 254 | } |
| 255 | const previousPrefix = olds?.objectNamePrefix ?? output?.objectNamePrefix ?? ""; |
| 256 | if ((news.objectNamePrefix ?? "") !== previousPrefix) { |
| 257 | return true; |
| 258 | } |
| 259 | const previousAttrs = olds?.customAttributes ?? output?.customAttributes; |
| 260 | if (!sameAttributes(toLabels(news.customAttributes), previousAttrs)) { |
| 261 | return true; |
| 262 | } |
| 263 | return false; |
| 264 | }; |
| 265 | |
| 266 | const getById = (bucketName: string, notificationId: string) => |
| 267 | storage |
| 268 | .getNotifications({ |
| 269 | bucket: bucketName, |
| 270 | notification: notificationId, |
| 271 | }) |
| 272 | .pipe(Effect.catchTag("NotFound", () => Effect.undefined)); |
| 273 | |
| 274 | const listOnBucket = (bucketName: string) => |
| 275 | storage.listNotifications({ bucket: bucketName }).pipe( |
| 276 | Effect.map((page) => page.items ?? []), |
| 277 | Effect.catchTag("NotFound", () => Effect.succeed([] as Array<storage.Notification>)), |
| 278 | ); |
| 279 | |
| 280 | /** A config on the bucket identical to the desired one (props match). */ |
| 281 | const findMatching = (bucketName: string, props: NotificationProps, project: string) => |
| 282 | listOnBucket(bucketName).pipe( |
| 283 | Effect.map((items) => items.find((item) => matchesDesired(item, props, project))), |
| 284 | ); |
| 285 | |
| 286 | const waitUntilGone = (bucketName: string, notificationId: string) => |
| 287 | getById(bucketName, notificationId).pipe( |
| 288 | Effect.map((existing) => (existing === undefined ? ("gone" as const) : ("found" as const))), |
| 289 | Effect.repeat({ |
| 290 | schedule: Schedule.spaced("1 second"), |
| 291 | until: (status) => status === "gone", |
| 292 | times: 30, |
| 293 | }), |
| 294 | Effect.flatMap((status): Effect.Effect<void, DeleteNotConfirmed> => |
| 295 | status === "gone" |
| 296 | ? Effect.void |
| 297 | : Effect.fail( |
| 298 | new DeleteNotConfirmed({ |
| 299 | resource: `${bucketName}/notificationConfigs/${notificationId}`, |
| 300 | }), |
| 301 | ), |
| 302 | ), |
| 303 | ); |
| 304 | |
| 305 | const deleteById = (bucketName: string, notificationId: string) => |
| 306 | storage |
| 307 | .deleteNotifications({ |
| 308 | bucket: bucketName, |
| 309 | notification: notificationId, |
| 310 | }) |
| 311 | .pipe(Effect.catchTag("NotFound", () => Effect.void)); |
| 312 | |
| 313 | const ensureGcsServiceIdentity = (project: string, email: string) => |
| 314 | Effect.gen(function* () { |
| 315 | const projectNumber = email.match(/^service-(\d+)@/)?.[1] ?? project; |
| 316 | const operation = yield* serviceusage |
| 317 | .generateServiceIdentityServices({ |
| 318 | parent: `projects/${projectNumber}/services/storage.googleapis.com`, |
| 319 | }) |
| 320 | .pipe(Effect.catchTag("Conflict", () => Effect.undefined)); |
| 321 | if (operation === undefined) return; |
| 322 | yield* waitForOperation(operation, (name) => serviceusage.getOperations({ name }), { |
| 323 | budget: "2 minutes", |
| 324 | interval: "2 seconds", |
| 325 | }); |
| 326 | }); |
| 327 | |
| 328 | const ensureTopicPublisher = (project: string, topic: string) => |
| 329 | Effect.gen(function* () { |
| 330 | const serviceAccount = yield* storage.getProjectsServiceAccount({ |
| 331 | projectId: project, |
| 332 | }); |
| 333 | const email = serviceAccount.email_address; |
| 334 | if (!email) { |
| 335 | return yield* new NotificationServiceAccountMissing({ project }); |
| 336 | } |
| 337 | yield* ensureGcsServiceIdentity(project, email); |
| 338 | const resource = toPubsubName(topic, project); |
| 339 | const member = `serviceAccount:${email}`; |
| 340 | const grant = Effect.gen(function* () { |
| 341 | const policy = yield* pubsub.getIamPolicyProjectsTopics({ |
| 342 | resource, |
| 343 | }); |
| 344 | const bindings = (policy.bindings ?? []).map((binding) => ({ |
| 345 | ...binding, |
| 346 | members: [...(binding.members ?? [])], |
| 347 | })); |
| 348 | const publisher = bindings.find((binding) => binding.role === PUBLISHER_ROLE); |
| 349 | if (publisher?.members?.includes(member)) { |
| 350 | return; |
| 351 | } |
| 352 | if (publisher) { |
| 353 | publisher.members = [...(publisher.members ?? []), member]; |
| 354 | } else { |
| 355 | bindings.push({ role: PUBLISHER_ROLE, members: [member] }); |
| 356 | } |
| 357 | yield* pubsub.setIamPolicyProjectsTopics({ |
| 358 | resource, |
| 359 | body: { |
| 360 | policy: { |
| 361 | ...policy, |
| 362 | bindings, |
| 363 | }, |
| 364 | }, |
| 365 | }); |
| 366 | }); |
| 367 | yield* grant.pipe( |
| 368 | Effect.retry({ |
| 369 | while: (error) => error._tag === "Conflict" || error._tag === "IamMemberNotFound", |
| 370 | times: 8, |
| 371 | schedule: Schedule.spaced("2 seconds"), |
| 372 | }), |
| 373 | ); |
| 374 | }); |
| 375 | |
| 376 | export const NotificationProvider = () => |
| 377 | Provider.succeed(Notification, { |
| 378 | stables: ["notificationId", "bucketName", "project", "selfLink"], |
| 379 | |
| 380 | diff: Effect.fn(function* ({ news, olds, output }) { |
| 381 | if (!isResolved(news)) return undefined; |
| 382 | if (output === undefined && olds === undefined) return undefined; |
| 383 | const env = yield* GcpEnvironment.current; |
| 384 | if (immutableChanged(news, olds, output, env.project)) { |
| 385 | return { action: "replace" as const, deleteFirst: true }; |
| 386 | } |
| 387 | return undefined; |
| 388 | }), |
| 389 | |
| 390 | read: Effect.fn(function* ({ id, olds, output, previous }) { |
| 391 | const env = yield* GcpEnvironment.current; |
| 392 | const bucketName = olds?.bucketName ?? output?.bucketName; |
| 393 | if (!bucketName) return undefined; |
| 394 | let existing = |
| 395 | output?.notificationId !== undefined |
| 396 | ? yield* getById(bucketName, output.notificationId) |
| 397 | : undefined; |
| 398 | if (existing === undefined && olds !== undefined) { |
| 399 | existing = yield* findMatching(bucketName, olds, env.project); |
| 400 | } |
| 401 | if (existing === undefined) return undefined; |
| 402 | const attrs = toAttrs(existing, bucketName, env.project); |
| 403 | // Configs created before markers were dropped still carry them. |
| 404 | const owned = |
| 405 | (existing.id !== undefined && existing.id === output?.notificationId) || |
| 406 | (yield* hasAlchemyLabels(id, tagRecord(existing.custom_attributes))); |
| 407 | return owned ? attrs : unownedUnlessPrevious(attrs, previous, "notificationId"); |
| 408 | }), |
| 409 | |
| 410 | list: () => |
| 411 | Effect.gen(function* () { |
| 412 | const env = yield* GcpEnvironment.current; |
| 413 | const buckets = yield* storage.listBuckets |
| 414 | .items({ project: env.project, maxResults: 1000 }) |
| 415 | .pipe( |
| 416 | Stream.runCollect, |
| 417 | Effect.map((chunk) => Array.from(chunk)), |
| 418 | ); |
| 419 | const pages = yield* Effect.forEach( |
| 420 | buckets, |
| 421 | (bucket) => { |
| 422 | const bucketName = bucket.name; |
| 423 | if (!bucketName) { |
| 424 | return Effect.succeed([] as Array<Notification["Attributes"]>); |
| 425 | } |
| 426 | // Only configs from before ownership markers were dropped are |
| 427 | // identifiable; the rest go away with their bucket. |
| 428 | return listOnBucket(bucketName).pipe( |
| 429 | Effect.map((items) => |
| 430 | items |
| 431 | .filter((item) => |
| 432 | Object.keys(item.custom_attributes ?? {}).some((key) => |
| 433 | key.startsWith(ALCHEMY_LABEL_PREFIX), |
| 434 | ), |
| 435 | ) |
| 436 | .map((item) => toAttrs(item, bucketName, env.project)), |
| 437 | ), |
| 438 | ); |
| 439 | }, |
| 440 | { concurrency: 8 }, |
| 441 | ); |
| 442 | return pages.flat(); |
| 443 | }), |
| 444 | |
| 445 | reconcile: Effect.fn(function* ({ news, output }) { |
| 446 | const env = yield* GcpEnvironment.current; |
| 447 | const bucketName = news.bucketName; |
| 448 | const payloadFormat = news.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT; |
| 449 | const canonicalTopic = toCanonicalTopic(news.topic, env.project); |
| 450 | const desiredAttributes = toLabels(news.customAttributes); |
| 451 | |
| 452 | let current: storage.Notification | undefined; |
| 453 | if (output?.notificationId) { |
| 454 | current = yield* getById(output.bucketName ?? bucketName, output.notificationId); |
| 455 | } |
| 456 | if (current === undefined) { |
| 457 | current = yield* findMatching(bucketName, news, env.project); |
| 458 | } |
| 459 | |
| 460 | if (current === undefined) { |
| 461 | yield* ensureTopicPublisher(env.project, canonicalTopic); |
| 462 | const created = yield* storage |
| 463 | .insertNotifications({ |
| 464 | bucket: bucketName, |
| 465 | body: { |
| 466 | topic: canonicalTopic, |
| 467 | payload_format: payloadFormat, |
| 468 | event_types: |
| 469 | news.eventTypes && news.eventTypes.length > 0 ? news.eventTypes : undefined, |
| 470 | object_name_prefix: news.objectNamePrefix, |
| 471 | custom_attributes: desiredAttributes, |
| 472 | }, |
| 473 | }) |
| 474 | .pipe( |
| 475 | Effect.retry({ |
| 476 | while: (error) => error._tag === "NotificationTopicUnavailable", |
| 477 | times: 8, |
| 478 | schedule: Schedule.spaced("2 seconds"), |
| 479 | }), |
| 480 | ); |
| 481 | current = created; |
| 482 | } |
| 483 | |
| 484 | if (current === undefined || !current.id) { |
| 485 | return yield* new NotificationNotResolved({ |
| 486 | bucketName, |
| 487 | notificationId: output?.notificationId, |
| 488 | }); |
| 489 | } |
| 490 | |
| 491 | return toAttrs(current, bucketName, env.project); |
| 492 | }), |
| 493 | |
| 494 | delete: Effect.fn(function* ({ output }) { |
| 495 | const bucketName = output.bucketName; |
| 496 | const notificationId = output.notificationId; |
| 497 | if (!bucketName || !notificationId) return; |
| 498 | yield* deleteById(bucketName, notificationId); |
| 499 | yield* waitUntilGone(bucketName, notificationId); |
| 500 | }), |
| 501 | }); |