The best way to control Claude Code, Codex and more from any computer or phone

main
ts 631 lines 20.6 KB
Raw
1 import * as eventarc from "@distilled.cloud/gcp/eventarc_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 type LoggingConfig,
14 changedFields,
15 compact,
16 cryptoKeyKey,
17 collectPages,
18 expandResource,
19 expandTopic,
20 hasAlchemyLabelKeys,
21 loggingKey,
22 normalizeLocation,
23 parentOf,
24 parseName,
25 resourceName,
26 rfc1035,
27 sameJson,
28 textKey,
29 toLoggingConfig,
30 toPhysicalId,
31 userAnnotations,
32 userLabels,
33 retryOnTransient,
34 waitForOperation,
35 waitUntilGone,
36 waitUntilPresent,
37 } from "./internal.ts";
38
39 const COLLECTION = "pipelines";
40
41 export type PipelinePayloadSchema = {
42 /** Entire schema definition stored inline. */
43 schemaDefinition?: string;
44 };
45
46 export type PipelinePayloadFormat = {
47 /** Protobuf payload format. */
48 protobuf?: PipelinePayloadSchema;
49 /** Avro payload format. */
50 avro?: PipelinePayloadSchema;
51 /** JSON payload format. An empty object selects JSON. */
52 json?: Record<string, never>;
53 };
54
55 export type PipelineTransformation = {
56 /**
57 * CEL expression template applied to transform messages.
58 */
59 transformationTemplate?: string;
60 };
61
62 export type PipelineMediation = {
63 /**
64 * How the pipeline transforms messages. Currently the only mediation.
65 */
66 transformation?: PipelineTransformation;
67 };
68
69 export type PipelineRetryPolicy = {
70 /**
71 * Maximum delivery attempts (1-100). Default 5.
72 */
73 maxAttempts?: number;
74 /**
75 * Minimum wait between retries, as a duration string (e.g. `"5s"`).
76 * 1-600 seconds. Default `"5s"`.
77 */
78 minRetryDelay?: string;
79 /**
80 * Maximum wait between retries, as a duration string (e.g. `"60s"`).
81 * 1-600 seconds. Default `"60s"`.
82 */
83 maxRetryDelay?: string;
84 };
85
86 export type PipelineDestinationNetworkConfig = {
87 /**
88 * Network attachment granting the pipeline access to the consumer
89 * VPC, as
90 * `projects/{project}/regions/{region}/networkAttachments/{name}`.
91 */
92 networkAttachment?: string;
93 };
94
95 export type PipelineDestinationHttpEndpoint = {
96 /**
97 * HTTPS URI of the HTTP endpoint (`https://svc.example.com/route`).
98 */
99 uri: string;
100 /**
101 * CEL expression that rewrites the destination-bound HTTP request.
102 */
103 messageBindingTemplate?: string;
104 };
105
106 export type PipelineOauthToken = {
107 /**
108 * Service account email used to mint the OAuth token. The caller
109 * needs `iam.serviceAccounts.actAs` on this account.
110 */
111 serviceAccount?: string;
112 /**
113 * OAuth scope. Defaults to `https://www.googleapis.com/auth/cloud-platform`.
114 */
115 scope?: string;
116 };
117
118 export type PipelineOidcToken = {
119 /**
120 * Service account email used to mint the OIDC token.
121 */
122 serviceAccount?: string;
123 /**
124 * Audience claim. Defaults to the destination URI.
125 */
126 audience?: string;
127 };
128
129 export type PipelineAuthenticationConfig = {
130 /** Attach an OAuth2 access token as an `Authorization` header. */
131 oauthToken?: PipelineOauthToken;
132 /** Attach a Google OIDC token as an `Authorization` header. */
133 googleOidc?: PipelineOidcToken;
134 };
135
136 export type PipelineDestination = {
137 /**
138 * HTTP endpoint destination. HTTPS only.
139 */
140 httpEndpoint?: PipelineDestinationHttpEndpoint;
141 /**
142 * Workflow whose executions are triggered by events, as
143 * `projects/{project}/locations/{location}/workflows/{workflow}`.
144 */
145 workflow?: string;
146 /**
147 * Pub/Sub topic to publish to, as `projects/{project}/topics/{topic}`.
148 */
149 topic?: string;
150 /**
151 * MessageBus to publish to, as
152 * `projects/{project}/locations/{location}/messageBuses/{messageBus}`.
153 */
154 messageBus?: string;
155 /**
156 * VPC network config used to resolve and reach the destination.
157 */
158 networkConfig?: PipelineDestinationNetworkConfig;
159 /**
160 * Authentication attached to destination requests.
161 */
162 authenticationConfig?: PipelineAuthenticationConfig;
163 /**
164 * Message format delivered to the destination. Requires
165 * `inputPayloadFormat` on the pipeline.
166 */
167 outputPayloadFormat?: PipelinePayloadFormat;
168 };
169
170 export type PipelineProps = {
171 /**
172 * Pipeline id (the `{pipeline}` segment of
173 * `projects/{project}/locations/{location}/pipelines/{pipeline}`). If
174 * omitted, a unique name is generated. Must match
175 * `[a-z]([a-z0-9-]*[a-z0-9])?` and be 1-63 characters. Immutable —
176 * changing it replaces the pipeline.
177 */
178 pipelineId?: string;
179 /**
180 * Eventarc Advanced location (`us-central1`, `us-east4`, …). Immutable
181 * — changing it replaces the pipeline. `US-CENTRAL1` is accepted and
182 * normalized to `us-central1`.
183 * @default the stack's GCP region (`GCP.Region`, else the profile
184 * region, else `us-central1`)
185 */
186 location?: string;
187 /**
188 * Destinations messages are forwarded to. The API currently accepts
189 * exactly one destination.
190 */
191 destinations: PipelineDestination[];
192 /**
193 * Payload format expected of incoming messages. Unset treats the
194 * payload as opaque bytes.
195 */
196 inputPayloadFormat?: PipelinePayloadFormat;
197 /**
198 * Mediation operations. Currently at most one transformation.
199 */
200 mediations?: PipelineMediation[];
201 /**
202 * Retry policy for non-responsive or retryable destinations.
203 */
204 retryPolicy?: PipelineRetryPolicy;
205 /**
206 * Human-readable display name.
207 */
208 displayName?: string;
209 /**
210 * Customer-managed Cloud KMS key used to encrypt/decrypt event data,
211 * as
212 * `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
213 */
214 cryptoKeyName?: string;
215 /**
216 * Platform logging configuration.
217 */
218 loggingConfig?: LoggingConfig;
219 /**
220 * User labels. Alchemy ownership labels are merged in automatically.
221 */
222 labels?: Record<string, string>;
223 /**
224 * Free-form annotations.
225 */
226 annotations?: Record<string, string>;
227 };
228
229 export type Pipeline = Resource<
230 "GCP.Eventarc.Pipeline",
231 PipelineProps,
232 {
233 /** Full resource name `projects/{project}/locations/{location}/pipelines/{pipeline}`. */
234 name: string;
235 /** Pipeline id (last path segment). */
236 pipelineId: string;
237 /** Project id. */
238 project: string;
239 /** Location id (`us-central1`, …). */
240 location: string;
241 /** Configured destinations. */
242 destinations: PipelineDestination[];
243 /** Incoming payload format. */
244 inputPayloadFormat: PipelinePayloadFormat | undefined;
245 /** Mediation operations. */
246 mediations: PipelineMediation[] | undefined;
247 /** Retry policy. */
248 retryPolicy: PipelineRetryPolicy | undefined;
249 /** Display name. */
250 displayName: string | undefined;
251 /** Customer-managed KMS key, if any. */
252 cryptoKeyName: string | undefined;
253 /** Platform logging configuration. */
254 loggingConfig: LoggingConfig | undefined;
255 /** User labels (Alchemy ownership labels stripped). */
256 labels: Record<string, string>;
257 /** User annotations. */
258 annotations: Record<string, string>;
259 /** Whether the pipeline satisfies physical zone separation. */
260 satisfiesPzs: boolean | undefined;
261 /** Server-assigned UUID4, stable until delete. */
262 uid: string | undefined;
263 /** Server checksum of the resource. */
264 etag: string | undefined;
265 /** RFC3339 creation timestamp. */
266 createTime: string | undefined;
267 /** RFC3339 last-update timestamp. */
268 updateTime: string | undefined;
269 },
270 never,
271 Providers
272 >;
273
274 /**
275 * An Eventarc Advanced pipeline that delivers matched enrollment
276 * messages to a destination (HTTP, Workflows, Pub/Sub, or another
277 * MessageBus), optionally transforming and retrying along the way.
278 *
279 * `pipelineId` and `location` are identity — changing either replaces
280 * the pipeline. Destinations, mediations, retry policy, logging, CMEK,
281 * display name, labels, and annotations update in place.
282 *
283 * ### Creating a Pipeline
284 * **Example:** Publish to a Pub/Sub topic
285 * ```typescript
286 * const topic = yield* GCP.PubSub.Topic("Sink", {});
287 * const pipeline = yield* GCP.Eventarc.Pipeline("Forward", {
288 * location: "us-central1",
289 * destinations: [{ topic: topic.name }],
290 * labels: { env: "prod" },
291 * });
292 * ```
293 *
294 * **Example:** HTTPS destination with OIDC
295 * ```typescript
296 * const pipeline = yield* GCP.Eventarc.Pipeline("HttpSink", {
297 * location: "us-central1",
298 * destinations: [{
299 * httpEndpoint: { uri: "https://service-xyz-uc.a.run.app" },
300 * authenticationConfig: {
301 * googleOidc: { serviceAccount: sa.email },
302 * },
303 * }],
304 * });
305 * ```
306 *
307 * ### Updating a Pipeline
308 * Re-declare the same logical id with changed props; the engine keeps the
309 * physical pipeline and patches it in place.
310 *
311 * **Example:** Change labels and retry policy
312 * ```typescript
313 * const pipeline = yield* GCP.Eventarc.Pipeline("Forward", {
314 * location: "us-central1",
315 * destinations: [{ topic: topic.name }],
316 * retryPolicy: { maxAttempts: 8 },
317 * labels: { env: "prod", role: "pipeline" },
318 * });
319 * ```
320 *
321 * @resource
322 * @category Eventarc
323 */
324 export const Pipeline = Resource<Pipeline>("GCP.Eventarc.Pipeline");
325
326 const toPayloadFormat = (
327 format:
328 | PipelinePayloadFormat
329 | eventarc.GoogleCloudEventarcV1PipelineMessagePayloadFormat
330 | undefined,
331 ): PipelinePayloadFormat | undefined => {
332 if (format === undefined) return undefined;
333 const next: PipelinePayloadFormat = {};
334 if (format.protobuf) {
335 next.protobuf = {
336 schemaDefinition: format.protobuf.schemaDefinition,
337 };
338 }
339 if (format.avro) {
340 next.avro = { schemaDefinition: format.avro.schemaDefinition };
341 }
342 if (format.json !== undefined) {
343 next.json = {};
344 }
345 return next.protobuf || next.avro || next.json ? next : undefined;
346 };
347
348 const toRetryPolicy = (
349 policy: PipelineRetryPolicy | eventarc.GoogleCloudEventarcV1PipelineRetryPolicy | undefined,
350 ): PipelineRetryPolicy | undefined => {
351 if (policy === undefined) return undefined;
352 if (
353 policy.maxAttempts === undefined &&
354 policy.minRetryDelay === undefined &&
355 policy.maxRetryDelay === undefined
356 ) {
357 return undefined;
358 }
359 return {
360 maxAttempts: policy.maxAttempts,
361 minRetryDelay: policy.minRetryDelay,
362 maxRetryDelay: policy.maxRetryDelay,
363 };
364 };
365
366 const toMediations = (
367 mediations:
368 | readonly PipelineMediation[]
369 | eventarc.GoogleCloudEventarcV1PipelineMediationList
370 | undefined,
371 ): PipelineMediation[] | undefined => {
372 if (mediations === undefined || mediations.length === 0) return undefined;
373 return mediations.map((mediation) => ({
374 transformation: mediation.transformation
375 ? {
376 transformationTemplate: mediation.transformation.transformationTemplate,
377 }
378 : undefined,
379 }));
380 };
381
382 const toDestination = (
383 destination: PipelineDestination | eventarc.GoogleCloudEventarcV1PipelineDestination,
384 project: string,
385 location: string,
386 ): PipelineDestination => {
387 const auth = destination.authenticationConfig;
388 return {
389 httpEndpoint: destination.httpEndpoint
390 ? {
391 uri: destination.httpEndpoint.uri ?? "",
392 messageBindingTemplate: destination.httpEndpoint.messageBindingTemplate,
393 }
394 : undefined,
395 workflow: destination.workflow
396 ? expandResource(destination.workflow, project, location, "workflows")
397 : undefined,
398 topic: destination.topic ? expandTopic(destination.topic, project) : undefined,
399 messageBus: destination.messageBus
400 ? expandResource(destination.messageBus, project, location, "messageBuses")
401 : undefined,
402 networkConfig: destination.networkConfig?.networkAttachment
403 ? { networkAttachment: destination.networkConfig.networkAttachment }
404 : undefined,
405 authenticationConfig: auth
406 ? {
407 oauthToken: auth.oauthToken
408 ? {
409 serviceAccount: auth.oauthToken.serviceAccount,
410 scope: auth.oauthToken.scope,
411 }
412 : undefined,
413 googleOidc: auth.googleOidc
414 ? {
415 serviceAccount: auth.googleOidc.serviceAccount,
416 audience: auth.googleOidc.audience,
417 }
418 : undefined,
419 }
420 : undefined,
421 outputPayloadFormat: toPayloadFormat(destination.outputPayloadFormat),
422 };
423 };
424
425 const toDestinations = (
426 destinations:
427 | readonly PipelineDestination[]
428 | eventarc.GoogleCloudEventarcV1PipelineDestinationList
429 | undefined,
430 project: string,
431 location: string,
432 ): PipelineDestination[] =>
433 (destinations ?? []).map((destination) => toDestination(destination, project, location));
434
435 const toAttrs = (pipeline: eventarc.Pipeline, project: string, region: string) => {
436 const name = pipeline.name ?? "";
437 const parsed = parseName(name, COLLECTION, region);
438 const location = parsed.location;
439 const resolvedProject = parsed.project || project;
440 return {
441 name,
442 pipelineId: parsed.id,
443 project: resolvedProject,
444 location,
445 destinations: toDestinations(pipeline.destinations, resolvedProject, location),
446 inputPayloadFormat: toPayloadFormat(pipeline.inputPayloadFormat),
447 mediations: toMediations(pipeline.mediations),
448 retryPolicy: toRetryPolicy(pipeline.retryPolicy),
449 displayName: pipeline.displayName,
450 cryptoKeyName: pipeline.cryptoKeyName,
451 loggingConfig: toLoggingConfig(pipeline.loggingConfig),
452 labels: userLabels(pipeline.labels),
453 annotations: userAnnotations(pipeline.annotations),
454 satisfiesPzs: pipeline.satisfiesPzs,
455 uid: pipeline.uid,
456 etag: pipeline.etag,
457 createTime: pipeline.createTime,
458 updateTime: pipeline.updateTime,
459 };
460 };
461
462 const getByName = (name: string) =>
463 eventarc
464 .getProjectsLocationsPipelines({ name })
465 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
466
467 export const PipelineProvider = () =>
468 Provider.succeed(Pipeline, {
469 stables: ["name", "pipelineId", "project", "location", "uid", "createTime"],
470
471 diff: Effect.fn(function* ({ news, olds, output }) {
472 if (!isResolved(news)) return undefined;
473 const env = yield* GcpEnvironment.current;
474 const previousId = olds?.pipelineId ?? output?.pipelineId;
475 const nextId = news.pipelineId ? rfc1035(news.pipelineId, "pipeline") : previousId;
476 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
477 const nextLocation = normalizeLocation(
478 news.location ?? olds?.location ?? output?.location,
479 env.region,
480 );
481 if (
482 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
483 previousLocation !== nextLocation
484 ) {
485 return { action: "replace" as const };
486 }
487 return undefined;
488 }),
489
490 read: Effect.fn(function* ({ id, olds, output }) {
491 const env = yield* GcpEnvironment.current;
492 const pipelineId = yield* toPhysicalId(id, olds?.pipelineId, output?.pipelineId, "pipeline");
493 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
494 const name = output?.name ?? resourceName(env.project, location, COLLECTION, pipelineId);
495 const existing = yield* getByName(name);
496 if (existing === undefined) return undefined;
497 const attrs = toAttrs(existing, env.project, env.region);
498 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
499 }),
500
501 list: () =>
502 Effect.gen(function* () {
503 const env = yield* GcpEnvironment.current;
504 const items = yield* collectPages(
505 eventarc.listProjectsLocationsPipelines.pages({
506 parent: parentOf(env.project, "-"),
507 pageSize: 1000,
508 }),
509 (page) => page.pipelines,
510 );
511 return items
512 .filter((item) => hasAlchemyLabelKeys(item.labels))
513 .map((item) => toAttrs(item, env.project, env.region));
514 }),
515
516 reconcile: Effect.fn(function* ({ id, news, output }) {
517 const env = yield* GcpEnvironment.current;
518 const pipelineId = yield* toPhysicalId(id, news.pipelineId, output?.pipelineId, "pipeline");
519 const location = normalizeLocation(news.location ?? output?.location, env.region);
520 const name = resourceName(env.project, location, COLLECTION, pipelineId);
521 const desiredLabels = {
522 ...toLabels(news.labels),
523 ...(yield* createInternalLabels(id)),
524 };
525 const destinations = toDestinations(news.destinations, env.project, location);
526 const desiredLogging = toLoggingConfig(news.loggingConfig);
527 const desiredCrypto =
528 news.cryptoKeyName && news.cryptoKeyName.length > 0 ? news.cryptoKeyName : undefined;
529 const desiredAnnotations = news.annotations ? tagRecord(news.annotations) : undefined;
530 const desiredFormat = toPayloadFormat(news.inputPayloadFormat);
531 const desiredMediations = toMediations(news.mediations);
532 const desiredRetry = toRetryPolicy(news.retryPolicy);
533
534 let current = yield* getByName(name);
535
536 if (current === undefined) {
537 const created = yield* eventarc
538 .createProjectsLocationsPipelines({
539 parent: parentOf(env.project, location),
540 pipelineId,
541 body: compact({
542 name,
543 destinations,
544 inputPayloadFormat: desiredFormat,
545 mediations: desiredMediations,
546 retryPolicy: desiredRetry,
547 displayName: news.displayName,
548 cryptoKeyName: desiredCrypto,
549 loggingConfig: desiredLogging,
550 labels: desiredLabels,
551 annotations: desiredAnnotations,
552 }),
553 })
554 .pipe(
555 Effect.retry({
556 while: (error) => error._tag === "Conflict",
557 times: 5,
558 schedule: Schedule.spaced("2 seconds"),
559 }),
560 Effect.catchTag("Conflict", () => Effect.undefined),
561 );
562 if (created !== undefined) {
563 yield* waitForOperation(created);
564 }
565 current = yield* waitUntilPresent(getByName(name), name);
566 }
567
568 const observed = toAttrs(current, env.project, env.region);
569 const observedLabels = tagRecord(current.labels);
570 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
571 const updateMask = changedFields([
572 ["labels", upsert.length > 0 || removed.length > 0],
573 ["destinations", !sameJson(observed.destinations, destinations)],
574 ["inputPayloadFormat", !sameJson(observed.inputPayloadFormat, desiredFormat)],
575 ["mediations", !sameJson(observed.mediations, desiredMediations)],
576 ["retryPolicy", !sameJson(observed.retryPolicy, desiredRetry)],
577 ["displayName", textKey(current.displayName) !== textKey(news.displayName)],
578 ["cryptoKeyName", cryptoKeyKey(current.cryptoKeyName) !== cryptoKeyKey(desiredCrypto)],
579 ["loggingConfig", loggingKey(current.loggingConfig) !== loggingKey(desiredLogging)],
580 ["annotations", !sameJson(tagRecord(current.annotations), desiredAnnotations ?? {})],
581 ]);
582
583 if (updateMask.length > 0) {
584 const patched = yield* eventarc.patchProjectsLocationsPipelines({
585 name: current.name ?? name,
586 updateMask: updateMask.join(","),
587 body: {
588 name: current.name ?? name,
589 destinations,
590 inputPayloadFormat: desiredFormat,
591 mediations: desiredMediations,
592 retryPolicy: desiredRetry,
593 displayName: news.displayName,
594 cryptoKeyName: desiredCrypto,
595 loggingConfig: desiredLogging,
596 labels: desiredLabels,
597 annotations: desiredAnnotations ?? {},
598 },
599 });
600 yield* waitForOperation(patched);
601 current = yield* waitUntilPresent(getByName(current.name ?? name), current.name ?? name);
602 }
603
604 return toAttrs(current, env.project, env.region);
605 }),
606
607 delete: Effect.fn(function* ({ output }) {
608 yield* retryOnTransient(
609 Effect.gen(function* () {
610 const existing = yield* getByName(output.name);
611 if (existing === undefined) return;
612 const operation = yield* eventarc
613 .deleteProjectsLocationsPipelines({
614 name: output.name,
615 allowMissing: true,
616 })
617 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
618 if (operation !== undefined) {
619 yield* waitForOperation(operation, { notFoundOk: true });
620 }
621 }),
622 ).pipe(
623 Effect.catchTag("GCP.OperationFailed", (error) =>
624 getByName(output.name).pipe(
625 Effect.flatMap((current) => (current === undefined ? Effect.void : Effect.fail(error))),
626 ),
627 ),
628 );
629 yield* waitUntilGone(getByName(output.name), output.name);
630 }),
631 });