@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 394 lines 13.4 KB
Raw
1 import * as resourcemanager from "@distilled.cloud/gcp/cloudresourcemanager_v3";
2 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
3 import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
4 import { Unowned } from "alchemy/AdoptPolicy";
5 import { isResolved } from "alchemy/Diff";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { tagRecord } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12 import { GcpEnvironment } from "../Environment.ts";
13 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
14 import type { Providers } from "../Providers.ts";
15 import { waitForOperation } from "./operations.ts";
16 import {
17 fingerprint,
18 hasAlchemyLabelMap,
19 lastSegment,
20 listMetadataFeeds,
21 locationParent,
22 normalizeLocation,
23 parseResourceName,
24 replaceIfChanged,
25 toPhysicalRfc1035,
26 userLabels,
27 } from "./shared.ts";
28
29 export type MetadataFeedScope = {
30 /** Listen to every entry in the organization. */
31 organizationLevel?: boolean;
32 /** Projects whose entries are published (`projects/{project}`). */
33 projects?: string[];
34 /** Entry groups whose entries are published. */
35 entryGroups?: string[];
36 };
37
38 export type MetadataFeedFilters = {
39 /** Entry types to listen to. */
40 entryTypes?: string[];
41 /** Aspect types to listen to. */
42 aspectTypes?: string[];
43 /** Change types (`CREATE`, `UPDATE`, `DELETE`). */
44 changeTypes?: Array<
45 dataplex.GoogleCloudDataplexV1MetadataFeedFiltersChangeTypesItemEnum | (string & {})
46 >;
47 };
48
49 export type MetadataFeedProps = {
50 /**
51 * Feed id (the `{metadataFeed}` segment of
52 * `projects/{project}/locations/{location}/metadataFeeds/{metadataFeed}`).
53 * If omitted, a unique name is generated. Immutable — changing it
54 * replaces the feed.
55 */
56 metadataFeedId?: string;
57 /**
58 * Region. Immutable — changing it replaces the feed.
59 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
60 */
61 location?: string;
62 /**
63 * Scope of catalog changes to publish. Defaults to the current project.
64 */
65 scope?: MetadataFeedScope;
66 /**
67 * Filters on entry types, aspect types, and change types.
68 */
69 filters?: MetadataFeedFilters;
70 /**
71 * Pub/Sub topic that receives feed messages
72 * (`projects/{project}/topics/{topic}`). Grant the Dataplex service
73 * account `roles/pubsub.publisher` on the topic.
74 */
75 pubsubTopic?: string;
76 /**
77 * User labels. Alchemy ownership labels are merged in automatically.
78 */
79 labels?: Record<string, string>;
80 };
81
82 export type MetadataFeed = Resource<
83 "GCP.Dataplex.MetadataFeed",
84 MetadataFeedProps,
85 {
86 /** Full resource name `.../metadataFeeds/{metadataFeed}`. */
87 name: string;
88 /** Feed id. */
89 metadataFeedId: string;
90 /** Project id. */
91 project: string;
92 /** Location id. */
93 location: string;
94 /** Organization-level scope. */
95 organizationLevel: boolean;
96 /** Scoped projects. */
97 projects: string[];
98 /** Scoped entry groups. */
99 entryGroups: string[];
100 /** Destination Pub/Sub topic. */
101 pubsubTopic: string | undefined;
102 /** User labels (Alchemy ownership labels stripped). */
103 labels: Record<string, string>;
104 /** Server-assigned uid. */
105 uid: string | undefined;
106 /** RFC3339 creation timestamp. */
107 createTime: string | undefined;
108 /** RFC3339 last-update timestamp. */
109 updateTime: string | undefined;
110 },
111 never,
112 Providers
113 >;
114
115 /**
116 * A Dataplex metadata feed that publishes catalog changes to Pub/Sub.
117 *
118 * Changing `metadataFeedId` or `location` replaces the feed. Scope,
119 * filters, destination topic, and labels update in place.
120 *
121 * ### Creating a Feed
122 * **Example:** Project-scoped feed
123 * ```typescript
124 * const feed = yield* GCP.Dataplex.MetadataFeed("Catalog", {
125 * pubsubTopic: topic.name,
126 * scope: { projects: [`projects/${project}`] },
127 * labels: { env: "dev" },
128 * });
129 * ```
130 *
131 * **Example:** Filtered create events
132 * ```typescript
133 * const feed = yield* GCP.Dataplex.MetadataFeed("Catalog", {
134 * pubsubTopic: topic.name,
135 * filters: { changeTypes: ["CREATE"] },
136 * });
137 * ```
138 *
139 * @resource
140 * @category Dataplex
141 */
142 export const MetadataFeed = Resource<MetadataFeed>("GCP.Dataplex.MetadataFeed");
143
144 export class MetadataFeedNotResolved extends Data.TaggedError(
145 "GCP.Dataplex.MetadataFeedNotResolved",
146 )<{
147 resourceName: string;
148 }> {}
149
150 export class MetadataFeedStillExists extends Data.TaggedError(
151 "GCP.Dataplex.MetadataFeedStillExists",
152 )<{
153 resourceName: string;
154 }> {}
155
156 const resourceName = (project: string, location: string, feedId: string) =>
157 `${locationParent(project, location)}/metadataFeeds/${feedId}`;
158
159 const scopeBody = (
160 project: string,
161 scope: MetadataFeedScope | undefined,
162 ): dataplex.GoogleCloudDataplexV1MetadataFeedScope => ({
163 organizationLevel: scope?.organizationLevel === true ? true : undefined,
164 projects:
165 scope?.projects ??
166 (scope?.organizationLevel === true || (scope?.entryGroups?.length ?? 0) > 0
167 ? undefined
168 : [`projects/${project}`]),
169 entryGroups: scope?.entryGroups,
170 });
171
172 const toAttrs = (feed: dataplex.GoogleCloudDataplexV1MetadataFeed, project: string) => {
173 const name = feed.name ?? "";
174 const parsed = parseResourceName(name, "metadataFeeds");
175 return {
176 name,
177 metadataFeedId: parsed.id,
178 project: parsed.project || project,
179 location: parsed.location,
180 organizationLevel: feed.scope?.organizationLevel === true,
181 projects: feed.scope?.projects ?? [],
182 entryGroups: feed.scope?.entryGroups ?? [],
183 pubsubTopic: feed.pubsubTopic,
184 labels: userLabels(feed.labels),
185 uid: feed.uid,
186 createTime: feed.createTime,
187 updateTime: feed.updateTime,
188 };
189 };
190
191 const getByName = (name: string) =>
192 dataplex
193 .getProjectsLocationsMetadataFeeds({ name })
194 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
195
196 const topicResource = (topic: string, project: string) =>
197 topic.includes("/") ? topic : `projects/${project}/topics/${topic}`;
198
199 const grantDataplexPubsub = (project: string, topic: string | undefined) =>
200 Effect.gen(function* () {
201 if (topic === undefined || topic.length === 0) return;
202 const resource = topicResource(topic, project);
203 const projectResource = yield* resourcemanager
204 .getProjects({ name: `projects/${project}` })
205 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
206 const projectNumber = lastSegment(projectResource?.name ?? "");
207 if (projectNumber.length === 0) return;
208 const member = `serviceAccount:service-${projectNumber}@gcp-sa-dataplex.iam.gserviceaccount.com`;
209 const policy = yield* pubsub.getIamPolicyProjectsTopics({ resource });
210 const bindings = (policy.bindings ?? []).map((binding) => ({
211 ...binding,
212 members: [...(binding.members ?? [])],
213 }));
214 for (const role of ["roles/pubsub.publisher", "roles/pubsub.viewer"]) {
215 const binding = bindings.find((item) => item.role === role);
216 if (binding?.members?.includes(member)) continue;
217 if (binding) {
218 binding.members = [...(binding.members ?? []), member];
219 } else {
220 bindings.push({ role, members: [member] });
221 }
222 }
223 yield* pubsub.setIamPolicyProjectsTopics({
224 resource,
225 body: { policy: { ...policy, bindings } },
226 });
227 }).pipe(
228 Effect.retry({
229 while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests",
230 times: 5,
231 schedule: Schedule.spaced("2 seconds"),
232 }),
233 );
234
235 const waitUntilExists = (name: string) =>
236 getByName(name).pipe(
237 Effect.filterOrFail(
238 (feed): feed is dataplex.GoogleCloudDataplexV1MetadataFeed => feed !== undefined,
239 () => new MetadataFeedNotResolved({ resourceName: name }),
240 ),
241 Effect.retry({
242 while: (error) => error._tag === "GCP.Dataplex.MetadataFeedNotResolved",
243 times: 8,
244 schedule: Schedule.spaced("2 seconds"),
245 }),
246 );
247
248 const waitUntilGone = (name: string) =>
249 getByName(name).pipe(
250 Effect.flatMap((feed) =>
251 feed === undefined
252 ? Effect.void
253 : Effect.fail(new MetadataFeedStillExists({ resourceName: name })),
254 ),
255 Effect.retry({
256 while: (error) => error._tag === "GCP.Dataplex.MetadataFeedStillExists",
257 times: 10,
258 schedule: Schedule.spaced("3 seconds"),
259 }),
260 );
261
262 export const MetadataFeedProvider = () =>
263 Provider.succeed(MetadataFeed, {
264 stables: ["name", "metadataFeedId", "project", "location", "uid", "createTime"],
265
266 diff: Effect.fn(function* ({ news, olds, output }) {
267 if (!isResolved(news)) return undefined;
268 const env = yield* GcpEnvironment.current;
269 const previousId = olds?.metadataFeedId ?? output?.metadataFeedId;
270 const nextId = news.metadataFeedId ?? previousId;
271 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
272 const nextLocation = normalizeLocation(
273 news.location ?? olds?.location ?? output?.location,
274 env.region,
275 );
276 if (
277 replaceIfChanged(previousId, nextId) ||
278 (output !== undefined && previousLocation !== nextLocation)
279 ) {
280 return {
281 action: "replace" as const,
282 deleteFirst:
283 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
284 };
285 }
286 return undefined;
287 }),
288
289 read: Effect.fn(function* ({ id, olds, output }) {
290 const env = yield* GcpEnvironment.current;
291 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
292 const feedId = yield* toPhysicalRfc1035(id, olds?.metadataFeedId, output?.metadataFeedId);
293 const name = output?.name ?? resourceName(env.project, location, feedId);
294 const existing = yield* getByName(name);
295 if (existing === undefined) return undefined;
296 const attrs = toAttrs(existing, env.project);
297 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
298 }),
299
300 list: () =>
301 Effect.gen(function* () {
302 const env = yield* GcpEnvironment.current;
303 const feeds = yield* listMetadataFeeds(env.project, env.region);
304 return feeds
305 .filter((feed) => hasAlchemyLabelMap(feed.labels))
306 .map((feed) => toAttrs(feed, env.project));
307 }),
308
309 reconcile: Effect.fn(function* ({ id, news, output }) {
310 const env = yield* GcpEnvironment.current;
311 const location = normalizeLocation(news.location ?? output?.location, env.region);
312 const feedId = yield* toPhysicalRfc1035(id, news.metadataFeedId, output?.metadataFeedId);
313 const name = output?.name ?? resourceName(env.project, location, feedId);
314 const desiredLabels = {
315 ...toLabels(news.labels),
316 ...(yield* createInternalLabels(id)),
317 };
318 const scope = scopeBody(env.project, news.scope);
319 const filters = news.filters;
320 yield* grantDataplexPubsub(env.project, news.pubsubTopic);
321
322 let current = yield* getByName(name);
323
324 if (current === undefined) {
325 const created = yield* dataplex
326 .createProjectsLocationsMetadataFeeds({
327 parent: locationParent(env.project, location),
328 metadataFeedId: feedId,
329 body: {
330 scope,
331 filters,
332 labels: desiredLabels,
333 pubsubTopic: news.pubsubTopic,
334 },
335 })
336 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
337 if (created !== undefined) {
338 yield* waitForOperation(created);
339 }
340 current = yield* waitUntilExists(name);
341 }
342
343 if (current === undefined) {
344 return yield* new MetadataFeedNotResolved({ resourceName: name });
345 }
346
347 const observedLabels = tagRecord(current.labels);
348 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
349 const labelsChanged = upsert.length > 0 || removed.length > 0;
350 const scopeChanged = fingerprint(current.scope) !== fingerprint(scope);
351 const filtersChanged = fingerprint(current.filters) !== fingerprint(filters);
352 const topicChanged = (current.pubsubTopic ?? "") !== (news.pubsubTopic ?? "");
353
354 if (labelsChanged || scopeChanged || filtersChanged || topicChanged) {
355 const updateMask = [
356 labelsChanged ? "labels" : undefined,
357 scopeChanged ? "scope" : undefined,
358 filtersChanged ? "filters" : undefined,
359 topicChanged ? "pubsub_topic" : undefined,
360 ].filter((field): field is string => field !== undefined);
361 const operation = yield* dataplex.patchProjectsLocationsMetadataFeeds({
362 name: current.name ?? name,
363 updateMask: updateMask.join(","),
364 body: {
365 scope,
366 filters,
367 labels: desiredLabels,
368 pubsubTopic: news.pubsubTopic,
369 },
370 });
371 yield* waitForOperation(operation);
372 current = yield* waitUntilExists(current.name ?? name);
373 }
374
375 return toAttrs(current, env.project);
376 }),
377
378 delete: Effect.fn(function* ({ output }) {
379 const operation = yield* dataplex
380 .deleteProjectsLocationsMetadataFeeds({ name: output.name })
381 .pipe(
382 Effect.retry({
383 while: (error) => error._tag === "Conflict",
384 times: 8,
385 schedule: Schedule.spaced("2 seconds"),
386 }),
387 Effect.catchTag("NotFound", () => Effect.undefined),
388 );
389 if (operation !== undefined) {
390 yield* waitForOperation(operation, { notFoundOk: true });
391 }
392 yield* waitUntilGone(output.name);
393 }),
394 });