@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 463 lines 15 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import { waitForOperation } from "./operations.ts";
14 import {
15 DataplexNotResolved,
16 fingerprint,
17 hasAlchemyLabelMap,
18 isPendingState,
19 listAssets,
20 listChildResources,
21 listLakes,
22 listZones,
23 normalizeLocation,
24 parseResourceName,
25 replaceIfChanged,
26 toPhysicalRfc1035,
27 userLabels,
28 } from "./shared.ts";
29
30 export type AssetResourceSpec = {
31 /**
32 * Relative name of the attached cloud resource
33 * (`projects/{project}/buckets/{bucket}` or
34 * `projects/{project}/datasets/{dataset}`). Immutable — changing it
35 * replaces the asset.
36 */
37 name?: string;
38 /**
39 * Resource type. Immutable — changing it replaces the asset.
40 */
41 type: dataplex.GoogleCloudDataplexV1AssetResourceSpecTypeEnum | (string & {});
42 /**
43 * Read access mode for Cloud Storage bucket assets (`DIRECT` or
44 * `MANAGED`).
45 */
46 readAccessMode?:
47 | dataplex.GoogleCloudDataplexV1AssetResourceSpecReadAccessModeEnum
48 | (string & {});
49 };
50
51 export type AssetDiscoverySpec = {
52 /** Whether discovery is enabled. */
53 enabled?: boolean;
54 /** Include glob / table-name patterns. */
55 includePatterns?: string[];
56 /** Exclude glob / table-name patterns. */
57 excludePatterns?: string[];
58 /** CSV discovery options. */
59 csvOptions?: {
60 headerRows?: number;
61 delimiter?: string;
62 encoding?: string;
63 disableTypeInference?: boolean;
64 };
65 /** JSON discovery options. */
66 jsonOptions?: {
67 encoding?: string;
68 disableTypeInference?: boolean;
69 };
70 /** Cron schedule (at least 60 minutes apart). */
71 schedule?: string;
72 };
73
74 export type LakesAssetProps = {
75 /**
76 * Parent zone. Full name
77 * `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}`.
78 * Immutable — changing it replaces the asset.
79 */
80 zone: string;
81 /**
82 * Region used when constructing names. Taken from `zone` when it is a
83 * full resource name.
84 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
85 */
86 location?: string;
87 /**
88 * Asset id. Must contain only lowercase letters, numbers, and hyphens;
89 * start with a letter; end with a letter or number; and be 1-63
90 * characters. Immutable — changing it replaces the asset.
91 */
92 assetId?: string;
93 /**
94 * User-friendly display name.
95 */
96 displayName?: string;
97 /**
98 * Human-readable description.
99 */
100 description?: string;
101 /**
102 * User labels. Alchemy ownership labels are merged in automatically.
103 */
104 labels?: Record<string, string>;
105 /**
106 * Cloud resource this asset manages.
107 */
108 resourceSpec: AssetResourceSpec;
109 /**
110 * Discovery settings. When omitted, the parent zone spec is used.
111 */
112 discoverySpec?: AssetDiscoverySpec;
113 };
114
115 export type LakesAsset = Resource<
116 "GCP.Dataplex.LakesAsset",
117 LakesAssetProps,
118 {
119 /** Full resource name `.../zones/{zone}/assets/{asset}`. */
120 name: string;
121 /** Asset id. */
122 assetId: string;
123 /** Parent zone resource name. */
124 zone: string;
125 /** Project id. */
126 project: string;
127 /** Location id. */
128 location: string;
129 /** User-friendly display name. */
130 displayName: string | undefined;
131 /** Description. */
132 description: string | undefined;
133 /** User labels (Alchemy ownership labels stripped). */
134 labels: Record<string, string>;
135 /** Attached resource type. */
136 resourceType: string | undefined;
137 /** Attached resource name. */
138 resourceName: string | undefined;
139 /** Resource status (`READY`, `ERROR`, …). */
140 resourceState: string | undefined;
141 /** Lifecycle state. */
142 state: string | undefined;
143 /** Server-assigned uid. */
144 uid: string | undefined;
145 /** RFC3339 creation timestamp. */
146 createTime: string | undefined;
147 /** RFC3339 last-update timestamp. */
148 updateTime: string | undefined;
149 },
150 never,
151 Providers
152 >;
153
154 /**
155 * A Dataplex asset — a Cloud Storage bucket or BigQuery dataset managed
156 * inside a lake zone.
157 *
158 * Changing `zone`, `assetId`, `location`, or the attached resource
159 * (`resourceSpec.name` / `type`) replaces the asset. Display name,
160 * description, labels, and discovery spec update in place.
161 *
162 * ### Creating an Asset
163 * **Example:** Cloud Storage bucket asset
164 * ```typescript
165 * const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
166 * zone: zone.name,
167 * resourceSpec: {
168 * type: "STORAGE_BUCKET",
169 * name: `projects/${projectNumber}/buckets/${bucket.bucketName}`,
170 * },
171 * labels: { env: "dev" },
172 * });
173 * ```
174 *
175 * **Example:** Named asset with discovery disabled
176 * ```typescript
177 * const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
178 * zone: zone.name,
179 * assetId: "landing-bucket",
180 * resourceSpec: {
181 * type: "STORAGE_BUCKET",
182 * name: `projects/${projectNumber}/buckets/${bucket.bucketName}`,
183 * },
184 * discoverySpec: { enabled: false },
185 * });
186 * ```
187 *
188 * @resource
189 * @category Dataplex
190 */
191 export const LakesAsset = Resource<LakesAsset>("GCP.Dataplex.LakesAsset");
192
193 export class LakesAssetNotResolved extends Data.TaggedError("GCP.Dataplex.LakesAssetNotResolved")<{
194 resourceName: string;
195 }> {}
196
197 export class LakesAssetStillExists extends Data.TaggedError("GCP.Dataplex.LakesAssetStillExists")<{
198 resourceName: string;
199 }> {}
200
201 const zoneOf = (zone: string) => zone;
202
203 const resourceNameOf = (zone: string, assetId: string) => `${zone}/assets/${assetId}`;
204
205 const toAttrs = (asset: dataplex.GoogleCloudDataplexV1Asset, project: string) => {
206 const name = asset.name ?? "";
207 const parsed = parseResourceName(name, "assets");
208 return {
209 name,
210 assetId: parsed.id,
211 zone: parsed.parent,
212 project: parsed.project || project,
213 location: parsed.location,
214 displayName: asset.displayName,
215 description: asset.description,
216 labels: userLabels(asset.labels),
217 resourceType: asset.resourceSpec?.type,
218 resourceName: asset.resourceSpec?.name,
219 resourceState: asset.resourceStatus?.state,
220 state: asset.state,
221 uid: asset.uid,
222 createTime: asset.createTime,
223 updateTime: asset.updateTime,
224 };
225 };
226
227 const getByName = (name: string) =>
228 name.length === 0 || name.includes("//")
229 ? Effect.undefined
230 : dataplex
231 .getProjectsLocationsLakesZonesAssets({ name })
232 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
233
234 const waitUntilReady = (name: string) =>
235 getByName(name).pipe(
236 Effect.filterOrFail(
237 (asset): asset is dataplex.GoogleCloudDataplexV1Asset => asset !== undefined,
238 () => new LakesAssetNotResolved({ resourceName: name }),
239 ),
240 Effect.filterOrFail(
241 (asset) => !isPendingState(asset.state),
242 () => new DataplexNotResolved({ resourceName: name }),
243 ),
244 Effect.retry({
245 while: (error) =>
246 error._tag === "GCP.Dataplex.LakesAssetNotResolved" ||
247 error._tag === "GCP.Dataplex.NotResolved" ||
248 error._tag === "TooManyRequests",
249 // Asset discovery settles in 1-3 minutes, longer while the lake is busy.
250 times: 36,
251 schedule: Schedule.spaced("10 seconds"),
252 }),
253 );
254
255 const waitUntilGone = (name: string) =>
256 getByName(name).pipe(
257 Effect.flatMap((asset) =>
258 asset === undefined
259 ? Effect.void
260 : Effect.fail(new LakesAssetStillExists({ resourceName: name })),
261 ),
262 Effect.retry({
263 while: (error) => error._tag === "GCP.Dataplex.LakesAssetStillExists",
264 times: 10,
265 schedule: Schedule.spaced("3 seconds"),
266 }),
267 );
268
269 const listOwnedAssets = (project: string, region: string) =>
270 Effect.gen(function* () {
271 const lakes = yield* listLakes(project, region);
272 const zones = yield* listChildResources(lakes, listZones);
273 const assets = yield* Effect.forEach(
274 zones.filter((zone) => (zone.name ?? "").length > 0),
275 (zone) => listAssets(zone.name!),
276 { concurrency: 4 },
277 );
278 return assets.flat();
279 });
280
281 export const LakesAssetProvider = () =>
282 Provider.succeed(LakesAsset, {
283 stables: [
284 "name",
285 "assetId",
286 "zone",
287 "project",
288 "location",
289 "resourceType",
290 "uid",
291 "createTime",
292 ],
293
294 diff: Effect.fn(function* ({ news, olds, output }) {
295 if (!isResolved(news)) return undefined;
296 const env = yield* GcpEnvironment.current;
297 const previousId = olds?.assetId ?? output?.assetId;
298 const nextId = news.assetId ?? previousId;
299 const previousZone = olds?.zone ?? output?.zone;
300 const nextZone = news.zone ?? previousZone;
301 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
302 const nextLocation = normalizeLocation(
303 news.location ?? olds?.location ?? output?.location,
304 env.region,
305 );
306 const previousType = (olds?.resourceSpec?.type ?? output?.resourceType ?? "").toUpperCase();
307 const nextType = (news.resourceSpec.type ?? previousType).toUpperCase();
308 const previousResource = olds?.resourceSpec?.name ?? output?.resourceName ?? "";
309 const nextResource = news.resourceSpec.name ?? previousResource;
310 if (
311 replaceIfChanged(previousId, nextId) ||
312 replaceIfChanged(previousZone, nextZone) ||
313 (output !== undefined && previousLocation !== nextLocation) ||
314 previousType !== nextType ||
315 previousResource !== nextResource
316 ) {
317 return {
318 action: "replace" as const,
319 deleteFirst:
320 previousZone === nextZone &&
321 previousLocation === nextLocation &&
322 previousId !== undefined &&
323 nextId === previousId,
324 };
325 }
326 return undefined;
327 }),
328
329 read: Effect.fn(function* ({ id, olds, output }) {
330 const env = yield* GcpEnvironment.current;
331 const zone = zoneOf(olds?.zone ?? output?.zone ?? "");
332 const assetId = yield* toPhysicalRfc1035(id, olds?.assetId, output?.assetId);
333 const name = output?.name ?? resourceNameOf(zone, assetId);
334 const existing = yield* getByName(name);
335 if (existing === undefined) return undefined;
336 const attrs = toAttrs(existing, env.project);
337 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
338 }),
339
340 list: () =>
341 Effect.gen(function* () {
342 const env = yield* GcpEnvironment.current;
343 const assets = yield* listOwnedAssets(env.project, env.region);
344 return assets
345 .filter((asset) => hasAlchemyLabelMap(asset.labels))
346 .map((asset) => toAttrs(asset, env.project));
347 }),
348
349 reconcile: Effect.fn(function* ({ id, news, output }) {
350 const env = yield* GcpEnvironment.current;
351 const zone = zoneOf(news.zone);
352 const assetId = yield* toPhysicalRfc1035(id, news.assetId, output?.assetId);
353 const name = output?.name ?? resourceNameOf(zone, assetId);
354 const desiredLabels = {
355 ...toLabels(news.labels),
356 ...(yield* createInternalLabels(id)),
357 };
358 const resourceSpec: dataplex.GoogleCloudDataplexV1AssetResourceSpec = {
359 name: news.resourceSpec.name,
360 type: news.resourceSpec.type,
361 readAccessMode: news.resourceSpec.readAccessMode,
362 };
363 const discoverySpec = news.discoverySpec
364 ? {
365 enabled: news.discoverySpec.enabled,
366 includePatterns: news.discoverySpec.includePatterns,
367 excludePatterns: news.discoverySpec.excludePatterns,
368 csvOptions: news.discoverySpec.csvOptions,
369 jsonOptions: news.discoverySpec.jsonOptions,
370 schedule: news.discoverySpec.schedule,
371 }
372 : undefined;
373
374 let current = yield* getByName(name);
375
376 if (current === undefined) {
377 const created = yield* dataplex
378 .createProjectsLocationsLakesZonesAssets({
379 parent: zone,
380 assetId,
381 body: {
382 displayName: news.displayName,
383 description: news.description,
384 labels: desiredLabels,
385 resourceSpec,
386 discoverySpec,
387 },
388 })
389 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
390 if (created !== undefined) {
391 yield* waitForOperation(created);
392 }
393 current = yield* waitUntilReady(name);
394 }
395
396 if (current === undefined) {
397 return yield* new LakesAssetNotResolved({ resourceName: name });
398 }
399
400 const observedLabels = tagRecord(current.labels);
401 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
402 const labelsChanged = upsert.length > 0 || removed.length > 0;
403 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
404 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
405 const discoveryChanged =
406 discoverySpec !== undefined &&
407 fingerprint(current.discoverySpec) !== fingerprint(discoverySpec);
408 const readModeChanged =
409 (news.resourceSpec.readAccessMode ?? "") !== (current.resourceSpec?.readAccessMode ?? "");
410
411 if (
412 labelsChanged ||
413 displayNameChanged ||
414 descriptionChanged ||
415 discoveryChanged ||
416 readModeChanged
417 ) {
418 const updateMask = [
419 labelsChanged ? "labels" : undefined,
420 displayNameChanged ? "display_name" : undefined,
421 descriptionChanged ? "description" : undefined,
422 discoveryChanged ? "discovery_spec" : undefined,
423 readModeChanged ? "resource_spec.read_access_mode" : undefined,
424 ].filter((field): field is string => field !== undefined);
425 const operation = yield* dataplex.patchProjectsLocationsLakesZonesAssets({
426 name: current.name ?? name,
427 updateMask: updateMask.join(","),
428 body: {
429 displayName: news.displayName,
430 description: news.description,
431 labels: desiredLabels,
432 resourceSpec,
433 discoverySpec,
434 },
435 });
436 yield* waitForOperation(operation);
437 current = yield* waitUntilReady(current.name ?? name);
438 }
439
440 return toAttrs(current, env.project);
441 }),
442
443 delete: Effect.fn(function* ({ output }) {
444 if (!output.name || output.name.includes("//")) return;
445 const operation = yield* dataplex
446 .deleteProjectsLocationsLakesZonesAssets({ name: output.name })
447 .pipe(
448 Effect.retry({
449 while: (error) => error._tag === "Conflict",
450 times: 8,
451 schedule: Schedule.spaced("2 seconds"),
452 }),
453 Effect.catchTags({
454 NotFound: () => Effect.undefined,
455 BadRequest: () => Effect.undefined,
456 }),
457 );
458 if (operation !== undefined) {
459 yield* waitForOperation(operation, { notFoundOk: true });
460 }
461 yield* waitUntilGone(output.name);
462 }),
463 });