@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 448 lines 14.5 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import type { Providers } from "../Providers.ts";
11 import {
12 fingerprint,
13 lastSegment,
14 listChildResources,
15 listEntities,
16 listLakes,
17 listZones,
18 parseResourceName,
19 replaceIfChanged,
20 toPhysicalSnake,
21 hasAlchemyLabelMap,
22 } from "./shared.ts";
23
24 export type EntitySchemaField = {
25 /** Field name (letters, numbers, underscores). */
26 name: string;
27 /** Field description. */
28 description?: string;
29 /** Field type (`STRING`, `INT64`, …). */
30 type: dataplex.GoogleCloudDataplexV1SchemaSchemaFieldTypeEnum | (string & {});
31 /** Field mode (`REQUIRED`, `NULLABLE`, `REPEATED`). */
32 mode: dataplex.GoogleCloudDataplexV1SchemaSchemaFieldModeEnum | (string & {});
33 /** Nested fields for `RECORD` types. */
34 fields?: EntitySchemaField[];
35 };
36
37 export type EntityPartitionField = {
38 /** Partition field name. Immutable. */
39 name: string;
40 /** Partition field type. Immutable. */
41 type: dataplex.GoogleCloudDataplexV1SchemaPartitionFieldTypeEnum | (string & {});
42 };
43
44 export type EntitySchema = {
45 /**
46 * When true, Dataplex will not evolve the schema.
47 * @default true
48 */
49 userManaged?: boolean;
50 /** Sequence of data fields. */
51 fields?: EntitySchemaField[];
52 /** Partition keys. Immutable once set. */
53 partitionFields?: EntityPartitionField[];
54 /** Partition path style (`HIVE_COMPATIBLE`). */
55 partitionStyle?: dataplex.GoogleCloudDataplexV1SchemaPartitionStyleEnum | (string & {});
56 };
57
58 export type EntityStorageFormat = {
59 /** File format (`PARQUET`, `CSV`, `JSON`, …). */
60 format?: dataplex.GoogleCloudDataplexV1StorageFormatFormatEnum | (string & {});
61 /** Compression (`GZIP`, `BZIP2`). */
62 compressionFormat?:
63 | dataplex.GoogleCloudDataplexV1StorageFormatCompressionFormatEnum
64 | (string & {});
65 /**
66 * MIME type. Derived from `format` for PARQUET, AVRO, ORC, CSV, and JSON
67 * when omitted.
68 */
69 mimeType?: string;
70 };
71
72 // Dataplex requires `format.mimeType`; derive it from the file format when
73 // the caller only names the format.
74 const MIME_TYPES: Record<string, string> = {
75 PARQUET: "application/x-parquet",
76 AVRO: "application/x-avro",
77 ORC: "application/x-orc",
78 CSV: "text/csv",
79 JSON: "application/json",
80 };
81
82 const withMimeType = (format: EntityStorageFormat | undefined): EntityStorageFormat | undefined => {
83 if (format === undefined || format.mimeType !== undefined) return format;
84 const mimeType = MIME_TYPES[(format.format ?? "").toUpperCase()];
85 return mimeType === undefined ? format : { ...format, mimeType };
86 };
87
88 export type LakesEntityProps = {
89 /**
90 * Parent zone. Full name
91 * `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}`.
92 * Immutable — changing it replaces the entity.
93 */
94 zone: string;
95 /**
96 * Entity id used as the published table name. Letters, numbers, and
97 * underscores; at most 256 characters. Immutable — changing it
98 * replaces the entity.
99 */
100 entityId?: string;
101 /**
102 * User-friendly display name (at most 256 characters).
103 */
104 displayName?: string;
105 /**
106 * Description (at most 1024 characters).
107 */
108 description?: string;
109 /**
110 * Entity type. Immutable — changing it replaces the entity.
111 * @default "TABLE"
112 */
113 type?: dataplex.GoogleCloudDataplexV1EntityTypeEnum | (string & {});
114 /**
115 * Asset id (not a full resource name) that stores the entity data.
116 * Immutable — changing it replaces the entity.
117 */
118 asset: string;
119 /**
120 * Storage path (`gs://bucket/path` or a BigQuery table name). Immutable
121 * — changing it replaces the entity.
122 */
123 dataPath: string;
124 /**
125 * Glob of objects that constitute the entity data.
126 */
127 dataPathPattern?: string;
128 /**
129 * Storage system. Immutable — changing it replaces the entity.
130 * @default "CLOUD_STORAGE"
131 */
132 system?: dataplex.GoogleCloudDataplexV1EntitySystemEnum | (string & {});
133 /**
134 * Storage format. Required for Cloud Storage entities.
135 */
136 format?: EntityStorageFormat;
137 /**
138 * Schema. Defaults to a user-managed empty schema when omitted.
139 */
140 schema?: EntitySchema;
141 };
142
143 export type LakesEntity = Resource<
144 "GCP.Dataplex.LakesEntity",
145 LakesEntityProps,
146 {
147 /** Full resource name `.../zones/{zone}/entities/{entity}`. */
148 name: string;
149 /** Entity id. */
150 entityId: string;
151 /** Parent zone resource name. */
152 zone: string;
153 /** Project id. */
154 project: string;
155 /** Location id. */
156 location: string;
157 /** User-friendly display name. */
158 displayName: string | undefined;
159 /** Entity description. */
160 description: string | undefined;
161 /** Entity type (`TABLE` or `FILESET`). */
162 type: string | undefined;
163 /** Asset id. */
164 asset: string | undefined;
165 /** Storage path. */
166 dataPath: string | undefined;
167 /** Storage system. */
168 system: string | undefined;
169 /** Storage format. */
170 format: string | undefined;
171 /** Data Catalog entry name. */
172 catalogEntry: string | undefined;
173 /** Server etag. */
174 etag: string | undefined;
175 /** Server-assigned uid. */
176 uid: string | undefined;
177 /** RFC3339 creation timestamp. */
178 createTime: string | undefined;
179 /** RFC3339 last-update timestamp. */
180 updateTime: string | undefined;
181 },
182 never,
183 Providers
184 >;
185
186 /**
187 * A Dataplex metadata entity — a table or fileset registered in a zone.
188 *
189 * Entities have no labels field: `read` reports an entity it finds
190 * without prior state as unowned (adopt it with `--adopt`), and `list`
191 * returns the entities of Alchemy-labeled lakes. Changing `zone`, `entityId`, `type`,
192 * `asset`, `dataPath`, or `system` replaces the entity. Display name,
193 * description, format, and schema update in place.
194 *
195 * ### Creating an Entity
196 * **Example:** User-managed Parquet table
197 * ```typescript
198 * const entity = yield* GCP.Dataplex.LakesEntity("Events", {
199 * zone: zone.name,
200 * asset: asset.assetId,
201 * type: "TABLE",
202 * system: "CLOUD_STORAGE",
203 * dataPath: `gs://${bucket.bucketName}/events`,
204 * format: { format: "PARQUET" },
205 * schema: {
206 * userManaged: true,
207 * fields: [{ name: "id", type: "STRING", mode: "REQUIRED" }],
208 * },
209 * });
210 * ```
211 *
212 * @resource
213 * @category Dataplex
214 */
215 export const LakesEntity = Resource<LakesEntity>("GCP.Dataplex.LakesEntity");
216
217 export class LakesEntityNotResolved extends Data.TaggedError(
218 "GCP.Dataplex.LakesEntityNotResolved",
219 )<{
220 resourceName: string;
221 }> {}
222
223 const DEFAULT_TYPE = "TABLE";
224 const DEFAULT_SYSTEM = "CLOUD_STORAGE";
225
226 const resourceNameOf = (zone: string, entityId: string) => `${zone}/entities/${entityId}`;
227
228 const assetIdOf = (asset: string) => lastSegment(asset);
229
230 const schemaBody = (schema: EntitySchema | undefined): dataplex.GoogleCloudDataplexV1Schema => ({
231 userManaged: schema?.userManaged !== false,
232 fields: schema?.fields,
233 partitionFields: schema?.partitionFields,
234 partitionStyle: schema?.partitionStyle,
235 });
236
237 const toAttrs = (entity: dataplex.GoogleCloudDataplexV1Entity, project: string) => {
238 const name = entity.name ?? "";
239 const parsed = parseResourceName(name, "entities");
240 return {
241 name,
242 entityId: entity.id ?? parsed.id,
243 zone: parsed.parent,
244 project: parsed.project || project,
245 location: parsed.location,
246 displayName: entity.displayName,
247 description: entity.description,
248 type: entity.type,
249 asset: entity.asset,
250 dataPath: entity.dataPath,
251 system: entity.system,
252 format: entity.format?.format,
253 catalogEntry: entity.catalogEntry,
254 etag: entity.etag,
255 uid: entity.uid,
256 createTime: entity.createTime,
257 updateTime: entity.updateTime,
258 };
259 };
260
261 const getByName = (name: string) =>
262 dataplex
263 .getProjectsLocationsLakesZonesEntities({ name, view: "FULL" })
264 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
265
266 const listOwnedEntities = (project: string, region: string) =>
267 Effect.gen(function* () {
268 const lakes = (yield* listLakes(project, region)).filter((lake) =>
269 hasAlchemyLabelMap(lake.labels),
270 );
271 const zones = yield* listChildResources(lakes, listZones);
272 const named = zones.filter((zone) => (zone.name ?? "").length > 0);
273 const tables = yield* Effect.forEach(named, (zone) => listEntities(zone.name!, "TABLES"), {
274 concurrency: 4,
275 });
276 const filesets = yield* Effect.forEach(named, (zone) => listEntities(zone.name!, "FILESETS"), {
277 concurrency: 4,
278 });
279 return [...tables.flat(), ...filesets.flat()];
280 });
281
282 export const LakesEntityProvider = () =>
283 Provider.succeed(LakesEntity, {
284 stables: [
285 "name",
286 "entityId",
287 "zone",
288 "project",
289 "location",
290 "type",
291 "asset",
292 "system",
293 "uid",
294 "createTime",
295 ],
296
297 diff: Effect.fn(function* ({ news, olds, output }) {
298 if (!isResolved(news)) return undefined;
299 const previousId = olds?.entityId ?? output?.entityId;
300 const nextId = news.entityId ?? previousId;
301 const previousZone = olds?.zone ?? output?.zone;
302 const nextZone = news.zone ?? previousZone;
303 const previousType = (olds?.type ?? output?.type ?? DEFAULT_TYPE).toUpperCase();
304 const nextType = (news.type ?? previousType).toUpperCase();
305 const previousAsset = assetIdOf(olds?.asset ?? output?.asset ?? "");
306 const nextAsset = assetIdOf(news.asset);
307 const previousPath = olds?.dataPath ?? output?.dataPath ?? "";
308 const nextPath = news.dataPath;
309 const previousSystem = (olds?.system ?? output?.system ?? DEFAULT_SYSTEM).toUpperCase();
310 const nextSystem = (news.system ?? previousSystem).toUpperCase();
311 if (
312 replaceIfChanged(previousId, nextId) ||
313 replaceIfChanged(previousZone, nextZone) ||
314 previousType !== nextType ||
315 previousAsset !== nextAsset ||
316 previousPath !== nextPath ||
317 previousSystem !== nextSystem
318 ) {
319 return {
320 action: "replace" as const,
321 deleteFirst:
322 previousZone === nextZone && previousId !== undefined && nextId === previousId,
323 };
324 }
325 return undefined;
326 }),
327
328 read: Effect.fn(function* ({ id, olds, output }) {
329 const env = yield* GcpEnvironment.current;
330 const zone = olds?.zone ?? output?.zone ?? "";
331 const entityId = yield* toPhysicalSnake(id, olds?.entityId, output?.entityId);
332 const name = output?.name ?? resourceNameOf(zone, entityId);
333 const existing = yield* getByName(name);
334 if (existing === undefined) return undefined;
335 const attrs = toAttrs(existing, env.project);
336 // No labels field: without prior state the entity may not be ours.
337 return output !== undefined ? attrs : Unowned(attrs);
338 }),
339
340 list: () =>
341 Effect.gen(function* () {
342 const env = yield* GcpEnvironment.current;
343 const entities = yield* listOwnedEntities(env.project, env.region);
344 const resolved = yield* Effect.forEach(
345 entities,
346 (entity) =>
347 entity.name
348 ? getByName(entity.name).pipe(Effect.map((full) => full ?? entity))
349 : Effect.succeed(entity),
350 { concurrency: 4 },
351 );
352 return resolved.map((entity) => toAttrs(entity, env.project));
353 }),
354
355 reconcile: Effect.fn(function* ({ id, news, output }) {
356 const env = yield* GcpEnvironment.current;
357 const zone = news.zone;
358 const entityId = yield* toPhysicalSnake(id, news.entityId, output?.entityId);
359 const name = output?.name ?? resourceNameOf(zone, entityId);
360 const description = news.description;
361 const type = (news.type ?? DEFAULT_TYPE).toUpperCase();
362 const system = (news.system ?? DEFAULT_SYSTEM).toUpperCase();
363 const asset = assetIdOf(news.asset);
364 const schema = schemaBody(news.schema);
365 const format = withMimeType(news.format);
366
367 let current = yield* getByName(name);
368
369 if (current === undefined) {
370 const created = yield* dataplex
371 .createProjectsLocationsLakesZonesEntities({
372 parent: zone,
373 body: {
374 id: entityId,
375 displayName: news.displayName,
376 description,
377 type,
378 asset,
379 dataPath: news.dataPath,
380 dataPathPattern: news.dataPathPattern,
381 system,
382 format,
383 schema,
384 },
385 })
386 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
387 current = created ?? undefined;
388 }
389
390 if (current === undefined) {
391 return yield* new LakesEntityNotResolved({ resourceName: name });
392 }
393
394 const descriptionChanged = (current.description ?? "") !== (description ?? "");
395 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
396 const patternChanged = (current.dataPathPattern ?? "") !== (news.dataPathPattern ?? "");
397 const formatChanged = fingerprint(current.format) !== fingerprint(format);
398 const schemaChanged = fingerprint(current.schema) !== fingerprint(schema);
399
400 if (
401 descriptionChanged ||
402 displayNameChanged ||
403 patternChanged ||
404 formatChanged ||
405 schemaChanged
406 ) {
407 current = yield* dataplex.updateProjectsLocationsLakesZonesEntities({
408 name: current.name ?? name,
409 body: {
410 id: entityId,
411 etag: current.etag,
412 displayName: news.displayName,
413 description,
414 type,
415 asset,
416 dataPath: news.dataPath,
417 dataPathPattern: news.dataPathPattern,
418 system,
419 format,
420 schema,
421 },
422 });
423 }
424
425 return toAttrs(current, env.project);
426 }),
427
428 delete: Effect.fn(function* ({ output }) {
429 // Delete requires the current etag, which moves while partitions are
430 // removed and discovery runs, so re-read it on every attempt.
431 yield* getByName(output.name).pipe(
432 Effect.flatMap((existing) =>
433 existing === undefined
434 ? Effect.void
435 : dataplex.deleteProjectsLocationsLakesZonesEntities({
436 name: output.name,
437 etag: existing.etag,
438 }),
439 ),
440 Effect.retry({
441 while: (error) => error._tag === "Conflict",
442 times: 10,
443 schedule: Schedule.spaced("3 seconds"),
444 }),
445 Effect.catchTag("NotFound", () => Effect.void),
446 );
447 }),
448 });