@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 444 lines 14.5 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import { waitForOperation } from "./operations.ts";
14 import {
15 DataplexNotResolved,
16 expandParent,
17 fingerprint,
18 hasAlchemyLabelMap,
19 isPendingState,
20 listChildResources,
21 listLakes,
22 listZones,
23 normalizeLocation,
24 parseResourceName,
25 replaceIfChanged,
26 toPhysicalRfc1035,
27 userLabels,
28 } from "./shared.ts";
29
30 const DEFAULT_TYPE = "RAW";
31 const DEFAULT_LOCATION_TYPE = "SINGLE_REGION";
32
33 export type ZoneDiscoveryCsvOptions = {
34 /** Header rows to skip. */
35 headerRows?: number;
36 /** Field delimiter. @default "," */
37 delimiter?: string;
38 /** Character encoding. @default "UTF-8" */
39 encoding?: string;
40 /** Register all columns as strings. */
41 disableTypeInference?: boolean;
42 };
43
44 export type ZoneDiscoveryJsonOptions = {
45 /** Character encoding. @default "UTF-8" */
46 encoding?: string;
47 /** Disable type inference. */
48 disableTypeInference?: boolean;
49 };
50
51 export type ZoneDiscoverySpec = {
52 /**
53 * Whether discovery is enabled.
54 * @default false
55 */
56 enabled?: boolean;
57 /** Include glob / table-name patterns. */
58 includePatterns?: string[];
59 /** Exclude glob / table-name patterns. */
60 excludePatterns?: string[];
61 /** CSV discovery options. */
62 csvOptions?: ZoneDiscoveryCsvOptions;
63 /** JSON discovery options. */
64 jsonOptions?: ZoneDiscoveryJsonOptions;
65 /** Cron schedule (at least 60 minutes apart). */
66 schedule?: string;
67 };
68
69 export type LakesZoneProps = {
70 /**
71 * Parent lake. Full name
72 * `projects/{project}/locations/{location}/lakes/{lake}` or the lake id
73 * (combined with `location`). Immutable — changing it replaces the zone.
74 */
75 lake: string;
76 /**
77 * Region used when `lake` is a bare id.
78 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
79 */
80 location?: string;
81 /**
82 * Zone id. Must contain only lowercase letters, numbers, and hyphens;
83 * start with a letter; end with a letter or number; and be 1-63
84 * characters. Unique across lakes in the project. Immutable — changing
85 * it replaces the zone.
86 */
87 zoneId?: string;
88 /**
89 * Zone type. Immutable — changing it replaces the zone.
90 * @default "RAW"
91 */
92 type?: dataplex.GoogleCloudDataplexV1ZoneTypeEnum | (string & {});
93 /**
94 * Location type of attached assets. Immutable — changing it replaces
95 * the zone.
96 * @default "SINGLE_REGION"
97 */
98 locationType?: dataplex.GoogleCloudDataplexV1ZoneResourceSpecLocationTypeEnum | (string & {});
99 /**
100 * User-friendly display name.
101 */
102 displayName?: string;
103 /**
104 * Human-readable description.
105 */
106 description?: string;
107 /**
108 * User labels. Alchemy ownership labels are merged in automatically.
109 */
110 labels?: Record<string, string>;
111 /**
112 * Discovery settings applied to data in this zone.
113 */
114 discoverySpec?: ZoneDiscoverySpec;
115 };
116
117 export type LakesZone = Resource<
118 "GCP.Dataplex.LakesZone",
119 LakesZoneProps,
120 {
121 /** Full resource name `.../lakes/{lake}/zones/{zone}`. */
122 name: string;
123 /** Zone id. */
124 zoneId: string;
125 /** Parent lake resource name. */
126 lake: string;
127 /** Project id. */
128 project: string;
129 /** Location id. */
130 location: string;
131 /** Zone type (`RAW` or `CURATED`). */
132 type: string;
133 /** Attached-asset location type. */
134 locationType: string;
135 /** User-friendly display name. */
136 displayName: string | undefined;
137 /** Description. */
138 description: string | undefined;
139 /** User labels (Alchemy ownership labels stripped). */
140 labels: Record<string, string>;
141 /** Whether discovery is enabled. */
142 discoveryEnabled: boolean;
143 /** Lifecycle state. */
144 state: string | undefined;
145 /** Server-assigned uid. */
146 uid: string | undefined;
147 /** RFC3339 creation timestamp. */
148 createTime: string | undefined;
149 /** RFC3339 last-update timestamp. */
150 updateTime: string | undefined;
151 },
152 never,
153 Providers
154 >;
155
156 /**
157 * A Dataplex zone — a logical grouping of assets inside a lake (`RAW` or
158 * `CURATED`).
159 *
160 * Changing `lake`, `zoneId`, `location`, `type`, or `locationType`
161 * replaces the zone. Display name, description, labels, and discovery
162 * spec update in place.
163 *
164 * ### Creating a Zone
165 * **Example:** RAW zone in a lake
166 * ```typescript
167 * const zone = yield* GCP.Dataplex.LakesZone("Landing", {
168 * lake: lake.name,
169 * type: "RAW",
170 * labels: { env: "dev" },
171 * });
172 * ```
173 *
174 * **Example:** Named zone with discovery disabled
175 * ```typescript
176 * const zone = yield* GCP.Dataplex.LakesZone("Landing", {
177 * lake: lake.name,
178 * zoneId: "landing-raw",
179 * discoverySpec: { enabled: false },
180 * });
181 * ```
182 *
183 * @resource
184 * @category Dataplex
185 */
186 export const LakesZone = Resource<LakesZone>("GCP.Dataplex.LakesZone");
187
188 export class LakesZoneNotResolved extends Data.TaggedError("GCP.Dataplex.LakesZoneNotResolved")<{
189 resourceName: string;
190 }> {}
191
192 export class LakesZoneStillExists extends Data.TaggedError("GCP.Dataplex.LakesZoneStillExists")<{
193 resourceName: string;
194 }> {}
195
196 const lakeOf = (lake: string, project: string, location: string) =>
197 expandParent(lake, project, location, "lakes");
198
199 const resourceName = (lake: string, zoneId: string) => `${lake}/zones/${zoneId}`;
200
201 const desiredType = (type: string | undefined) => (type ?? DEFAULT_TYPE).toUpperCase();
202
203 const desiredLocationType = (value: string | undefined) =>
204 (value ?? DEFAULT_LOCATION_TYPE).toUpperCase();
205
206 const discoveryBody = (
207 spec: ZoneDiscoverySpec | undefined,
208 ): dataplex.GoogleCloudDataplexV1ZoneDiscoverySpec => ({
209 enabled: spec?.enabled === true,
210 includePatterns: spec?.includePatterns,
211 excludePatterns: spec?.excludePatterns,
212 csvOptions: spec?.csvOptions,
213 jsonOptions: spec?.jsonOptions,
214 schedule: spec?.schedule,
215 });
216
217 const toAttrs = (zone: dataplex.GoogleCloudDataplexV1Zone, project: string) => {
218 const name = zone.name ?? "";
219 const parsed = parseResourceName(name, "zones");
220 return {
221 name,
222 zoneId: parsed.id,
223 lake: parsed.parent,
224 project: parsed.project || project,
225 location: parsed.location,
226 type: zone.type ?? DEFAULT_TYPE,
227 locationType: zone.resourceSpec?.locationType ?? DEFAULT_LOCATION_TYPE,
228 displayName: zone.displayName,
229 description: zone.description,
230 labels: userLabels(zone.labels),
231 discoveryEnabled: zone.discoverySpec?.enabled === true,
232 state: zone.state,
233 uid: zone.uid,
234 createTime: zone.createTime,
235 updateTime: zone.updateTime,
236 };
237 };
238
239 const getByName = (name: string) =>
240 name.length === 0 || name.includes("//")
241 ? Effect.undefined
242 : dataplex
243 .getProjectsLocationsLakesZones({ name })
244 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
245
246 const waitUntilReady = (name: string) =>
247 getByName(name).pipe(
248 Effect.filterOrFail(
249 (zone): zone is dataplex.GoogleCloudDataplexV1Zone => zone !== undefined,
250 () => new LakesZoneNotResolved({ resourceName: name }),
251 ),
252 Effect.filterOrFail(
253 (zone) => !isPendingState(zone.state),
254 () => new DataplexNotResolved({ resourceName: name }),
255 ),
256 Effect.retry({
257 while: (error) =>
258 error._tag === "GCP.Dataplex.LakesZoneNotResolved" ||
259 error._tag === "GCP.Dataplex.NotResolved" ||
260 error._tag === "TooManyRequests",
261 // Lakes provision a metastore-backed control plane for 2-5 minutes.
262 times: 36,
263 schedule: Schedule.spaced("10 seconds"),
264 }),
265 );
266
267 const waitUntilGone = (name: string) =>
268 getByName(name).pipe(
269 Effect.flatMap((zone) =>
270 zone === undefined
271 ? Effect.void
272 : Effect.fail(new LakesZoneStillExists({ resourceName: name })),
273 ),
274 Effect.retry({
275 while: (error) => error._tag === "GCP.Dataplex.LakesZoneStillExists",
276 times: 10,
277 schedule: Schedule.spaced("3 seconds"),
278 }),
279 );
280
281 export const LakesZoneProvider = () =>
282 Provider.succeed(LakesZone, {
283 stables: [
284 "name",
285 "zoneId",
286 "lake",
287 "project",
288 "location",
289 "type",
290 "locationType",
291 "uid",
292 "createTime",
293 ],
294
295 diff: Effect.fn(function* ({ news, olds, output }) {
296 if (!isResolved(news)) return undefined;
297 const env = yield* GcpEnvironment.current;
298 const previousId = olds?.zoneId ?? output?.zoneId;
299 const nextId = news.zoneId ?? previousId;
300 const previousLake = olds?.lake ?? output?.lake;
301 const nextLake = news.lake ?? previousLake;
302 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
303 const nextLocation = normalizeLocation(
304 news.location ?? olds?.location ?? output?.location,
305 env.region,
306 );
307 const previousType = desiredType(olds?.type ?? output?.type);
308 const nextType = desiredType(news.type ?? previousType);
309 const previousLocationType = desiredLocationType(olds?.locationType ?? output?.locationType);
310 const nextLocationType = desiredLocationType(news.locationType ?? previousLocationType);
311 if (
312 replaceIfChanged(previousId, nextId) ||
313 replaceIfChanged(previousLake, nextLake) ||
314 (output !== undefined && previousLocation !== nextLocation) ||
315 previousType !== nextType ||
316 previousLocationType !== nextLocationType
317 ) {
318 return {
319 action: "replace" as const,
320 deleteFirst:
321 previousLake === nextLake &&
322 previousLocation === nextLocation &&
323 previousId !== undefined &&
324 nextId === previousId,
325 };
326 }
327 return undefined;
328 }),
329
330 read: Effect.fn(function* ({ id, olds, output }) {
331 const env = yield* GcpEnvironment.current;
332 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
333 const lake = lakeOf(olds?.lake ?? output?.lake ?? "", env.project, location);
334 const zoneId = yield* toPhysicalRfc1035(id, olds?.zoneId, output?.zoneId);
335 const name = output?.name ?? resourceName(lake, zoneId);
336 const existing = yield* getByName(name);
337 if (existing === undefined) return undefined;
338 const attrs = toAttrs(existing, env.project);
339 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
340 }),
341
342 list: () =>
343 Effect.gen(function* () {
344 const env = yield* GcpEnvironment.current;
345 const lakes = yield* listLakes(env.project, env.region);
346 const zones = yield* listChildResources(lakes, listZones);
347 return zones
348 .filter((zone) => hasAlchemyLabelMap(zone.labels))
349 .map((zone) => toAttrs(zone, env.project));
350 }),
351
352 reconcile: Effect.fn(function* ({ id, news, output }) {
353 const env = yield* GcpEnvironment.current;
354 const location = normalizeLocation(news.location ?? output?.location, env.region);
355 const lake = lakeOf(news.lake, env.project, location);
356 const zoneId = yield* toPhysicalRfc1035(id, news.zoneId, output?.zoneId);
357 const name = output?.name ?? resourceName(lake, zoneId);
358 const type = desiredType(news.type);
359 const locationType = desiredLocationType(news.locationType);
360 const desiredLabels = {
361 ...toLabels(news.labels),
362 ...(yield* createInternalLabels(id)),
363 };
364 const desiredDiscovery = discoveryBody(news.discoverySpec);
365
366 let current = yield* getByName(name);
367
368 if (current === undefined) {
369 const created = yield* dataplex
370 .createProjectsLocationsLakesZones({
371 parent: lake,
372 zoneId,
373 body: {
374 type,
375 resourceSpec: { locationType },
376 displayName: news.displayName,
377 description: news.description,
378 labels: desiredLabels,
379 discoverySpec: desiredDiscovery,
380 },
381 })
382 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
383 if (created !== undefined) {
384 yield* waitForOperation(created);
385 }
386 current = yield* waitUntilReady(name);
387 }
388
389 if (current === undefined) {
390 return yield* new LakesZoneNotResolved({ resourceName: name });
391 }
392
393 const observedLabels = tagRecord(current.labels);
394 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
395 const labelsChanged = upsert.length > 0 || removed.length > 0;
396 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
397 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
398 const discoveryChanged = fingerprint(current.discoverySpec) !== fingerprint(desiredDiscovery);
399
400 if (labelsChanged || displayNameChanged || descriptionChanged || discoveryChanged) {
401 const updateMask = [
402 labelsChanged ? "labels" : undefined,
403 displayNameChanged ? "display_name" : undefined,
404 descriptionChanged ? "description" : undefined,
405 discoveryChanged ? "discovery_spec" : undefined,
406 ].filter((field): field is string => field !== undefined);
407 const operation = yield* dataplex.patchProjectsLocationsLakesZones({
408 name: current.name ?? name,
409 updateMask: updateMask.join(","),
410 body: {
411 displayName: news.displayName,
412 description: news.description,
413 labels: desiredLabels,
414 discoverySpec: desiredDiscovery,
415 },
416 });
417 yield* waitForOperation(operation);
418 current = yield* waitUntilReady(current.name ?? name);
419 }
420
421 return toAttrs(current, env.project);
422 }),
423
424 delete: Effect.fn(function* ({ output }) {
425 if (!output.name || output.name.includes("//")) return;
426 const operation = yield* dataplex
427 .deleteProjectsLocationsLakesZones({ name: output.name })
428 .pipe(
429 Effect.retry({
430 while: (error) => error._tag === "Conflict",
431 times: 8,
432 schedule: Schedule.spaced("2 seconds"),
433 }),
434 Effect.catchTags({
435 NotFound: () => Effect.undefined,
436 BadRequest: () => Effect.undefined,
437 }),
438 );
439 if (operation !== undefined) {
440 yield* waitForOperation(operation, { notFoundOk: true });
441 }
442 yield* waitUntilGone(output.name);
443 }),
444 });