@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 337 lines 11.4 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import { waitForOperation } from "./operations.ts";
14 import {
15 DataplexNotResolved,
16 hasAlchemyLabelMap,
17 isPendingState,
18 listLakes,
19 locationParent,
20 normalizeLocation,
21 parseResourceName,
22 replaceIfChanged,
23 toPhysicalRfc1035,
24 userLabels,
25 } from "./shared.ts";
26
27 export type LakeMetastore = {
28 /** Dataproc Metastore service `projects/{project}/locations/{location}/services/{service}`. */
29 service?: string;
30 };
31
32 export type LakeProps = {
33 /**
34 * Lake id (the `{lake}` segment of
35 * `projects/{project}/locations/{location}/lakes/{lake}`). If omitted, a
36 * unique name is generated from the stack, stage, and logical id. Must
37 * contain only lowercase letters, numbers, and hyphens; start with a
38 * letter; end with a letter or number; and be 1-63 characters. Immutable
39 * — changing it replaces the lake.
40 */
41 lakeId?: string;
42 /**
43 * Region (`us-central1`, …). Immutable — changing it replaces the lake.
44 * `US-CENTRAL1` is accepted and normalized to `us-central1`.
45 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
46 */
47 location?: string;
48 /**
49 * User-friendly display name.
50 */
51 displayName?: string;
52 /**
53 * Human-readable description.
54 */
55 description?: string;
56 /**
57 * User labels. Alchemy ownership labels are merged in automatically.
58 */
59 labels?: Record<string, string>;
60 /**
61 * Optional Dataproc Metastore association.
62 */
63 metastore?: LakeMetastore;
64 };
65
66 export type Lake = Resource<
67 "GCP.Dataplex.Lake",
68 LakeProps,
69 {
70 /** Full resource name `projects/{project}/locations/{location}/lakes/{lake}`. */
71 name: string;
72 /** Lake id (last path segment). */
73 lakeId: string;
74 /** Project id. */
75 project: string;
76 /** Location id. */
77 location: string;
78 /** User-friendly display name. */
79 displayName: string | undefined;
80 /** Description. */
81 description: string | undefined;
82 /** User labels (Alchemy ownership labels stripped). */
83 labels: Record<string, string>;
84 /** Lifecycle state (`ACTIVE`, `CREATING`, …). */
85 state: string | undefined;
86 /** Lake service account. */
87 serviceAccount: string | undefined;
88 /** Associated Dataproc Metastore service, if any. */
89 metastoreService: string | undefined;
90 /** Server-assigned uid. */
91 uid: string | undefined;
92 /** RFC3339 creation timestamp. */
93 createTime: string | undefined;
94 /** RFC3339 last-update timestamp. */
95 updateTime: string | undefined;
96 },
97 never,
98 Providers
99 >;
100
101 /**
102 * A Dataplex lake — a regional container for zones, assets, and tasks.
103 *
104 * Changing `lakeId` or `location` replaces the lake. Display name,
105 * description, labels, and metastore association update in place.
106 *
107 * ### Creating a Lake
108 * **Example:** Generated name
109 * ```typescript
110 * const lake = yield* GCP.Dataplex.Lake("Warehouse", {
111 * labels: { env: "dev" },
112 * });
113 * ```
114 *
115 * **Example:** Named lake with a description
116 * ```typescript
117 * const lake = yield* GCP.Dataplex.Lake("Warehouse", {
118 * lakeId: "analytics-lake",
119 * location: "us-central1",
120 * displayName: "Analytics",
121 * description: "curated warehouse",
122 * labels: { env: "prod" },
123 * });
124 * ```
125 *
126 * @resource
127 * @category Dataplex
128 */
129 export const Lake = Resource<Lake>("GCP.Dataplex.Lake");
130
131 export class LakeNotResolved extends Data.TaggedError("GCP.Dataplex.LakeNotResolved")<{
132 resourceName: string;
133 }> {}
134
135 export class LakeStillExists extends Data.TaggedError("GCP.Dataplex.LakeStillExists")<{
136 resourceName: string;
137 }> {}
138
139 const resourceName = (project: string, location: string, lakeId: string) =>
140 `${locationParent(project, location)}/lakes/${lakeId}`;
141
142 const toAttrs = (lake: dataplex.GoogleCloudDataplexV1Lake, project: string) => {
143 const name = lake.name ?? "";
144 const parsed = parseResourceName(name, "lakes");
145 return {
146 name,
147 lakeId: parsed.id,
148 project: parsed.project || project,
149 location: parsed.location,
150 displayName: lake.displayName,
151 description: lake.description,
152 labels: userLabels(lake.labels),
153 state: lake.state,
154 serviceAccount: lake.serviceAccount,
155 metastoreService: lake.metastore?.service,
156 uid: lake.uid,
157 createTime: lake.createTime,
158 updateTime: lake.updateTime,
159 };
160 };
161
162 const getByName = (name: string) =>
163 dataplex
164 .getProjectsLocationsLakes({ name })
165 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
166
167 // Lakes provision a metastore-backed control plane for 2-5 minutes.
168 const waitUntilReady = (name: string) =>
169 Effect.gen(function* () {
170 const lake = yield* getByName(name);
171 if (lake === undefined) {
172 return yield* new LakeNotResolved({ resourceName: name });
173 }
174 if (isPendingState(lake.state)) {
175 return yield* new DataplexNotResolved({ resourceName: name });
176 }
177 return lake;
178 }).pipe(
179 Effect.retry({
180 while: (error) =>
181 error._tag === "GCP.Dataplex.LakeNotResolved" ||
182 error._tag === "GCP.Dataplex.NotResolved" ||
183 error._tag === "TooManyRequests",
184 times: 36,
185 schedule: Schedule.spaced("10 seconds"),
186 }),
187 );
188
189 const waitUntilGone = (name: string) =>
190 getByName(name).pipe(
191 Effect.flatMap((lake) =>
192 lake === undefined ? Effect.void : Effect.fail(new LakeStillExists({ resourceName: name })),
193 ),
194 Effect.retry({
195 while: (error) => error._tag === "GCP.Dataplex.LakeStillExists",
196 times: 10,
197 schedule: Schedule.spaced("3 seconds"),
198 }),
199 );
200
201 export const LakeProvider = () =>
202 Provider.succeed(Lake, {
203 stables: ["name", "lakeId", "project", "location", "uid", "createTime"],
204
205 diff: Effect.fn(function* ({ news, olds, output }) {
206 if (!isResolved(news)) return undefined;
207 const env = yield* GcpEnvironment.current;
208 const previousId = olds?.lakeId ?? output?.lakeId;
209 const nextId = news.lakeId ?? previousId;
210 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
211 const nextLocation = normalizeLocation(
212 news.location ?? olds?.location ?? output?.location,
213 env.region,
214 );
215 if (
216 replaceIfChanged(previousId, nextId) ||
217 (output !== undefined && previousLocation !== nextLocation)
218 ) {
219 return {
220 action: "replace" as const,
221 deleteFirst:
222 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
223 };
224 }
225 return undefined;
226 }),
227
228 read: Effect.fn(function* ({ id, olds, output }) {
229 const env = yield* GcpEnvironment.current;
230 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
231 const lakeId = yield* toPhysicalRfc1035(id, olds?.lakeId, output?.lakeId);
232 const name = output?.name ?? resourceName(env.project, location, lakeId);
233 const existing = yield* getByName(name);
234 if (existing === undefined) return undefined;
235 const attrs = toAttrs(existing, env.project);
236 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
237 }),
238
239 list: () =>
240 Effect.gen(function* () {
241 const env = yield* GcpEnvironment.current;
242 const lakes = yield* listLakes(env.project, env.region);
243 return lakes
244 .filter((lake) => hasAlchemyLabelMap(lake.labels))
245 .map((lake) => toAttrs(lake, env.project));
246 }),
247
248 reconcile: Effect.fn(function* ({ id, news, output }) {
249 const env = yield* GcpEnvironment.current;
250 const location = normalizeLocation(news.location ?? output?.location, env.region);
251 const lakeId = yield* toPhysicalRfc1035(id, news.lakeId, output?.lakeId);
252 const name = output?.name ?? resourceName(env.project, location, lakeId);
253 const desiredLabels = {
254 ...toLabels(news.labels),
255 ...(yield* createInternalLabels(id)),
256 };
257 const desiredMetastore = news.metastore?.service;
258
259 let current = yield* getByName(name);
260
261 if (current === undefined) {
262 const created = yield* dataplex
263 .createProjectsLocationsLakes({
264 parent: locationParent(env.project, location),
265 lakeId,
266 body: {
267 displayName: news.displayName,
268 description: news.description,
269 labels: desiredLabels,
270 metastore: desiredMetastore ? { service: desiredMetastore } : undefined,
271 },
272 })
273 .pipe(
274 Effect.retry({
275 while: (error) => error._tag === "TooManyRequests",
276 times: 8,
277 schedule: Schedule.spaced("2 seconds"),
278 }),
279 Effect.catchTag("Conflict", () => Effect.undefined),
280 );
281 if (created !== undefined) {
282 yield* waitForOperation(created);
283 }
284 current = yield* waitUntilReady(name);
285 }
286
287 if (current === undefined) {
288 return yield* new LakeNotResolved({ resourceName: name });
289 }
290
291 const observedLabels = tagRecord(current.labels);
292 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
293 const labelsChanged = upsert.length > 0 || removed.length > 0;
294 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
295 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
296 const metastoreChanged = (current.metastore?.service ?? "") !== (desiredMetastore ?? "");
297
298 if (labelsChanged || displayNameChanged || descriptionChanged || metastoreChanged) {
299 const updateMask = [
300 labelsChanged ? "labels" : undefined,
301 displayNameChanged ? "display_name" : undefined,
302 descriptionChanged ? "description" : undefined,
303 metastoreChanged ? "metastore.service" : undefined,
304 ].filter((field): field is string => field !== undefined);
305 const operation = yield* dataplex.patchProjectsLocationsLakes({
306 name: current.name ?? name,
307 updateMask: updateMask.join(","),
308 body: {
309 displayName: news.displayName,
310 description: news.description,
311 labels: desiredLabels,
312 metastore: desiredMetastore ? { service: desiredMetastore } : undefined,
313 },
314 });
315 yield* waitForOperation(operation);
316 current = yield* waitUntilReady(current.name ?? name);
317 }
318
319 return toAttrs(current, env.project);
320 }),
321
322 delete: Effect.fn(function* ({ output }) {
323 if (!output.name) return;
324 const operation = yield* dataplex.deleteProjectsLocationsLakes({ name: output.name }).pipe(
325 Effect.retry({
326 while: (error) => error._tag === "Conflict",
327 times: 8,
328 schedule: Schedule.spaced("2 seconds"),
329 }),
330 Effect.catchTag("NotFound", () => Effect.undefined),
331 );
332 if (operation !== undefined) {
333 yield* waitForOperation(operation, { notFoundOk: true });
334 }
335 yield* waitUntilGone(output.name);
336 }),
337 });