@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 380 lines 12.3 KB
Raw
1 import * as metastore from "@distilled.cloud/gcp/metastore_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 DEFAULT_HIVE_VERSION,
14 fieldMask,
15 fingerprint,
16 listAtLocation,
17 listLabeledPages,
18 locationParent,
19 normalizeLocation,
20 parseName,
21 replaceOnIdentity,
22 ResourceNotResolved,
23 toPhysicalId,
24 userLabels,
25 waitForOperation,
26 waitUntilExists,
27 waitUntilGone,
28 waitUntilReady,
29 } from "./internal.ts";
30
31 export type BackendMetastore = {
32 /**
33 * Relative resource name of the backend. BigQuery uses
34 * `projects/{project}`. Dataproc Metastore uses
35 * `projects/{project}/locations/{location}/services/{service}`.
36 */
37 name: string;
38 /**
39 * Backend type (`BIGQUERY` or `DATAPROC_METASTORE`).
40 */
41 metastoreType?: metastore.BackendMetastoreMetastoreTypeEnum | (string & {});
42 };
43
44 export type FederationProps = {
45 /**
46 * Federation id (the `{federation}` segment of
47 * `projects/{project}/locations/{location}/federations/{federation}`).
48 * If omitted, a unique RFC1035 name is generated. Must be 2-63
49 * characters, start with a letter, and end with a letter or number.
50 * Immutable — changing it replaces the federation.
51 */
52 federationId?: string;
53 /**
54 * Region (`us-central1`, …). Immutable — changing it replaces the
55 * federation. `US-CENTRAL1` is accepted and normalized to `us-central1`.
56 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
57 */
58 location?: string;
59 /**
60 * Apache Hive metastore version of the federation. All backends must
61 * be compatible. Immutable — changing it replaces the federation.
62 * @default "3.1.2"
63 */
64 version?: string;
65 /**
66 * Ranked backends used to resolve database names at query time. Keys
67 * are ranks (`0`, `1`, …); lower ranks are evaluated first. When
68 * omitted, a BigQuery backend for the current project is used.
69 */
70 backendMetastores?: Record<string, BackendMetastore>;
71 /**
72 * User labels. Alchemy ownership labels are merged in automatically.
73 */
74 labels?: Record<string, string>;
75 };
76
77 export type Federation = Resource<
78 "GCP.Metastore.Federation",
79 FederationProps,
80 {
81 /** Full resource name. */
82 name: string;
83 /** Federation id (last path segment). */
84 federationId: string;
85 /** Project id. */
86 project: string;
87 /** Location id. */
88 location: string;
89 /** Hive metastore version. */
90 version: string | undefined;
91 /** Ranked backend metastores. */
92 backendMetastores: Record<string, BackendMetastore>;
93 /** User labels (Alchemy ownership labels stripped). */
94 labels: Record<string, string>;
95 /** Federation endpoint URI. */
96 endpointUri: string | undefined;
97 /** Server-reported state. */
98 state: string | undefined;
99 /** Extra status text, if any. */
100 stateMessage: string | undefined;
101 /** Server-generated resource uid. */
102 uid: string | undefined;
103 /** RFC3339 creation timestamp. */
104 createTime: string | undefined;
105 /** RFC3339 last-update timestamp. */
106 updateTime: string | undefined;
107 },
108 never,
109 Providers
110 >;
111
112 /**
113 * A Dataproc Metastore federation that serves metadata from ranked
114 * BigQuery and Dataproc Metastore backends.
115 *
116 * Changing `federationId`, `location`, or `version` replaces the
117 * federation. Labels and `backendMetastores` update in place.
118 *
119 * ### Creating a Federation
120 * **Example:** BigQuery backend
121 * ```typescript
122 * const federation = yield* GCP.Metastore.Federation("Lakehouse", {
123 * version: "3.1.2",
124 * backendMetastores: {
125 * "1": {
126 * name: "projects/my-project",
127 * metastoreType: "BIGQUERY",
128 * },
129 * },
130 * labels: { env: "prod" },
131 * });
132 * ```
133 *
134 * **Example:** Generated name
135 * ```typescript
136 * const federation = yield* GCP.Metastore.Federation("Lakehouse", {
137 * labels: { env: "test" },
138 * });
139 * ```
140 *
141 * ### Updating a Federation
142 * **Example:** Labels and backends
143 * ```typescript
144 * // Same logical id, changed props: the engine updates it in place.
145 * const federation = yield* GCP.Metastore.Federation("Lakehouse", {
146 * version: "3.1.2",
147 * backendMetastores: {
148 * "1": {
149 * name: "projects/my-project",
150 * metastoreType: "BIGQUERY",
151 * },
152 * },
153 * labels: { env: "prod", team: "data" },
154 * });
155 * ```
156 *
157 * @resource
158 * @category Metastore
159 */
160 export const Federation = Resource<Federation>("GCP.Metastore.Federation");
161
162 const resourceName = (project: string, location: string, federationId: string) =>
163 `${locationParent(project, location)}/federations/${federationId}`;
164
165 const toBackends = (
166 value: Record<string, metastore.BackendMetastore | undefined> | undefined,
167 ): Record<string, BackendMetastore> => {
168 const next: Record<string, BackendMetastore> = {};
169 for (const [rank, item] of Object.entries(value ?? {})) {
170 if (item === undefined) continue;
171 next[rank] = {
172 name: item.name ?? "",
173 metastoreType: item.metastoreType,
174 };
175 }
176 return next;
177 };
178
179 const desiredBackends = (
180 project: string,
181 value: Record<string, BackendMetastore> | undefined,
182 ): Record<string, BackendMetastore> => {
183 if (value !== undefined) return toBackends(value);
184 return {
185 "1": {
186 name: `projects/${project}`,
187 metastoreType: "BIGQUERY",
188 },
189 };
190 };
191
192 const toAttrs = (item: metastore.Federation, project: string, region: string) => {
193 const name = item.name ?? "";
194 const parsed = parseName(name, "federations", region);
195 return {
196 name,
197 federationId: parsed.id,
198 project: parsed.project || project,
199 location: parsed.location,
200 version: item.version,
201 backendMetastores: toBackends(item.backendMetastores),
202 labels: userLabels(item.labels),
203 endpointUri: item.endpointUri,
204 state: item.state,
205 stateMessage: item.stateMessage,
206 uid: item.uid,
207 createTime: item.createTime,
208 updateTime: item.updateTime,
209 };
210 };
211
212 const getByName = (name: string) =>
213 name.length === 0
214 ? Effect.undefined
215 : metastore
216 .getProjectsLocationsFederations({ name })
217 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
218
219 const listOwned = (project: string) =>
220 listAtLocation(project, (parent) =>
221 listLabeledPages(
222 metastore.listProjectsLocationsFederations.pages({
223 parent,
224 pageSize: 1000,
225 }),
226 (page) => page.federations,
227 (item) => item.labels,
228 ),
229 );
230
231 export const FederationProvider = () =>
232 Provider.succeed(Federation, {
233 stables: ["name", "federationId", "project", "location", "version", "uid", "createTime"],
234
235 diff: Effect.fn(function* ({ news, olds, output }) {
236 if (!isResolved(news)) return undefined;
237 const env = yield* GcpEnvironment.current;
238 const previousVersion = olds?.version ?? output?.version;
239 const nextVersion = news.version ?? olds?.version ?? output?.version ?? DEFAULT_HIVE_VERSION;
240 return replaceOnIdentity({
241 previousId: olds?.federationId ?? output?.federationId,
242 nextId: news.federationId ?? olds?.federationId ?? output?.federationId,
243 previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
244 nextLocation: normalizeLocation(
245 news.location ?? olds?.location ?? output?.location,
246 env.region,
247 ),
248 extra: previousVersion !== undefined && nextVersion !== previousVersion,
249 });
250 }),
251
252 read: Effect.fn(function* ({ id, olds, output }) {
253 const env = yield* GcpEnvironment.current;
254 const federationId = yield* toPhysicalId(
255 id,
256 olds?.federationId,
257 output?.federationId,
258 "federation",
259 );
260 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
261 const name = output?.name ?? resourceName(env.project, location, federationId);
262 const existing = yield* getByName(name);
263 if (existing === undefined) return undefined;
264 const attrs = toAttrs(existing, env.project, env.region);
265 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
266 }),
267
268 list: () =>
269 Effect.gen(function* () {
270 const env = yield* GcpEnvironment.current;
271 const items = yield* listOwned(env.project);
272 return items.map((item) => toAttrs(item, env.project, env.region));
273 }),
274
275 reconcile: Effect.fn(function* ({ id, news, output }) {
276 const env = yield* GcpEnvironment.current;
277 const federationId = yield* toPhysicalId(
278 id,
279 news.federationId,
280 output?.federationId,
281 "federation",
282 );
283 const location = normalizeLocation(news.location ?? output?.location, env.region);
284 const name = resourceName(env.project, location, federationId);
285 const version = news.version ?? output?.version ?? DEFAULT_HIVE_VERSION;
286 const backends = desiredBackends(env.project, news.backendMetastores);
287 const desiredLabels = {
288 ...toLabels(news.labels),
289 ...(yield* createInternalLabels(id)),
290 };
291
292 let current = yield* getByName(output?.name ?? name);
293
294 if (current === undefined) {
295 const created = yield* metastore
296 .createProjectsLocationsFederations({
297 parent: locationParent(env.project, location),
298 federationId,
299 body: {
300 version,
301 backendMetastores: backends,
302 labels: desiredLabels,
303 },
304 })
305 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
306 if (created !== undefined) {
307 // ALREADY_EXISTS (6): a concurrent create won the race.
308 yield* waitForOperation(created).pipe(
309 Effect.catchTag("GCP.OperationFailed", (error) =>
310 error.code === 6 ? Effect.void : Effect.fail(error),
311 ),
312 );
313 }
314 current = yield* waitUntilExists(getByName(name), name);
315 current = yield* waitUntilReady(
316 getByName(name),
317 name,
318 (item) => item.state,
319 (item) => item.stateMessage,
320 );
321 }
322
323 if (current === undefined) {
324 return yield* new ResourceNotResolved({ resourceName: name });
325 }
326
327 const observedLabels = tagRecord(current.labels);
328 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
329 const labelsChanged = upsert.length > 0 || removed.length > 0;
330 const backendsChanged =
331 fingerprint(toBackends(current.backendMetastores)) !== fingerprint(backends);
332 const mask = fieldMask([labelsChanged && "labels", backendsChanged && "backendMetastores"]);
333
334 if (mask.length > 0) {
335 const operation = yield* metastore.patchProjectsLocationsFederations({
336 name: current.name ?? name,
337 updateMask: mask,
338 body: {
339 labels: desiredLabels,
340 backendMetastores: backends,
341 },
342 });
343 yield* waitForOperation(operation);
344 current = yield* waitUntilReady(
345 getByName(current.name ?? name),
346 current.name ?? name,
347 (item) => item.state,
348 (item) => item.stateMessage,
349 );
350 }
351
352 if (current === undefined) {
353 return yield* new ResourceNotResolved({ resourceName: name });
354 }
355
356 return toAttrs(current, env.project, env.region);
357 }),
358
359 delete: Effect.fn(function* ({ output }) {
360 const operation = yield* metastore
361 .deleteProjectsLocationsFederations({ name: output.name })
362 .pipe(
363 Effect.retry({
364 while: (error) => error._tag === "Conflict",
365 times: 8,
366 schedule: Schedule.spaced("2 seconds"),
367 }),
368 Effect.catchTag("NotFound", () => Effect.undefined),
369 );
370 if (operation !== undefined) {
371 // NOT_FOUND (5): already gone.
372 yield* waitForOperation(operation).pipe(
373 Effect.catchTag("GCP.OperationFailed", (error) =>
374 error.code === 5 ? Effect.void : Effect.fail(error),
375 ),
376 );
377 }
378 yield* waitUntilGone(getByName(output.name), output.name);
379 }),
380 });