@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 563 lines 19.4 KB
Raw
1 import * as metastore from "@distilled.cloud/gcp/metastore_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 DEFAULT_HIVE_VERSION,
14 fieldMask,
15 fingerprint,
16 listAtLocation,
17 listLabeledPages,
18 locationParent,
19 normalizeLocation,
20 parseName,
21 replaceOnIdentity,
22 ResourceNotResolved,
23 sameText,
24 toPhysicalId,
25 userLabels,
26 waitForOperation,
27 waitUntilExists,
28 waitUntilGone,
29 waitUntilReady,
30 } from "./internal.ts";
31
32 export type HiveMetastoreConfig = {
33 /**
34 * Hive metastore schema version. Immutable.
35 * @default "3.1.2"
36 */
37 version?: string;
38 /**
39 * hive-site.xml overrides applied to the primary service.
40 */
41 configOverrides?: Record<string, string>;
42 /**
43 * Endpoint protocol (`THRIFT` or `GRPC`).
44 */
45 endpointProtocol?: metastore.HiveMetastoreConfigEndpointProtocolEnum | (string & {});
46 /**
47 * Kerberos service-principal configuration.
48 */
49 kerberosConfig?: metastore.KerberosConfig;
50 /**
51 * Auxiliary Hive versions keyed by service name.
52 */
53 auxiliaryVersions?: metastore.AuxiliaryVersionConfigMap;
54 };
55
56 export type ServiceProps = {
57 /**
58 * Service id (the `{service}` segment of
59 * `projects/{project}/locations/{location}/services/{service}`). If
60 * omitted, a unique RFC1035 name is generated. Must be 2-63 characters,
61 * start with a letter, and end with a letter or number. Immutable —
62 * changing it replaces the service.
63 */
64 serviceId?: string;
65 /**
66 * Region (`us-central1`, …). Immutable — changing it replaces the
67 * service. `US-CENTRAL1` is accepted and normalized to `us-central1`.
68 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
69 */
70 location?: string;
71 /**
72 * Hive metastore software configuration. `version` is immutable.
73 */
74 hiveMetastoreConfig?: HiveMetastoreConfig;
75 /**
76 * Service tier (`DEVELOPER` or `ENTERPRISE`).
77 * @default "DEVELOPER"
78 */
79 tier?: metastore.ServiceTierEnum | (string & {});
80 /**
81 * TCP port for the Hive endpoint.
82 * @default 9083
83 */
84 port?: number;
85 /**
86 * VPC network
87 * (`projects/{project}/global/networks/{network}`). Immutable.
88 */
89 network?: string;
90 /**
91 * Backend database type. Immutable.
92 */
93 databaseType?: metastore.ServiceDatabaseTypeEnum | (string & {});
94 /**
95 * When true, delete is rejected until protection is cleared.
96 * @default false
97 */
98 deletionProtection?: boolean;
99 /**
100 * Release channel. Immutable.
101 */
102 releaseChannel?: metastore.ServiceReleaseChannelEnum | (string & {});
103 /**
104 * Customer-managed encryption. Immutable.
105 */
106 encryptionConfig?: metastore.EncryptionConfig;
107 /**
108 * Consumer-side network configuration. Immutable.
109 */
110 networkConfig?: metastore.NetworkConfig;
111 /**
112 * Scaling configuration.
113 */
114 scalingConfig?: metastore.ScalingConfig;
115 /**
116 * Telemetry / log format. Defaults to JSON.
117 */
118 telemetryConfig?: metastore.TelemetryConfig;
119 /**
120 * Scheduled backup configuration.
121 */
122 scheduledBackup?: metastore.ScheduledBackup;
123 /**
124 * External metadata integrations (Data Catalog).
125 */
126 metadataIntegration?: metastore.MetadataIntegration;
127 /**
128 * One-hour UTC maintenance window. Not used with SPANNER.
129 */
130 maintenanceWindow?: metastore.MaintenanceWindow;
131 /**
132 * User labels. Alchemy ownership labels are merged in automatically.
133 */
134 labels?: Record<string, string>;
135 };
136
137 export type Service = Resource<
138 "GCP.Metastore.Service",
139 ServiceProps,
140 {
141 /** Full resource name. */
142 name: string;
143 /** Service id (last path segment). */
144 serviceId: string;
145 /** Project id. */
146 project: string;
147 /** Location id. */
148 location: string;
149 /** Hive metastore configuration. */
150 hiveMetastoreConfig: HiveMetastoreConfig | undefined;
151 /** Service tier. */
152 tier: string | undefined;
153 /** Hive endpoint port. */
154 port: number | undefined;
155 /** VPC network. */
156 network: string | undefined;
157 /** Backend database type. */
158 databaseType: string | undefined;
159 /** Whether delete is protected. */
160 deletionProtection: boolean;
161 /** Release channel. */
162 releaseChannel: string | undefined;
163 /** Customer-managed encryption, if any. */
164 encryptionConfig: metastore.EncryptionConfig | undefined;
165 /** Consumer network configuration. */
166 networkConfig: metastore.NetworkConfig | undefined;
167 /** Scaling configuration. */
168 scalingConfig: metastore.ScalingConfig | undefined;
169 /** Telemetry configuration. */
170 telemetryConfig: metastore.TelemetryConfig | undefined;
171 /** Scheduled backup configuration. */
172 scheduledBackup: metastore.ScheduledBackup | undefined;
173 /** External metadata integrations. */
174 metadataIntegration: metastore.MetadataIntegration | undefined;
175 /** Maintenance window. */
176 maintenanceWindow: metastore.MaintenanceWindow | undefined;
177 /** User labels (Alchemy ownership labels stripped). */
178 labels: Record<string, string>;
179 /** Hive endpoint URI. */
180 endpointUri: string | undefined;
181 /** Artifact Cloud Storage URI. */
182 artifactGcsUri: string | undefined;
183 /** Server-reported state. */
184 state: string | undefined;
185 /** Extra status text, if any. */
186 stateMessage: string | undefined;
187 /** Server-generated resource uid. */
188 uid: string | undefined;
189 /** RFC3339 creation timestamp. */
190 createTime: string | undefined;
191 /** RFC3339 last-update timestamp. */
192 updateTime: string | undefined;
193 },
194 never,
195 Providers
196 >;
197
198 /**
199 * A Dataproc Metastore service — a managed Hive metastore.
200 *
201 * Changing `serviceId`, `location`, Hive `version`, `network`,
202 * `databaseType`, `encryptionConfig`, `networkConfig`, or
203 * `releaseChannel` replaces the service. Labels, port, tier, scaling,
204 * telemetry, scheduled backups, metadata integration, maintenance
205 * window, Hive overrides, and deletion protection update in place.
206 *
207 * ### Creating a Service
208 * **Example:** Developer tier
209 * ```typescript
210 * const service = yield* GCP.Metastore.Service("Hive", {
211 * hiveMetastoreConfig: { version: "3.1.2" },
212 * tier: "DEVELOPER",
213 * labels: { env: "test" },
214 * });
215 * ```
216 *
217 * **Example:** Explicit id
218 * ```typescript
219 * const service = yield* GCP.Metastore.Service("Hive", {
220 * serviceId: "analytics-hive",
221 * hiveMetastoreConfig: { version: "3.1.2" },
222 * port: 9083,
223 * });
224 * ```
225 *
226 * ### Updating a Service
227 * **Example:** Labels and port
228 * ```typescript
229 * // Same logical id, changed props: the engine updates it in place.
230 * const service = yield* GCP.Metastore.Service("Hive", {
231 * hiveMetastoreConfig: { version: "3.1.2" },
232 * port: 9084,
233 * labels: { env: "prod" },
234 * });
235 * ```
236 *
237 * @resource
238 * @category Metastore
239 */
240 export const Service = Resource<Service>("GCP.Metastore.Service");
241
242 const resourceName = (project: string, location: string, serviceId: string) =>
243 `${locationParent(project, location)}/services/${serviceId}`;
244
245 const toHiveConfig = (
246 config: metastore.HiveMetastoreConfig | undefined,
247 ): HiveMetastoreConfig | undefined =>
248 config === undefined
249 ? undefined
250 : {
251 version: config.version,
252 configOverrides: tagRecord(config.configOverrides),
253 endpointProtocol: config.endpointProtocol,
254 kerberosConfig: config.kerberosConfig,
255 auxiliaryVersions: config.auxiliaryVersions,
256 };
257
258 const desiredHiveConfig = (
259 config: HiveMetastoreConfig | undefined,
260 ): metastore.HiveMetastoreConfig => ({
261 version: config?.version ?? DEFAULT_HIVE_VERSION,
262 configOverrides: config?.configOverrides,
263 endpointProtocol: config?.endpointProtocol,
264 kerberosConfig: config?.kerberosConfig,
265 auxiliaryVersions: config?.auxiliaryVersions,
266 });
267
268 const toAttrs = (item: metastore.Service, project: string, region: string) => {
269 const name = item.name ?? "";
270 const parsed = parseName(name, "services", region);
271 return {
272 name,
273 serviceId: parsed.id,
274 project: parsed.project || project,
275 location: parsed.location,
276 hiveMetastoreConfig: toHiveConfig(item.hiveMetastoreConfig),
277 tier: item.tier,
278 port: item.port,
279 network: item.network,
280 databaseType: item.databaseType,
281 deletionProtection: item.deletionProtection === true,
282 releaseChannel: item.releaseChannel,
283 encryptionConfig: item.encryptionConfig,
284 networkConfig: item.networkConfig,
285 scalingConfig: item.scalingConfig,
286 telemetryConfig: item.telemetryConfig,
287 scheduledBackup: item.scheduledBackup,
288 metadataIntegration: item.metadataIntegration,
289 maintenanceWindow: item.maintenanceWindow,
290 labels: userLabels(item.labels),
291 endpointUri: item.endpointUri,
292 artifactGcsUri: item.artifactGcsUri,
293 state: item.state,
294 stateMessage: item.stateMessage,
295 uid: item.uid,
296 createTime: item.createTime,
297 updateTime: item.updateTime,
298 };
299 };
300
301 const getByName = (name: string) =>
302 name.length === 0
303 ? Effect.undefined
304 : metastore
305 .getProjectsLocationsServices({ name })
306 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
307
308 const listOwned = (project: string) =>
309 listAtLocation(project, (parent) =>
310 listLabeledPages(
311 metastore.listProjectsLocationsServices.pages({
312 parent,
313 pageSize: 1000,
314 }),
315 (page) => page.services,
316 (item) => item.labels,
317 ),
318 );
319
320 export const ServiceProvider = () =>
321 Provider.succeed(Service, {
322 stables: [
323 "name",
324 "serviceId",
325 "project",
326 "location",
327 "network",
328 "databaseType",
329 "releaseChannel",
330 "uid",
331 "createTime",
332 ],
333
334 diff: Effect.fn(function* ({ news, olds, output }) {
335 if (!isResolved(news)) return undefined;
336 const env = yield* GcpEnvironment.current;
337 const previousVersion =
338 olds?.hiveMetastoreConfig?.version ?? output?.hiveMetastoreConfig?.version;
339 const nextVersion = news.hiveMetastoreConfig?.version ?? previousVersion;
340 const previousNetwork = olds?.network ?? output?.network;
341 const previousDatabase = olds?.databaseType ?? output?.databaseType;
342 const previousChannel = olds?.releaseChannel ?? output?.releaseChannel;
343 return replaceOnIdentity({
344 previousId: olds?.serviceId ?? output?.serviceId,
345 nextId: news.serviceId ?? olds?.serviceId ?? output?.serviceId,
346 previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
347 nextLocation: normalizeLocation(
348 news.location ?? olds?.location ?? output?.location,
349 env.region,
350 ),
351 extra:
352 (previousVersion !== undefined &&
353 nextVersion !== undefined &&
354 nextVersion !== previousVersion) ||
355 (previousNetwork !== undefined &&
356 news.network !== undefined &&
357 news.network !== previousNetwork) ||
358 (previousDatabase !== undefined &&
359 news.databaseType !== undefined &&
360 news.databaseType !== previousDatabase) ||
361 (previousChannel !== undefined &&
362 news.releaseChannel !== undefined &&
363 news.releaseChannel !== previousChannel) ||
364 (olds?.encryptionConfig !== undefined &&
365 news.encryptionConfig !== undefined &&
366 fingerprint(news.encryptionConfig) !== fingerprint(olds.encryptionConfig)) ||
367 (olds?.networkConfig !== undefined &&
368 news.networkConfig !== undefined &&
369 fingerprint(news.networkConfig) !== fingerprint(olds.networkConfig)),
370 });
371 }),
372
373 read: Effect.fn(function* ({ id, olds, output }) {
374 const env = yield* GcpEnvironment.current;
375 const serviceId = yield* toPhysicalId(id, olds?.serviceId, output?.serviceId, "service");
376 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
377 const name = output?.name ?? resourceName(env.project, location, serviceId);
378 const existing = yield* getByName(name);
379 if (existing === undefined) return undefined;
380 const attrs = toAttrs(existing, env.project, env.region);
381 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
382 }),
383
384 list: () =>
385 Effect.gen(function* () {
386 const env = yield* GcpEnvironment.current;
387 const items = yield* listOwned(env.project);
388 return items.map((item) => toAttrs(item, env.project, env.region));
389 }),
390
391 reconcile: Effect.fn(function* ({ id, news, output }) {
392 const env = yield* GcpEnvironment.current;
393 const serviceId = yield* toPhysicalId(id, news.serviceId, output?.serviceId, "service");
394 const location = normalizeLocation(news.location ?? output?.location, env.region);
395 const name = resourceName(env.project, location, serviceId);
396 const hive = desiredHiveConfig(news.hiveMetastoreConfig);
397 const desiredLabels = {
398 ...toLabels(news.labels),
399 ...(yield* createInternalLabels(id)),
400 };
401
402 let current = yield* getByName(output?.name ?? name);
403
404 if (current === undefined) {
405 const created = yield* metastore
406 .createProjectsLocationsServices({
407 parent: locationParent(env.project, location),
408 serviceId,
409 body: {
410 hiveMetastoreConfig: hive,
411 tier: news.tier,
412 port: news.port,
413 network: news.network,
414 databaseType: news.databaseType,
415 deletionProtection: news.deletionProtection,
416 releaseChannel: news.releaseChannel,
417 encryptionConfig: news.encryptionConfig,
418 networkConfig: news.networkConfig,
419 scalingConfig: news.scalingConfig,
420 telemetryConfig: news.telemetryConfig,
421 scheduledBackup: news.scheduledBackup,
422 metadataIntegration: news.metadataIntegration,
423 maintenanceWindow: news.maintenanceWindow,
424 labels: desiredLabels,
425 },
426 })
427 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
428 if (created !== undefined) {
429 // ALREADY_EXISTS (6): a concurrent create won the race.
430 yield* waitForOperation(created).pipe(
431 Effect.catchTag("GCP.OperationFailed", (error) =>
432 error.code === 6 ? Effect.void : Effect.fail(error),
433 ),
434 );
435 }
436 current = yield* waitUntilExists(getByName(name), name);
437 current = yield* waitUntilReady(
438 getByName(name),
439 name,
440 (item) => item.state,
441 (item) => item.stateMessage,
442 );
443 }
444
445 if (current === undefined) {
446 return yield* new ResourceNotResolved({ resourceName: name });
447 }
448
449 const observedLabels = tagRecord(current.labels);
450 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
451 const labelsChanged = upsert.length > 0 || removed.length > 0;
452 const hiveChanged =
453 fingerprint({
454 configOverrides: current.hiveMetastoreConfig?.configOverrides,
455 endpointProtocol: current.hiveMetastoreConfig?.endpointProtocol,
456 kerberosConfig: current.hiveMetastoreConfig?.kerberosConfig,
457 auxiliaryVersions: current.hiveMetastoreConfig?.auxiliaryVersions,
458 }) !==
459 fingerprint({
460 configOverrides: hive.configOverrides,
461 endpointProtocol: hive.endpointProtocol,
462 kerberosConfig: hive.kerberosConfig,
463 auxiliaryVersions: hive.auxiliaryVersions,
464 });
465 const mask = fieldMask([
466 labelsChanged && "labels",
467 news.port !== undefined && news.port !== current.port && "port",
468 news.tier !== undefined && !sameText(current.tier, news.tier) && "tier",
469 news.deletionProtection !== undefined &&
470 (current.deletionProtection === true) !== (news.deletionProtection === true) &&
471 "deletionProtection",
472 news.scalingConfig !== undefined &&
473 fingerprint(current.scalingConfig) !== fingerprint(news.scalingConfig) &&
474 "scalingConfig",
475 news.telemetryConfig !== undefined &&
476 fingerprint(current.telemetryConfig) !== fingerprint(news.telemetryConfig) &&
477 "telemetryConfig",
478 news.scheduledBackup !== undefined &&
479 fingerprint(current.scheduledBackup) !== fingerprint(news.scheduledBackup) &&
480 "scheduledBackup",
481 news.metadataIntegration !== undefined &&
482 fingerprint(current.metadataIntegration) !== fingerprint(news.metadataIntegration) &&
483 "metadataIntegration",
484 news.maintenanceWindow !== undefined &&
485 fingerprint(current.maintenanceWindow) !== fingerprint(news.maintenanceWindow) &&
486 "maintenanceWindow",
487 hiveChanged && "hiveMetastoreConfig",
488 ]);
489
490 if (mask.length > 0) {
491 const operation = yield* metastore.patchProjectsLocationsServices({
492 name: current.name ?? name,
493 updateMask: mask,
494 body: {
495 labels: desiredLabels,
496 port: news.port,
497 tier: news.tier,
498 deletionProtection: news.deletionProtection,
499 scalingConfig: news.scalingConfig,
500 telemetryConfig: news.telemetryConfig,
501 scheduledBackup: news.scheduledBackup,
502 metadataIntegration: news.metadataIntegration,
503 maintenanceWindow: news.maintenanceWindow,
504 hiveMetastoreConfig: hive,
505 },
506 });
507 yield* waitForOperation(operation);
508 current = yield* waitUntilReady(
509 getByName(current.name ?? name),
510 current.name ?? name,
511 (item) => item.state,
512 (item) => item.stateMessage,
513 );
514 }
515
516 if (current === undefined) {
517 return yield* new ResourceNotResolved({ resourceName: name });
518 }
519
520 return toAttrs(current, env.project, env.region);
521 }),
522
523 delete: Effect.fn(function* ({ output }) {
524 const existing = yield* getByName(output.name);
525 if (existing === undefined) return;
526 if (existing.deletionProtection === true) {
527 const unlocked = yield* metastore
528 .patchProjectsLocationsServices({
529 name: output.name,
530 updateMask: "deletionProtection",
531 body: { deletionProtection: false },
532 })
533 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
534 if (unlocked !== undefined) {
535 // NOT_FOUND (5): already gone.
536 yield* waitForOperation(unlocked).pipe(
537 Effect.catchTag("GCP.OperationFailed", (error) =>
538 error.code === 5 ? Effect.void : Effect.fail(error),
539 ),
540 );
541 }
542 }
543 const operation = yield* metastore
544 .deleteProjectsLocationsServices({ name: output.name })
545 .pipe(
546 Effect.retry({
547 while: (error) => error._tag === "Conflict",
548 times: 8,
549 schedule: Schedule.spaced("2 seconds"),
550 }),
551 Effect.catchTag("NotFound", () => Effect.undefined),
552 );
553 if (operation !== undefined) {
554 // NOT_FOUND (5): already gone.
555 yield* waitForOperation(operation).pipe(
556 Effect.catchTag("GCP.OperationFailed", (error) =>
557 error.code === 5 ? Effect.void : Effect.fail(error),
558 ),
559 );
560 }
561 yield* waitUntilGone(getByName(output.name), output.name);
562 }),
563 });