@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 1,016 lines 36.7 KB
Raw
1 import * as dataproc from "@distilled.cloud/gcp/dataproc_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { GcpEnvironment } from "../Environment.ts";
13 import {
14 createInternalLabels,
15 diffLabels,
16 hasAlchemyLabels,
17 stripInternalLabels,
18 toLabels,
19 } from "../Labels.ts";
20 import { OperationFailed } from "../Operation.ts";
21 import type { Providers } from "../Providers.ts";
22 import { MAX_POLLS, waitForOperation } from "./internal.ts";
23
24 const DEFAULT_CLUSTER_TYPE = "SINGLE_NODE";
25 const DEFAULT_MACHINE_TYPE = "e2-standard-2";
26 const DEFAULT_BOOT_DISK_GB = 30;
27 const DEFAULT_BOOT_DISK_TYPE = "pd-standard";
28 const DEFAULT_MASTER_INSTANCES = 1;
29 const DEFAULT_STANDARD_WORKERS = 2;
30 const MAX_NAME_LENGTH = 51;
31
32 const LIST_REGIONS = [
33 "us-central1",
34 "us-east1",
35 "us-east4",
36 "us-west1",
37 "europe-west1",
38 "asia-east1",
39 ] as const;
40
41 export type InitializationAction = {
42 /** Cloud Storage URI of the executable (`gs://...`). */
43 executableFile: string;
44 /**
45 * How long the executable may run (JSON Duration, e.g. `"10m"`).
46 * @default "10m"
47 */
48 executionTimeout?: string;
49 };
50
51 export type ClusterProps = {
52 /**
53 * Cluster id unique within the project. If omitted, a unique RFC1035
54 * name is generated from the stack, stage, and logical id. Must start
55 * with a lowercase letter, contain only lowercase letters, numbers, and
56 * hyphens, and not end with a hyphen (up to 51 characters). Immutable —
57 * changing it replaces the cluster.
58 */
59 clusterName?: string;
60 /**
61 * Dataproc region (`us-central1`, `us-east1`, …). Immutable — changing
62 * it replaces the cluster. `US-CENTRAL1` is accepted and normalized to
63 * `us-central1`.
64 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
65 */
66 region?: string;
67 /**
68 * User labels. Alchemy ownership labels are merged in automatically.
69 */
70 labels?: Record<string, string>;
71 /**
72 * Cluster topology. `SINGLE_NODE` is a 1-VM cluster (no workers).
73 * Immutable — changing it replaces the cluster.
74 * @default "SINGLE_NODE"
75 */
76 clusterType?: dataproc.ClusterConfigClusterTypeEnum | (string & {});
77 /**
78 * Dataproc image version (`2.2`, `2.2-debian12`, …). Prefix matches
79 * the server-resolved version. Immutable — changing it replaces the
80 * cluster.
81 */
82 imageVersion?: string;
83 /**
84 * Optional components to activate (`JUPYTER`, `ZEPPELIN`, `TRINO`, …).
85 * Immutable — changing them replaces the cluster.
86 */
87 optionalComponents?: dataproc.SoftwareConfigOptionalComponentsItemEnumList;
88 /**
89 * Daemon properties (`core:hadoop.tmp.dir`, `spark:spark.executor.memory`,
90 * …). Immutable — changing them replaces the cluster.
91 */
92 properties?: Record<string, string>;
93 /**
94 * Compute Engine zone (`us-central1-a`). Immutable. If omitted,
95 * Dataproc picks a zone in `region`.
96 */
97 zone?: string;
98 /**
99 * VPC network name or URI. Cannot be set with `subnetwork`. Immutable.
100 */
101 network?: string;
102 /**
103 * Subnetwork name or URI. Cannot be set with `network`. Immutable.
104 */
105 subnetwork?: string;
106 /**
107 * Restrict VMs to internal IPs. Immutable. Defaults to `false` so the
108 * cluster can reach Google APIs without Private Google Access.
109 * @default false
110 */
111 internalIpOnly?: boolean;
112 /**
113 * VM service account email. Immutable.
114 */
115 serviceAccount?: string;
116 /**
117 * Extra OAuth scopes for cluster VMs. Immutable.
118 */
119 serviceAccountScopes?: string[];
120 /**
121 * Network tags applied to every VM. Immutable.
122 */
123 tags?: string[];
124 /**
125 * Compute Engine metadata entries on every VM. Immutable.
126 */
127 metadata?: Record<string, string>;
128 /**
129 * Master instances. `1` for standard or single-node, `3` for HA.
130 * Immutable — changing it replaces the cluster.
131 * @default 1
132 */
133 masterNumInstances?: number;
134 /**
135 * Master machine type short name or URI.
136 * @default "e2-standard-2"
137 */
138 masterMachineType?: string;
139 /**
140 * Master boot disk size in GB.
141 * @default 30
142 */
143 masterBootDiskSizeGb?: number;
144 /**
145 * Master boot disk type (`pd-standard`, `pd-balanced`, `pd-ssd`).
146 * @default "pd-standard"
147 */
148 masterBootDiskType?: string;
149 /**
150 * Primary worker count. `0` for single-node. Resizes in place.
151 * @default 0
152 */
153 workerNumInstances?: number;
154 /**
155 * Primary worker machine type short name or URI. Immutable.
156 * @default "e2-standard-2"
157 */
158 workerMachineType?: string;
159 /**
160 * Primary worker boot disk size in GB. Immutable.
161 * @default 30
162 */
163 workerBootDiskSizeGb?: number;
164 /**
165 * Primary worker boot disk type. Immutable.
166 * @default "pd-standard"
167 */
168 workerBootDiskType?: string;
169 /**
170 * Secondary (preemptible or non-preemptible) worker count. Resizes in
171 * place.
172 */
173 secondaryWorkerNumInstances?: number;
174 /**
175 * Secondary worker machine type. Immutable.
176 */
177 secondaryWorkerMachineType?: string;
178 /**
179 * Secondary worker boot disk size in GB. Immutable.
180 */
181 secondaryWorkerBootDiskSizeGb?: number;
182 /**
183 * Secondary worker preemptibility (`PREEMPTIBLE`, `SPOT`,
184 * `NON_PREEMPTIBLE`). Immutable.
185 */
186 secondaryWorkerPreemptibility?: dataproc.InstanceGroupConfigPreemptibilityEnum | (string & {});
187 /**
188 * Autoscaling policy resource name. Updated in place.
189 */
190 autoscalingPolicyUri?: string;
191 /**
192 * Idle TTL before the cluster is deleted (JSON Duration). Immutable.
193 */
194 idleDeleteTtl?: string;
195 /**
196 * Lifetime TTL before the cluster is deleted (JSON Duration). Immutable.
197 */
198 autoDeleteTtl?: string;
199 /**
200 * Staging bucket name (not a `gs://` URI). Immutable.
201 */
202 configBucket?: string;
203 /**
204 * Temp bucket name (not a `gs://` URI). Immutable.
205 */
206 tempBucket?: string;
207 /**
208 * Enable Component Gateway HTTP port access.
209 * @default false
210 */
211 enableHttpPortAccess?: boolean;
212 /**
213 * Customer-managed KMS key for PD and job-argument encryption.
214 * Immutable.
215 */
216 kmsKey?: string;
217 /**
218 * Scripts to run on each node after config completes. Immutable.
219 */
220 initializationActions?: InitializationAction[];
221 };
222
223 export type Cluster = Resource<
224 "GCP.Dataproc.Cluster",
225 ClusterProps,
226 {
227 /** Synthetic name `projects/{project}/regions/{region}/clusters/{cluster}`. */
228 name: string;
229 /** Cluster id. */
230 clusterName: string;
231 /** Project id. */
232 project: string;
233 /** Region id (`us-central1`, …). */
234 region: string;
235 /** Server-assigned cluster UUID. */
236 clusterUuid: string | undefined;
237 /** Server-reported state (`RUNNING`, `CREATING`, …). */
238 state: string | undefined;
239 /** Extra status text, if any. */
240 statusDetail: string | undefined;
241 /** User labels (Alchemy ownership labels stripped). */
242 labels: Record<string, string>;
243 /** Cluster topology (`SINGLE_NODE`, `STANDARD`, …). */
244 clusterType: string;
245 /** Resolved image version. */
246 imageVersion: string | undefined;
247 /** Provisioned zone. */
248 zone: string | undefined;
249 /** VPC network. */
250 network: string | undefined;
251 /** Subnetwork. */
252 subnetwork: string | undefined;
253 /** Master instance count. */
254 masterNumInstances: number | undefined;
255 /** Master machine type short name. */
256 masterMachineType: string | undefined;
257 /** Master boot disk size in GB. */
258 masterBootDiskSizeGb: number | undefined;
259 /** Master boot disk type. */
260 masterBootDiskType: string | undefined;
261 /** Primary worker count. */
262 workerNumInstances: number | undefined;
263 /** Primary worker machine type short name. */
264 workerMachineType: string | undefined;
265 /** Primary worker boot disk size in GB. */
266 workerBootDiskSizeGb: number | undefined;
267 /** Primary worker boot disk type. */
268 workerBootDiskType: string | undefined;
269 /** Whether VMs use internal IPs only. */
270 internalIpOnly: boolean | undefined;
271 /** VM service account email. */
272 serviceAccount: string | undefined;
273 /** Secondary worker count. */
274 secondaryWorkerNumInstances: number | undefined;
275 /** Autoscaling policy resource name. */
276 autoscalingPolicyUri: string | undefined;
277 /** Staging bucket name. */
278 configBucket: string | undefined;
279 /** Temp bucket name. */
280 tempBucket: string | undefined;
281 /** Component Gateway HTTP ports, when enabled. */
282 httpPorts: Record<string, string>;
283 },
284 never,
285 Providers
286 >;
287
288 /**
289 * A Dataproc cluster of Compute Engine VMs.
290 *
291 * Defaults to a single-node cluster (`e2-standard-2`, 30 GB boot disk) so
292 * a bare `Cluster("Spark", {})` stays cheap. Set `clusterType: "STANDARD"`
293 * and `workerNumInstances` for a multi-VM cluster.
294 *
295 * Changing `clusterName`, `region`, topology, image, machine types, disks,
296 * network, zone, or software config replaces the cluster. Labels, primary
297 * worker count, secondary worker count, and the autoscaling policy URI
298 * update in place.
299 *
300 * Provisioning typically takes several minutes.
301 *
302 * ### Creating a Cluster
303 * **Example:** Generated name, single-node
304 * ```typescript
305 * const cluster = yield* GCP.Dataproc.Cluster("Spark", {});
306 * ```
307 *
308 * **Example:** Explicit id, labels, and machine type
309 * ```typescript
310 * const cluster = yield* GCP.Dataproc.Cluster("Spark", {
311 * clusterName: "app-spark",
312 * region: "us-central1",
313 * clusterType: "SINGLE_NODE",
314 * masterMachineType: "e2-standard-2",
315 * masterBootDiskSizeGb: 30,
316 * labels: { env: "prod" },
317 * });
318 * ```
319 *
320 * ### Standard topology
321 * **Example:** One master and two workers
322 * ```typescript
323 * const cluster = yield* GCP.Dataproc.Cluster("Spark", {
324 * clusterType: "STANDARD",
325 * workerNumInstances: 2,
326 * workerMachineType: "e2-standard-2",
327 * workerBootDiskSizeGb: 30,
328 * });
329 * ```
330 *
331 * @resource
332 * @category Dataproc
333 */
334 export const Cluster = Resource<Cluster>("GCP.Dataproc.Cluster");
335
336 export class ClusterNotResolved extends Data.TaggedError("GCP.Dataproc.ClusterNotResolved")<{
337 clusterName: string;
338 region: string;
339 }> {}
340
341 export class ClusterFailed extends Data.TaggedError("GCP.Dataproc.ClusterFailed")<{
342 clusterName: string;
343 state: string | undefined;
344 detail: string | undefined;
345 }> {}
346
347 export class ClusterNotReady extends Data.TaggedError("GCP.Dataproc.ClusterNotReady")<{
348 clusterName: string;
349 state: string | undefined;
350 }> {}
351
352 export class ClusterStillExists extends Data.TaggedError("GCP.Dataproc.ClusterStillExists")<{
353 clusterName: string;
354 }> {}
355
356 const lastSegment = (value: string) => {
357 const trimmed = value.replace(/\/+$/, "");
358 const parts = trimmed.split("/");
359 return parts[parts.length - 1] || trimmed;
360 };
361
362 const normalizeRegion = (region: string | undefined, fallback: string) =>
363 lastSegment(region ?? fallback).toLowerCase();
364
365 const linkKey = (value: string | undefined) =>
366 value === undefined || value === "" ? "" : lastSegment(value).toLowerCase();
367
368 const normalizeClusterType = (type: string | undefined, workerNumInstances: number | undefined) => {
369 const value = (type ?? "").toUpperCase();
370 if (value !== "" && value !== "CLUSTER_TYPE_UNSPECIFIED" && value !== "UNSPECIFIED") {
371 return value;
372 }
373 if (workerNumInstances === 0) return "SINGLE_NODE";
374 return DEFAULT_CLUSTER_TYPE;
375 };
376
377 const desiredClusterType = (news: ClusterProps) =>
378 normalizeClusterType(news.clusterType, news.workerNumInstances);
379
380 const rfc1035 = (name: string): string => {
381 let next = name
382 .toLowerCase()
383 .replace(/[^a-z0-9-]/g, "-")
384 .replace(/-+/g, "-")
385 .replace(/^-+|-+$/g, "");
386 if (!/^[a-z]/.test(next)) next = `c${next}`;
387 next = next.slice(0, MAX_NAME_LENGTH).replace(/-+$/g, "");
388 if (next.length === 0) return "cluster";
389 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, MAX_NAME_LENGTH - 1)}0`;
390 return next.slice(0, MAX_NAME_LENGTH);
391 };
392
393 const resourceName = (project: string, region: string, clusterName: string) =>
394 `projects/${project}/regions/${region}/clusters/${clusterName}`;
395
396 const parseName = (name: string, fallbackRegion: string) => {
397 const parts = name.split("/").filter((part) => part.length > 0);
398 const clustersAt = parts.lastIndexOf("clusters");
399 const regionsAt = parts.lastIndexOf("regions");
400 const projectsAt = parts.lastIndexOf("projects");
401 return {
402 project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
403 region: regionsAt >= 0 && parts[regionsAt + 1] ? parts[regionsAt + 1]! : fallbackRegion,
404 clusterName:
405 clustersAt >= 0 && parts[clustersAt + 1] ? parts[clustersAt + 1]! : lastSegment(name),
406 };
407 };
408
409 const userLabels = (
410 labels: Record<string, string | undefined> | null | undefined,
411 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
412
413 const toId = (id: string, clusterName: string | undefined, existing?: string) =>
414 Effect.gen(function* () {
415 return (
416 clusterName ??
417 existing ??
418 rfc1035(
419 yield* createPhysicalName({
420 id,
421 maxLength: MAX_NAME_LENGTH,
422 lowercase: true,
423 }),
424 )
425 );
426 });
427
428 const configsOf = (
429 configs: Record<string, string | undefined> | null | undefined,
430 ): Record<string, string> =>
431 Object.fromEntries(
432 Object.entries(configs ?? {}).filter(
433 (entry): entry is [string, string] => entry[1] !== undefined,
434 ),
435 );
436
437 const sortedKey = (values: readonly string[] | undefined) =>
438 JSON.stringify([...(values ?? [])].map((value) => value.toUpperCase()).sort());
439
440 const propertiesKey = (properties: Record<string, string | undefined> | null | undefined) =>
441 JSON.stringify(
442 Object.fromEntries(
443 Object.entries(configsOf(properties)).sort(([a], [b]) => a.localeCompare(b)),
444 ),
445 );
446
447 const metadataKey = (metadata: Record<string, string | undefined> | null | undefined) =>
448 propertiesKey(metadata);
449
450 const initKey = (actions: InitializationAction[] | undefined) =>
451 JSON.stringify(
452 (actions ?? []).map((action) => ({
453 executableFile: action.executableFile,
454 executionTimeout: action.executionTimeout ?? "",
455 })),
456 );
457
458 const imageMatches = (desired: string | undefined, observed: string | undefined) => {
459 if (desired === undefined || desired === "") return true;
460 if (observed === undefined || observed === "") return false;
461 const next = desired.toLowerCase();
462 const current = observed.toLowerCase();
463 return current === next || current.startsWith(`${next}-`) || current.startsWith(`${next}.`);
464 };
465
466 const groupNum = (group: dataproc.InstanceGroupConfig | undefined) => group?.numInstances;
467
468 const toAttrs = (cluster: dataproc.Cluster, project: string, region: string) => {
469 const clusterName = cluster.clusterName ?? "";
470 const config = cluster.config;
471 const master = config?.masterConfig;
472 const worker = config?.workerConfig;
473 const secondary = config?.secondaryWorkerConfig;
474 return {
475 name: resourceName(project, region, clusterName),
476 clusterName,
477 project: cluster.projectId || project,
478 region,
479 clusterUuid: cluster.clusterUuid,
480 state: cluster.status?.state,
481 statusDetail: cluster.status?.detail,
482 labels: userLabels(cluster.labels),
483 clusterType: normalizeClusterType(config?.clusterType, worker?.numInstances),
484 imageVersion: config?.softwareConfig?.imageVersion,
485 zone: config?.gceClusterConfig?.zoneUri
486 ? lastSegment(config.gceClusterConfig.zoneUri)
487 : undefined,
488 network: config?.gceClusterConfig?.networkUri
489 ? lastSegment(config.gceClusterConfig.networkUri)
490 : undefined,
491 subnetwork: config?.gceClusterConfig?.subnetworkUri
492 ? lastSegment(config.gceClusterConfig.subnetworkUri)
493 : undefined,
494 masterNumInstances: master?.numInstances,
495 masterMachineType: master?.machineTypeUri ? lastSegment(master.machineTypeUri) : undefined,
496 masterBootDiskSizeGb: master?.diskConfig?.bootDiskSizeGb,
497 masterBootDiskType: master?.diskConfig?.bootDiskType,
498 workerNumInstances: worker?.numInstances,
499 workerMachineType: worker?.machineTypeUri ? lastSegment(worker.machineTypeUri) : undefined,
500 workerBootDiskSizeGb: worker?.diskConfig?.bootDiskSizeGb,
501 workerBootDiskType: worker?.diskConfig?.bootDiskType,
502 internalIpOnly: config?.gceClusterConfig?.internalIpOnly,
503 serviceAccount: config?.gceClusterConfig?.serviceAccount,
504 secondaryWorkerNumInstances: secondary?.numInstances,
505 autoscalingPolicyUri: config?.autoscalingConfig?.policyUri,
506 configBucket: config?.configBucket,
507 tempBucket: config?.tempBucket,
508 httpPorts: configsOf(config?.endpointConfig?.httpPorts),
509 };
510 };
511
512 const getById = (projectId: string, region: string, clusterName: string) =>
513 dataproc
514 .getProjectsRegionsClusters({ projectId, region, clusterName })
515 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
516
517 const waitUntilExists = (projectId: string, region: string, clusterName: string) =>
518 getById(projectId, region, clusterName).pipe(
519 Effect.flatMap((cluster) =>
520 cluster
521 ? Effect.succeed(cluster)
522 : Effect.fail(new ClusterNotResolved({ clusterName, region })),
523 ),
524 Effect.retry({
525 while: (error) => error._tag === "GCP.Dataproc.ClusterNotResolved",
526 times: 8,
527 schedule: Schedule.spaced("1 second"),
528 }),
529 );
530
531 const terminalError = (state: string | undefined) =>
532 state === "ERROR" || state === "ERROR_DUE_TO_UPDATE";
533
534 const waitUntilRunning = (projectId: string, region: string, clusterName: string) =>
535 getById(projectId, region, clusterName).pipe(
536 Effect.filterOrFail(
537 (cluster): cluster is dataproc.Cluster => cluster !== undefined,
538 () => new ClusterNotResolved({ clusterName, region }),
539 ),
540 Effect.filterOrFail(
541 (cluster) => !terminalError(cluster.status?.state),
542 (cluster) =>
543 new ClusterFailed({
544 clusterName,
545 state: cluster.status?.state,
546 detail: cluster.status?.detail,
547 }),
548 ),
549 Effect.filterOrFail(
550 (cluster) => (cluster.status?.state ?? "") === "RUNNING",
551 (cluster) =>
552 new ClusterNotReady({
553 clusterName,
554 state: cluster.status?.state,
555 }),
556 ),
557 Effect.retry({
558 while: (error) =>
559 error._tag === "GCP.Dataproc.ClusterNotReady" ||
560 error._tag === "GCP.Dataproc.ClusterNotResolved",
561 times: MAX_POLLS,
562 schedule: Schedule.spaced("5 seconds"),
563 }),
564 );
565
566 const waitUntilGone = (projectId: string, region: string, clusterName: string) =>
567 getById(projectId, region, clusterName).pipe(
568 Effect.flatMap((cluster) =>
569 cluster === undefined ? Effect.void : Effect.fail(new ClusterStillExists({ clusterName })),
570 ),
571 Effect.retry({
572 while: (error) => error._tag === "GCP.Dataproc.ClusterStillExists",
573 times: MAX_POLLS,
574 schedule: Schedule.spaced("5 seconds"),
575 }),
576 );
577
578 const deleteCluster = (projectId: string, region: string, clusterName: string) =>
579 Effect.gen(function* () {
580 const operation = yield* dataproc
581 .deleteProjectsRegionsClusters({ projectId, region, clusterName })
582 .pipe(
583 Effect.catchTag("NotFound", () => Effect.undefined),
584 Effect.retry({
585 while: (error) => error._tag === "Conflict",
586 times: 8,
587 schedule: Schedule.spaced("5 seconds"),
588 }),
589 );
590 if (operation !== undefined) {
591 yield* waitForOperation(operation, { notFoundOk: true });
592 }
593 yield* waitUntilGone(projectId, region, clusterName);
594 });
595
596 /**
597 * A create operation that fails because the VPC subnet is transiently busy
598 * (`The resource '.../subnetworks/default' is not ready`, raised while other
599 * operations mutate the shared subnet). LRO errors carry only a code and a
600 * message, so the message is the only discriminator.
601 */
602 const isSubnetNotReady = (error: { readonly _tag: string }) =>
603 error instanceof OperationFailed &&
604 error.message.includes("/subnetworks/") &&
605 error.message.includes("is not ready");
606
607 /**
608 * Create the cluster and wait for the create operation. A create that fails
609 * on a busy subnet leaves the cluster in `ERROR`; it is deleted and the
610 * create retried.
611 */
612 const createCluster = (
613 projectId: string,
614 region: string,
615 clusterName: string,
616 body: dataproc.Cluster,
617 ) =>
618 Effect.gen(function* () {
619 const existing = yield* getById(projectId, region, clusterName);
620 if (existing !== undefined && terminalError(existing.status?.state)) {
621 yield* deleteCluster(projectId, region, clusterName);
622 }
623 const created = yield* dataproc
624 .createProjectsRegionsClusters({ projectId, region, body })
625 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
626 if (created !== undefined) {
627 yield* waitForOperation(created);
628 }
629 }).pipe(
630 Effect.retry({
631 while: isSubnetNotReady,
632 times: 6,
633 schedule: Schedule.spaced("30 seconds"),
634 }),
635 );
636
637 const instanceGroup = (options: {
638 numInstances?: number;
639 machineType?: string;
640 bootDiskSizeGb?: number;
641 bootDiskType?: string;
642 preemptibility?: dataproc.InstanceGroupConfigPreemptibilityEnum | (string & {});
643 }): dataproc.InstanceGroupConfig | undefined => {
644 const group: dataproc.InstanceGroupConfig = {};
645 if (options.numInstances !== undefined) {
646 group.numInstances = options.numInstances;
647 }
648 if (options.machineType !== undefined) {
649 group.machineTypeUri = options.machineType;
650 }
651 const disk: dataproc.DiskConfig = {};
652 if (options.bootDiskSizeGb !== undefined) {
653 disk.bootDiskSizeGb = options.bootDiskSizeGb;
654 }
655 if (options.bootDiskType !== undefined) {
656 disk.bootDiskType = options.bootDiskType;
657 }
658 if (Object.keys(disk).length > 0) {
659 group.diskConfig = disk;
660 }
661 if (options.preemptibility !== undefined) {
662 group.preemptibility = options.preemptibility;
663 }
664 return Object.keys(group).length > 0 ? group : undefined;
665 };
666
667 const toCreateBody = (
668 news: ClusterProps,
669 projectId: string,
670 clusterName: string,
671 desiredLabels: Record<string, string>,
672 ): dataproc.Cluster => {
673 const clusterType = desiredClusterType(news);
674 const singleNode = clusterType === "SINGLE_NODE";
675 const workerNum = news.workerNumInstances ?? (singleNode ? 0 : DEFAULT_STANDARD_WORKERS);
676 const masterNum = news.masterNumInstances ?? DEFAULT_MASTER_INSTANCES;
677 const masterMachine = news.masterMachineType ?? DEFAULT_MACHINE_TYPE;
678 const workerMachine = news.workerMachineType ?? news.masterMachineType ?? DEFAULT_MACHINE_TYPE;
679 const masterDiskGb = news.masterBootDiskSizeGb ?? DEFAULT_BOOT_DISK_GB;
680 const workerDiskGb =
681 news.workerBootDiskSizeGb ?? news.masterBootDiskSizeGb ?? DEFAULT_BOOT_DISK_GB;
682 const masterDiskType = news.masterBootDiskType ?? DEFAULT_BOOT_DISK_TYPE;
683 const workerDiskType =
684 news.workerBootDiskType ?? news.masterBootDiskType ?? DEFAULT_BOOT_DISK_TYPE;
685
686 const gce: dataproc.GceClusterConfig = {};
687 if (news.zone !== undefined) gce.zoneUri = news.zone;
688 if (news.network !== undefined) gce.networkUri = news.network;
689 if (news.subnetwork !== undefined) gce.subnetworkUri = news.subnetwork;
690 gce.internalIpOnly = news.internalIpOnly === true;
691 if (news.serviceAccount !== undefined) gce.serviceAccount = news.serviceAccount;
692 if (news.serviceAccountScopes !== undefined) {
693 gce.serviceAccountScopes = news.serviceAccountScopes;
694 }
695 if (news.tags !== undefined) gce.tags = news.tags;
696 if (news.metadata !== undefined) gce.metadata = news.metadata;
697
698 const software: dataproc.SoftwareConfig = {};
699 if (news.imageVersion !== undefined) software.imageVersion = news.imageVersion;
700 if (news.optionalComponents !== undefined) {
701 software.optionalComponents = news.optionalComponents;
702 }
703 if (news.properties !== undefined) software.properties = news.properties;
704
705 const lifecycle: dataproc.LifecycleConfig = {};
706 if (news.idleDeleteTtl !== undefined) lifecycle.idleDeleteTtl = news.idleDeleteTtl;
707 if (news.autoDeleteTtl !== undefined) lifecycle.autoDeleteTtl = news.autoDeleteTtl;
708
709 const config: dataproc.ClusterConfig = {
710 clusterType,
711 masterConfig: instanceGroup({
712 numInstances: masterNum,
713 machineType: masterMachine,
714 bootDiskSizeGb: masterDiskGb,
715 bootDiskType: masterDiskType,
716 }),
717 workerConfig: instanceGroup({
718 numInstances: workerNum,
719 machineType: singleNode ? undefined : workerMachine,
720 bootDiskSizeGb: singleNode ? undefined : workerDiskGb,
721 bootDiskType: singleNode ? undefined : workerDiskType,
722 }),
723 };
724
725 if (Object.keys(gce).length > 0) config.gceClusterConfig = gce;
726 if (Object.keys(software).length > 0) config.softwareConfig = software;
727 if (Object.keys(lifecycle).length > 0) config.lifecycleConfig = lifecycle;
728 if (news.autoscalingPolicyUri !== undefined) {
729 config.autoscalingConfig = { policyUri: news.autoscalingPolicyUri };
730 }
731 if (news.configBucket !== undefined) config.configBucket = news.configBucket;
732 if (news.tempBucket !== undefined) config.tempBucket = news.tempBucket;
733 if (news.enableHttpPortAccess === true) {
734 config.endpointConfig = { enableHttpPortAccess: true };
735 }
736 if (news.kmsKey !== undefined) {
737 config.encryptionConfig = { kmsKey: news.kmsKey };
738 }
739 if (news.initializationActions !== undefined) {
740 config.initializationActions = news.initializationActions;
741 }
742 if (
743 news.secondaryWorkerNumInstances !== undefined ||
744 news.secondaryWorkerMachineType !== undefined ||
745 news.secondaryWorkerBootDiskSizeGb !== undefined ||
746 news.secondaryWorkerPreemptibility !== undefined
747 ) {
748 config.secondaryWorkerConfig = instanceGroup({
749 numInstances: news.secondaryWorkerNumInstances,
750 machineType: news.secondaryWorkerMachineType,
751 bootDiskSizeGb: news.secondaryWorkerBootDiskSizeGb,
752 preemptibility: news.secondaryWorkerPreemptibility,
753 });
754 }
755
756 return {
757 projectId,
758 clusterName,
759 labels: desiredLabels,
760 config,
761 };
762 };
763
764 const listRegion = (projectId: string, region: string) =>
765 dataproc.listProjectsRegionsClusters
766 .pages({
767 projectId,
768 region,
769 pageSize: 1000,
770 })
771 .pipe(
772 Stream.flatMap((page) => Stream.fromIterable(page.clusters ?? [])),
773 Stream.filter((cluster) =>
774 Object.keys(cluster.labels ?? {}).some((key) => key.startsWith("alchemy-")),
775 ),
776 Stream.map((cluster) => toAttrs(cluster, projectId, region)),
777 Stream.runCollect,
778 Effect.map((chunk) => Array.from(chunk)),
779 Effect.catchTag("NotFound", () => Effect.succeed([])),
780 );
781
782 export const ClusterProvider = () =>
783 Provider.succeed(Cluster, {
784 stables: ["name", "clusterName", "project", "region", "clusterUuid"],
785
786 diff: Effect.fn(function* ({ news, olds, output }) {
787 if (!isResolved(news)) return undefined;
788 const env = yield* GcpEnvironment.current;
789
790 const previousName = olds?.clusterName ?? output?.clusterName;
791 const nextName = news.clusterName ?? previousName;
792 const previousRegion = normalizeRegion(olds?.region ?? output?.region, env.region);
793 const nextRegion = normalizeRegion(news.region ?? output?.region, env.region);
794 const previousType = normalizeClusterType(
795 olds?.clusterType ?? output?.clusterType,
796 olds?.workerNumInstances ?? output?.workerNumInstances,
797 );
798 const nextType = desiredClusterType({
799 clusterType: news.clusterType ?? previousType,
800 workerNumInstances: news.workerNumInstances ?? output?.workerNumInstances,
801 });
802
803 const replace =
804 (previousName !== undefined && nextName !== undefined && nextName !== previousName) ||
805 previousRegion !== nextRegion ||
806 previousType !== nextType ||
807 (news.imageVersion !== undefined &&
808 !imageMatches(news.imageVersion, olds?.imageVersion ?? output?.imageVersion)) ||
809 (news.masterNumInstances !== undefined &&
810 news.masterNumInstances !== (olds?.masterNumInstances ?? output?.masterNumInstances)) ||
811 (news.masterMachineType !== undefined &&
812 linkKey(news.masterMachineType) !==
813 linkKey(olds?.masterMachineType ?? output?.masterMachineType)) ||
814 (news.workerMachineType !== undefined &&
815 linkKey(news.workerMachineType) !==
816 linkKey(olds?.workerMachineType ?? output?.workerMachineType)) ||
817 (news.masterBootDiskSizeGb !== undefined &&
818 news.masterBootDiskSizeGb !==
819 (olds?.masterBootDiskSizeGb ?? output?.masterBootDiskSizeGb)) ||
820 (news.workerBootDiskSizeGb !== undefined &&
821 news.workerBootDiskSizeGb !==
822 (olds?.workerBootDiskSizeGb ?? output?.workerBootDiskSizeGb)) ||
823 (news.masterBootDiskType !== undefined &&
824 news.masterBootDiskType.toLowerCase() !==
825 (olds?.masterBootDiskType ?? output?.masterBootDiskType ?? "").toLowerCase()) ||
826 (news.workerBootDiskType !== undefined &&
827 news.workerBootDiskType.toLowerCase() !==
828 (olds?.workerBootDiskType ?? output?.workerBootDiskType ?? "").toLowerCase()) ||
829 (news.zone !== undefined && linkKey(news.zone) !== linkKey(olds?.zone ?? output?.zone)) ||
830 (news.network !== undefined &&
831 linkKey(news.network) !== linkKey(olds?.network ?? output?.network)) ||
832 (news.subnetwork !== undefined &&
833 linkKey(news.subnetwork) !== linkKey(olds?.subnetwork ?? output?.subnetwork)) ||
834 (news.internalIpOnly !== undefined &&
835 news.internalIpOnly !== (olds?.internalIpOnly ?? output?.internalIpOnly)) ||
836 (news.serviceAccount !== undefined &&
837 news.serviceAccount !== (olds?.serviceAccount ?? output?.serviceAccount)) ||
838 (news.serviceAccountScopes !== undefined &&
839 sortedKey(news.serviceAccountScopes) !== sortedKey(olds?.serviceAccountScopes)) ||
840 (news.tags !== undefined && sortedKey(news.tags) !== sortedKey(olds?.tags)) ||
841 (news.metadata !== undefined &&
842 metadataKey(news.metadata) !== metadataKey(olds?.metadata)) ||
843 (news.optionalComponents !== undefined &&
844 sortedKey(news.optionalComponents) !== sortedKey(olds?.optionalComponents)) ||
845 (news.properties !== undefined &&
846 propertiesKey(news.properties) !== propertiesKey(olds?.properties)) ||
847 (news.secondaryWorkerMachineType !== undefined &&
848 linkKey(news.secondaryWorkerMachineType) !== linkKey(olds?.secondaryWorkerMachineType)) ||
849 (news.secondaryWorkerBootDiskSizeGb !== undefined &&
850 news.secondaryWorkerBootDiskSizeGb !==
851 (olds?.secondaryWorkerBootDiskSizeGb ?? undefined)) ||
852 (news.secondaryWorkerPreemptibility !== undefined &&
853 (news.secondaryWorkerPreemptibility ?? "").toUpperCase() !==
854 (olds?.secondaryWorkerPreemptibility ?? "").toUpperCase()) ||
855 (news.idleDeleteTtl !== undefined &&
856 news.idleDeleteTtl !== (olds?.idleDeleteTtl ?? undefined)) ||
857 (news.autoDeleteTtl !== undefined &&
858 news.autoDeleteTtl !== (olds?.autoDeleteTtl ?? undefined)) ||
859 (news.configBucket !== undefined &&
860 news.configBucket !== (olds?.configBucket ?? output?.configBucket)) ||
861 (news.tempBucket !== undefined &&
862 news.tempBucket !== (olds?.tempBucket ?? output?.tempBucket)) ||
863 (news.enableHttpPortAccess !== undefined &&
864 news.enableHttpPortAccess !== (olds?.enableHttpPortAccess ?? undefined)) ||
865 (news.kmsKey !== undefined && news.kmsKey !== (olds?.kmsKey ?? "")) ||
866 (news.initializationActions !== undefined &&
867 initKey(news.initializationActions) !== initKey(olds?.initializationActions));
868
869 if (!replace) return undefined;
870 return {
871 action: "replace" as const,
872 deleteFirst:
873 previousRegion === nextRegion && previousName !== undefined && nextName === previousName,
874 };
875 }),
876
877 read: Effect.fn(function* ({ id, olds, output }) {
878 const env = yield* GcpEnvironment.current;
879 const clusterName = yield* toId(id, olds?.clusterName, output?.clusterName);
880 const region = normalizeRegion(olds?.region ?? output?.region, env.region);
881 const parsed = output?.name ? parseName(output.name, region) : undefined;
882 const project = parsed?.project || env.project;
883 const existing = yield* getById(project, region, clusterName);
884 if (existing === undefined) return undefined;
885 const attrs = toAttrs(existing, project, region);
886 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
887 }),
888
889 list: () =>
890 Effect.gen(function* () {
891 const env = yield* GcpEnvironment.current;
892 const pages = yield* Effect.forEach(
893 [...new Set([env.region, ...LIST_REGIONS])],
894 (region) => listRegion(env.project, region),
895 { concurrency: 4 },
896 );
897 return pages.flat();
898 }),
899
900 reconcile: Effect.fn(function* ({ id, news, output }) {
901 const env = yield* GcpEnvironment.current;
902 const clusterName = yield* toId(id, news.clusterName, output?.clusterName);
903 const region = normalizeRegion(news.region ?? output?.region, env.region);
904 const parsed = output?.name ? parseName(output.name, region) : undefined;
905 const projectId = parsed?.project || env.project;
906 const desiredLabels = {
907 ...toLabels(news.labels),
908 ...(yield* createInternalLabels(id)),
909 };
910
911 let current = yield* getById(projectId, region, output?.clusterName ?? clusterName);
912
913 if (current?.status?.state === "DELETING") {
914 yield* waitUntilGone(projectId, region, clusterName);
915 current = undefined;
916 }
917
918 if (current === undefined) {
919 yield* createCluster(
920 projectId,
921 region,
922 clusterName,
923 toCreateBody(news, projectId, clusterName, desiredLabels),
924 );
925 current = yield* waitUntilExists(projectId, region, clusterName);
926 }
927
928 if (current === undefined) {
929 return yield* new ClusterNotResolved({ clusterName, region });
930 }
931
932 const state = current.status?.state ?? "";
933 if (terminalError(state)) {
934 return yield* new ClusterFailed({
935 clusterName,
936 state,
937 detail: current.status?.detail,
938 });
939 }
940
941 if (state === "STOPPED" || state === "STOPPING") {
942 const started = yield* dataproc.startProjectsRegionsClusters({
943 projectId,
944 region,
945 clusterName,
946 });
947 yield* waitForOperation(started);
948 }
949
950 if ((current.status?.state ?? "") !== "RUNNING") {
951 current = yield* waitUntilRunning(projectId, region, clusterName);
952 }
953
954 const observedLabels = tagRecord(current.labels);
955 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
956 const labelsChanged = upsert.length > 0 || removed.length > 0;
957
958 const observedWorkers = groupNum(current.config?.workerConfig);
959 const desiredWorkers = news.workerNumInstances;
960 const workersChanged = desiredWorkers !== undefined && observedWorkers !== desiredWorkers;
961
962 const observedSecondary = groupNum(current.config?.secondaryWorkerConfig);
963 const desiredSecondary = news.secondaryWorkerNumInstances;
964 const secondaryChanged =
965 desiredSecondary !== undefined && observedSecondary !== desiredSecondary;
966
967 const observedPolicy = current.config?.autoscalingConfig?.policyUri ?? "";
968 const desiredPolicy = news.autoscalingPolicyUri;
969 const policyChanged = desiredPolicy !== undefined && observedPolicy !== desiredPolicy;
970
971 if (labelsChanged || workersChanged || secondaryChanged || policyChanged) {
972 const updateMask = [
973 labelsChanged ? "labels" : undefined,
974 workersChanged ? "config.worker_config.num_instances" : undefined,
975 secondaryChanged ? "config.secondary_worker_config.num_instances" : undefined,
976 policyChanged ? "config.autoscaling_config.policy_uri" : undefined,
977 ].filter((field): field is string => field !== undefined);
978
979 const patched = yield* dataproc
980 .patchProjectsRegionsClusters({
981 projectId,
982 region,
983 clusterName,
984 updateMask: updateMask.join(","),
985 body: {
986 projectId,
987 clusterName,
988 labels: desiredLabels,
989 config: {
990 workerConfig:
991 desiredWorkers !== undefined ? { numInstances: desiredWorkers } : undefined,
992 secondaryWorkerConfig:
993 desiredSecondary !== undefined ? { numInstances: desiredSecondary } : undefined,
994 autoscalingConfig:
995 desiredPolicy !== undefined ? { policyUri: desiredPolicy } : undefined,
996 },
997 },
998 })
999 .pipe(
1000 Effect.retry({
1001 while: (error) => error._tag === "Conflict",
1002 times: 8,
1003 schedule: Schedule.spaced("5 seconds"),
1004 }),
1005 );
1006 yield* waitForOperation(patched);
1007 current = yield* waitUntilRunning(projectId, region, clusterName);
1008 }
1009
1010 return toAttrs(current, projectId, region);
1011 }),
1012
1013 delete: Effect.fn(function* ({ output }) {
1014 yield* deleteCluster(output.project, output.region, output.clusterName);
1015 }),
1016 });