@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 580 lines 19.4 KB
Raw
1 import * as bigtable from "@distilled.cloud/gcp/bigtableadmin_v2";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import type { Providers } from "../Providers.ts";
11 import {
12 clusterLocation,
13 clusterName,
14 DEFAULT_SERVE_NODES,
15 DEFAULT_ZONE,
16 lastSegment,
17 listAlchemyInstances,
18 MAX_CLUSTER_ID_LENGTH,
19 MIN_CLUSTER_ID_LENGTH,
20 parentOwned,
21 parseResourceName,
22 toPhysicalId,
23 waitForOperation,
24 zoneOf,
25 collectPages,
26 } from "./operations.ts";
27
28 const DEFAULT_STORAGE_TYPE = "SSD";
29
30 export type AutoscalingLimits = {
31 /** Minimum nodes the autoscaler can scale down to. */
32 minServeNodes?: number;
33 /** Maximum nodes the autoscaler can scale up to. */
34 maxServeNodes?: number;
35 };
36
37 export type AutoscalingTargets = {
38 /**
39 * Target CPU utilization percent (`10`–`80`).
40 */
41 cpuUtilizationPercent?: number;
42 /**
43 * Target storage utilization in GiB per node. SSD default `2560`;
44 * HDD default `8192`.
45 */
46 storageUtilizationGibPerNode?: number;
47 };
48
49 export type ClusterAutoscalingConfig = {
50 /** Minimum and maximum node counts. */
51 autoscalingLimits?: AutoscalingLimits;
52 /** CPU and storage utilization targets. */
53 autoscalingTargets?: AutoscalingTargets;
54 };
55
56 export type ClusterConfig = {
57 /** Autoscaling configuration. Mutually exclusive with `serveNodes`. */
58 clusterAutoscalingConfig?: ClusterAutoscalingConfig;
59 };
60
61 export type EncryptionConfig = {
62 /**
63 * Cloud KMS key
64 * `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
65 * Must be in the same region as the cluster. Immutable — changing it
66 * replaces the cluster.
67 */
68 kmsKeyName?: string;
69 };
70
71 export type ClusterResourceProps = {
72 /**
73 * Parent instance id (the `{instance}` segment of
74 * `projects/{project}/instances/{instance}`). Full resource names are
75 * accepted and reduced to the last path segment. Immutable — changing
76 * it replaces the cluster.
77 */
78 instance: string;
79 /**
80 * Cluster id (the `{cluster}` segment of
81 * `projects/{project}/instances/{instance}/clusters/{cluster}`). If
82 * omitted, a unique name is generated from the stack, stage, and
83 * logical id. Must be 6-30 characters and match
84 * `[a-z][-a-z0-9]*`. Immutable — changing it replaces the cluster.
85 */
86 clusterId?: string;
87 /**
88 * Zone where nodes and storage reside (e.g. `us-central1-b`). Full
89 * location names `projects/{project}/locations/{zone}` are accepted.
90 * Immutable — changing it replaces the cluster.
91 * @default "us-central1-b"
92 */
93 location?: string;
94 /**
95 * Storage type used by this cluster (`SSD` or `HDD`). Immutable —
96 * changing it replaces the cluster.
97 * @default "SSD"
98 */
99 defaultStorageType?: bigtable.ClusterDefaultStorageTypeEnum | (string & {});
100 /**
101 * Manual node count. Mutually exclusive with
102 * `clusterConfig.clusterAutoscalingConfig`. Ignored while
103 * autoscaling is enabled.
104 * @default 1
105 */
106 serveNodes?: number;
107 /**
108 * Node scaling factor. Immutable — changing it replaces the cluster.
109 */
110 nodeScalingFactor?: bigtable.ClusterNodeScalingFactorEnum | (string & {});
111 /**
112 * Customer-managed encryption. Immutable — changing it replaces the
113 * cluster.
114 */
115 encryptionConfig?: EncryptionConfig;
116 /**
117 * Cluster configuration. Set `clusterAutoscalingConfig` to enable
118 * autoscaling (and omit `serveNodes`).
119 */
120 clusterConfig?: ClusterConfig;
121 };
122
123 export type Cluster = Resource<
124 "GCP.Bigtable.Cluster",
125 ClusterResourceProps,
126 {
127 /** Full resource name `projects/{project}/instances/{instance}/clusters/{cluster}`. */
128 name: string;
129 /** Cluster id (last path segment). */
130 clusterId: string;
131 /** Parent instance resource name. */
132 instance: string;
133 /** Parent instance id. */
134 instanceId: string;
135 /** Project id. */
136 project: string;
137 /** Zone id (`us-central1-b`, …). */
138 location: string;
139 /** Storage type (`SSD`, `HDD`). */
140 defaultStorageType: string | undefined;
141 /** Current node count. */
142 serveNodes: number | undefined;
143 /** Node scaling factor. */
144 nodeScalingFactor: string | undefined;
145 /** CMEK configuration, if any. */
146 encryptionConfig: EncryptionConfig | undefined;
147 /** Cluster configuration currently applied. */
148 clusterConfig: ClusterConfig | undefined;
149 /** Server-reported state (`READY`, `CREATING`, `RESIZING`, …). */
150 state: string | undefined;
151 },
152 never,
153 Providers
154 >;
155
156 /**
157 * A Cloud Bigtable cluster — a resizable group of nodes in one zone
158 * that serves every table in the parent instance.
159 *
160 * An instance always has at least one cluster (created with the
161 * instance). Use this resource to add additional clusters or to manage
162 * an existing cluster's node count and autoscaling. You cannot delete
163 * the last cluster in an instance.
164 *
165 * Changing `clusterId`, `instance`, `location`, `defaultStorageType`,
166 * `nodeScalingFactor`, or `encryptionConfig.kmsKeyName` replaces the
167 * cluster. `serveNodes` and `clusterConfig.clusterAutoscalingConfig`
168 * update in place.
169 *
170 * Clusters have no labels field. `list` enumerates clusters on
171 * alchemy-labeled instances so `pnpm nuke:gcp` can find leaked rows.
172 *
173 * Provisioning typically takes several minutes.
174 *
175 * ### Creating a Cluster
176 * **Example:** Additional cluster on an existing instance
177 * ```typescript
178 * const replica = yield* GCP.Bigtable.Cluster("Replica", {
179 * instance: instance.instanceId,
180 * location: "us-central1-b",
181 * serveNodes: 1,
182 * });
183 * ```
184 *
185 * **Example:** Explicit id and HDD storage
186 * ```typescript
187 * const replica = yield* GCP.Bigtable.Cluster("Replica", {
188 * instance: "app-instance",
189 * clusterId: "app-replica",
190 * location: "us-central1-c",
191 * defaultStorageType: "HDD",
192 * serveNodes: 1,
193 * });
194 * ```
195 *
196 * ### Autoscaling
197 * **Example:** Autoscaling between 1 and 3 nodes
198 * ```typescript
199 * const replica = yield* GCP.Bigtable.Cluster("Replica", {
200 * instance: "app-instance",
201 * location: "us-central1-f",
202 * clusterConfig: {
203 * clusterAutoscalingConfig: {
204 * autoscalingLimits: { minServeNodes: 1, maxServeNodes: 3 },
205 * autoscalingTargets: { cpuUtilizationPercent: 50 },
206 * },
207 * },
208 * });
209 * ```
210 *
211 * @resource
212 * @category Bigtable
213 */
214 export const Cluster = Resource<Cluster>("GCP.Bigtable.Cluster");
215
216 export class ClusterNotResolved extends Data.TaggedError("GCP.Bigtable.ClusterNotResolved")<{
217 resourceName: string;
218 }> {}
219
220 export class ClusterNotReady extends Data.TaggedError("GCP.Bigtable.ClusterNotReady")<{
221 resourceName: string;
222 state: string;
223 }> {}
224
225 export class ClusterStillExists extends Data.TaggedError("GCP.Bigtable.ClusterStillExists")<{
226 resourceName: string;
227 }> {}
228
229 const instanceIdOf = (value: string) => lastSegment(value);
230
231 const normalizeStorage = (storage: string | undefined) => {
232 const value = (storage ?? DEFAULT_STORAGE_TYPE).toUpperCase();
233 return value === "STORAGE_TYPE_UNSPECIFIED" ? DEFAULT_STORAGE_TYPE : value;
234 };
235
236 const normalizeScaling = (factor: string | undefined) => (factor ?? "").toUpperCase();
237
238 const encryptionOf = (
239 config: bigtable.EncryptionConfig | EncryptionConfig | undefined,
240 ): EncryptionConfig | undefined => {
241 const kmsKeyName = config?.kmsKeyName;
242 if (kmsKeyName === undefined || kmsKeyName.length === 0) return undefined;
243 return { kmsKeyName };
244 };
245
246 const encryptionKey = (config: bigtable.EncryptionConfig | EncryptionConfig | undefined) =>
247 encryptionOf(config)?.kmsKeyName ?? "";
248
249 const autoscalingOf = (
250 config: bigtable.ClusterAutoscalingConfig | ClusterAutoscalingConfig | undefined,
251 ): ClusterAutoscalingConfig | undefined => {
252 if (config === undefined) return undefined;
253 const limits = config.autoscalingLimits;
254 const targets = config.autoscalingTargets;
255 if (limits === undefined && targets === undefined) return undefined;
256 return {
257 autoscalingLimits: limits
258 ? {
259 minServeNodes: limits.minServeNodes,
260 maxServeNodes: limits.maxServeNodes,
261 }
262 : undefined,
263 autoscalingTargets: targets
264 ? {
265 cpuUtilizationPercent: targets.cpuUtilizationPercent,
266 storageUtilizationGibPerNode: targets.storageUtilizationGibPerNode,
267 }
268 : undefined,
269 };
270 };
271
272 const clusterConfigOf = (
273 config: bigtable.ClusterConfig | ClusterConfig | undefined,
274 ): ClusterConfig | undefined => {
275 const autoscaling = autoscalingOf(config?.clusterAutoscalingConfig);
276 if (autoscaling === undefined) return undefined;
277 return { clusterAutoscalingConfig: autoscaling };
278 };
279
280 const autoscalingKey = (config: bigtable.ClusterConfig | ClusterConfig | undefined) => {
281 const autoscaling = clusterConfigOf(config)?.clusterAutoscalingConfig;
282 return JSON.stringify({
283 minServeNodes: autoscaling?.autoscalingLimits?.minServeNodes ?? null,
284 maxServeNodes: autoscaling?.autoscalingLimits?.maxServeNodes ?? null,
285 cpuUtilizationPercent: autoscaling?.autoscalingTargets?.cpuUtilizationPercent ?? null,
286 storageUtilizationGibPerNode:
287 autoscaling?.autoscalingTargets?.storageUtilizationGibPerNode ?? null,
288 });
289 };
290
291 const toAttrs = (cluster: bigtable.Cluster, project: string) => {
292 const name = cluster.name ?? "";
293 const parsed = parseResourceName(name);
294 return {
295 name,
296 clusterId: parsed.clusterId,
297 instance: parsed.instance,
298 instanceId: parsed.instanceId,
299 project: parsed.project || project,
300 location: zoneOf(cluster.location || DEFAULT_ZONE),
301 defaultStorageType: cluster.defaultStorageType,
302 serveNodes: cluster.serveNodes,
303 nodeScalingFactor: cluster.nodeScalingFactor,
304 encryptionConfig: encryptionOf(cluster.encryptionConfig),
305 clusterConfig: clusterConfigOf(cluster.clusterConfig),
306 state: cluster.state,
307 };
308 };
309
310 const getByName = (name: string) =>
311 bigtable
312 .getProjectsInstancesClusters({ name })
313 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
314
315 const isBusy = (state: string | undefined) =>
316 state === "CREATING" ||
317 state === "RESIZING" ||
318 state === "STATE_NOT_KNOWN" ||
319 state === undefined;
320
321 const waitUntilExists = (name: string) =>
322 getByName(name).pipe(
323 Effect.flatMap((cluster) =>
324 cluster
325 ? Effect.succeed(cluster)
326 : Effect.fail(new ClusterNotResolved({ resourceName: name })),
327 ),
328 Effect.retry({
329 while: (error) => error._tag === "GCP.Bigtable.ClusterNotResolved",
330 times: 8,
331 schedule: Schedule.spaced("1 second"),
332 }),
333 );
334
335 const waitUntilReady = (name: string) =>
336 getByName(name).pipe(
337 Effect.filterOrFail(
338 (cluster): cluster is bigtable.Cluster => cluster !== undefined,
339 () => new ClusterNotResolved({ resourceName: name }),
340 ),
341 Effect.filterOrFail(
342 (cluster) => !isBusy(cluster.state),
343 (cluster) =>
344 new ClusterNotReady({
345 resourceName: name,
346 state: cluster.state ?? "STATE_NOT_KNOWN",
347 }),
348 ),
349 Effect.retry({
350 while: (error) =>
351 error._tag === "GCP.Bigtable.ClusterNotReady" ||
352 error._tag === "GCP.Bigtable.ClusterNotResolved",
353 times: 10,
354 schedule: Schedule.spaced("8 seconds"),
355 }),
356 );
357
358 const waitUntilGone = (name: string) =>
359 getByName(name).pipe(
360 Effect.flatMap((cluster) =>
361 cluster === undefined
362 ? Effect.void
363 : Effect.fail(new ClusterStillExists({ resourceName: name })),
364 ),
365 Effect.retry({
366 while: (error) => error._tag === "GCP.Bigtable.ClusterStillExists",
367 times: 10,
368 schedule: Schedule.spaced("8 seconds"),
369 }),
370 );
371
372 const toCreateBody = (
373 news: ClusterResourceProps,
374 project: string,
375 zone: string,
376 storage: string,
377 ): bigtable.Cluster => {
378 const autoscaling = clusterConfigOf(news.clusterConfig);
379 return {
380 location: clusterLocation(project, zone),
381 defaultStorageType: storage,
382 nodeScalingFactor: news.nodeScalingFactor,
383 encryptionConfig: news.encryptionConfig,
384 serveNodes: autoscaling === undefined ? (news.serveNodes ?? DEFAULT_SERVE_NODES) : undefined,
385 clusterConfig: autoscaling,
386 };
387 };
388
389 export const ClusterProvider = () =>
390 Provider.succeed(Cluster, {
391 stables: ["name", "clusterId", "instance", "instanceId", "project", "location"],
392
393 diff: Effect.fn(function* ({ news, olds, output }) {
394 if (!isResolved(news)) return undefined;
395
396 const previousId = olds?.clusterId ?? output?.clusterId;
397 const nextId = news.clusterId ?? previousId;
398 const previousInstance = instanceIdOf(
399 olds?.instance ?? output?.instance ?? output?.instanceId ?? "",
400 );
401 const nextInstance = instanceIdOf(news.instance);
402 const previousZone = zoneOf(olds?.location ?? output?.location);
403 const nextZone = zoneOf(news.location ?? output?.location);
404 const previousStorage = normalizeStorage(
405 olds?.defaultStorageType ?? output?.defaultStorageType,
406 );
407 const nextStorage = normalizeStorage(news.defaultStorageType ?? output?.defaultStorageType);
408 const previousScaling = normalizeScaling(
409 olds?.nodeScalingFactor ?? output?.nodeScalingFactor,
410 );
411 const nextScaling = normalizeScaling(news.nodeScalingFactor ?? output?.nodeScalingFactor);
412 const previousCmek = encryptionKey(olds?.encryptionConfig ?? output?.encryptionConfig);
413 const nextCmek = encryptionKey(news.encryptionConfig ?? output?.encryptionConfig);
414
415 const replace =
416 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
417 (previousInstance.length > 0 &&
418 nextInstance.length > 0 &&
419 previousInstance !== nextInstance) ||
420 previousZone !== nextZone ||
421 previousStorage !== nextStorage ||
422 (previousScaling.length > 0 && nextScaling.length > 0 && previousScaling !== nextScaling) ||
423 previousCmek !== nextCmek;
424
425 if (!replace) return undefined;
426 return {
427 action: "replace" as const,
428 deleteFirst:
429 previousInstance === nextInstance && previousId !== undefined && nextId === previousId,
430 };
431 }),
432
433 read: Effect.fn(function* ({ id, olds, output }) {
434 const env = yield* GcpEnvironment.current;
435 const instanceId = instanceIdOf(
436 olds?.instance ?? output?.instance ?? output?.instanceId ?? "",
437 );
438 if (instanceId.length === 0) return undefined;
439 const clusterId = yield* toPhysicalId(
440 id,
441 olds?.clusterId,
442 output?.clusterId,
443 MAX_CLUSTER_ID_LENGTH,
444 MIN_CLUSTER_ID_LENGTH,
445 );
446 const name = output?.name ?? clusterName(env.project, instanceId, clusterId);
447 const existing = yield* getByName(name);
448 if (existing === undefined) return undefined;
449 const attrs = toAttrs(existing, env.project);
450 return (yield* parentOwned(attrs.instance)) ? attrs : Unowned(attrs);
451 }),
452
453 list: () =>
454 Effect.gen(function* () {
455 const env = yield* GcpEnvironment.current;
456 const instances = yield* listAlchemyInstances(env.project);
457 const labeled = new Set(
458 instances.map((instance) => instance.name ?? "").filter((name) => name.length > 0),
459 );
460 if (labeled.size === 0) return [];
461 const clusters = yield* collectPages(
462 bigtable.listProjectsInstancesClusters.pages({
463 parent: `projects/${env.project}/instances/-`,
464 }),
465 (page) => page.clusters,
466 );
467 return clusters
468 .filter((cluster) => labeled.has(parseResourceName(cluster.name ?? "").instance))
469 .map((cluster) => toAttrs(cluster, env.project));
470 }),
471
472 reconcile: Effect.fn(function* ({ id, news, output }) {
473 const env = yield* GcpEnvironment.current;
474 const instanceId = instanceIdOf(news.instance);
475 const clusterId = yield* toPhysicalId(
476 id,
477 news.clusterId,
478 output?.clusterId,
479 MAX_CLUSTER_ID_LENGTH,
480 MIN_CLUSTER_ID_LENGTH,
481 );
482 const zone = zoneOf(news.location ?? output?.location);
483 const storage = normalizeStorage(news.defaultStorageType);
484 const name = clusterName(env.project, instanceId, clusterId);
485 const desiredAutoscaling = clusterConfigOf(news.clusterConfig);
486
487 let current = yield* getByName(output?.name ?? name);
488
489 if (current === undefined) {
490 const created = yield* bigtable
491 .createProjectsInstancesClusters({
492 parent: `projects/${env.project}/instances/${instanceId}`,
493 clusterId,
494 body: toCreateBody(news, env.project, zone, storage),
495 })
496 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
497 if (created !== undefined) {
498 yield* waitForOperation(created, { alreadyExistsOk: true });
499 }
500 current = yield* waitUntilExists(name);
501 }
502
503 if (isBusy(current.state)) {
504 current = yield* waitUntilReady(name);
505 }
506
507 const observedAutoscaling = clusterConfigOf(current.clusterConfig);
508 const autoscalingChanged =
509 desiredAutoscaling !== undefined &&
510 autoscalingKey(current.clusterConfig) !== autoscalingKey(news.clusterConfig);
511 const disableAutoscaling =
512 desiredAutoscaling === undefined &&
513 news.serveNodes !== undefined &&
514 observedAutoscaling !== undefined;
515 const serveNodesChanged =
516 desiredAutoscaling === undefined &&
517 news.serveNodes !== undefined &&
518 (current.serveNodes ?? DEFAULT_SERVE_NODES) !== news.serveNodes;
519
520 if (autoscalingChanged) {
521 const patched = yield* bigtable.partialUpdateClusterProjectsInstancesClusters({
522 name,
523 updateMask: "cluster_config.cluster_autoscaling_config",
524 body: {
525 name,
526 clusterConfig: desiredAutoscaling,
527 },
528 });
529 yield* waitForOperation(patched);
530 current = yield* waitUntilReady(name);
531 } else if (disableAutoscaling || serveNodesChanged) {
532 const serveNodes = news.serveNodes ?? current.serveNodes ?? DEFAULT_SERVE_NODES;
533 if (disableAutoscaling) {
534 const patched = yield* bigtable.partialUpdateClusterProjectsInstancesClusters({
535 name,
536 updateMask: "serve_nodes,cluster_config.cluster_autoscaling_config",
537 body: {
538 name,
539 serveNodes,
540 clusterConfig: {},
541 },
542 });
543 yield* waitForOperation(patched);
544 } else {
545 const updated = yield* bigtable.updateProjectsInstancesClusters({
546 name,
547 body: {
548 name,
549 location: current.location ?? clusterLocation(env.project, zone),
550 serveNodes,
551 defaultStorageType: current.defaultStorageType ?? storage,
552 },
553 });
554 yield* waitForOperation(updated);
555 }
556 current = yield* waitUntilReady(name);
557 }
558
559 return toAttrs(current, env.project);
560 }),
561
562 delete: Effect.fn(function* ({ output }) {
563 const result = yield* bigtable.deleteProjectsInstancesClusters({ name: output.name }).pipe(
564 Effect.retry({
565 while: (error) => error._tag === "Conflict",
566 times: 8,
567 schedule: Schedule.spaced("5 seconds"),
568 }),
569 Effect.as("deleted" as const),
570 Effect.catchTags({
571 NotFound: () => Effect.succeed("deleted" as const),
572 // The instance's last cluster goes away with the instance.
573 LastClusterDeletion: () => Effect.succeed("kept" as const),
574 }),
575 );
576 if (result === "deleted") {
577 yield* waitUntilGone(output.name);
578 }
579 }),
580 });