@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 410 lines 13.6 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 hasAlchemyLabelMap,
16 normalizeLocation,
17 parseResourceName,
18 specifiedEquals,
19 toPhysicalSnake,
20 userLabels,
21 } from "./helpers.ts";
22 import { listLocations } from "./names.ts";
23 import { waitForOperation } from "./operations.ts";
24
25 const MAX_NAME_LENGTH = 60;
26
27 export type FeaturestoreOnlineServingScaling = {
28 minNodeCount?: number;
29 maxNodeCount?: number;
30 cpuUtilizationTarget?: number;
31 };
32
33 export type FeaturestoreOnlineServingConfig = {
34 /** Fixed node count. `0` disables online serving. */
35 fixedNodeCount?: number;
36 /** Autoscaling. Mutually exclusive with `fixedNodeCount`. */
37 scaling?: FeaturestoreOnlineServingScaling;
38 };
39
40 export type FeaturestoreProps = {
41 /**
42 * Featurestore id. Valid characters `[a-z0-9_]`, must start with a
43 * letter, max 60. Immutable — changing it replaces the store.
44 */
45 featurestoreId?: string;
46 /**
47 * Region. Immutable — changing it replaces the store.
48 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
49 */
50 location?: string;
51 /**
52 * User labels. Alchemy ownership labels are merged in automatically.
53 */
54 labels?: Record<string, string>;
55 /**
56 * Online serving resources. Omit for an offline-only store.
57 */
58 onlineServingConfig?: FeaturestoreOnlineServingConfig;
59 /**
60 * TTL in days for online storage. Defaults to 4000.
61 */
62 onlineStorageTtlDays?: number;
63 /**
64 * Customer-managed encryption. Immutable.
65 */
66 encryptionSpec?: { kmsKeyName?: string };
67 };
68
69 export type Featurestore = Resource<
70 "GCP.AIPlatform.Featurestore",
71 FeaturestoreProps,
72 {
73 /** Full resource name. */
74 name: string;
75 /** Featurestore id. */
76 featurestoreId: string;
77 /** Project id. */
78 project: string;
79 /** Location id. */
80 location: string;
81 /** User labels (Alchemy ownership labels stripped). */
82 labels: Record<string, string>;
83 /** Online serving config. */
84 onlineServingConfig: FeaturestoreOnlineServingConfig | undefined;
85 /** Online storage TTL in days. */
86 onlineStorageTtlDays: number | undefined;
87 /** Store state (`STABLE`, `UPDATING`). */
88 state: string | undefined;
89 /** Customer-managed encryption key. */
90 encryptionSpec: { kmsKeyName?: string } | undefined;
91 /** RFC3339 creation timestamp. */
92 createTime: string | undefined;
93 /** RFC3339 last-update timestamp. */
94 updateTime: string | undefined;
95 /** Server etag. */
96 etag: string | undefined;
97 },
98 never,
99 Providers
100 >;
101
102 /**
103 * A Vertex AI Feature Store (legacy) — a container for Entity Types and
104 * Features.
105 *
106 * Id, location, and encryption are identity. Labels, online serving, and
107 * online TTL update in place. Omit `onlineServingConfig` for an
108 * offline-only store.
109 *
110 * ### Creating a Featurestore
111 * **Example:** Offline-only store
112 * ```typescript
113 * const store = yield* GCP.AIPlatform.Featurestore("Features", {
114 * labels: { env: "prod" },
115 * });
116 * ```
117 *
118 * **Example:** Online serving with a fixed node
119 * ```typescript
120 * const store = yield* GCP.AIPlatform.Featurestore("Features", {
121 * onlineServingConfig: { fixedNodeCount: 1 },
122 * });
123 * ```
124 *
125 * @resource
126 * @category AIPlatform
127 */
128 export const Featurestore = Resource<Featurestore>("GCP.AIPlatform.Featurestore");
129
130 export class FeaturestoreNotResolved extends Data.TaggedError(
131 "GCP.AIPlatform.FeaturestoreNotResolved",
132 )<{
133 resourceName: string;
134 }> {}
135
136 export class FeaturestoreStillExists extends Data.TaggedError(
137 "GCP.AIPlatform.FeaturestoreStillExists",
138 )<{
139 resourceName: string;
140 }> {}
141
142 const resourceName = (project: string, location: string, storeId: string) =>
143 `projects/${project}/locations/${location}/featurestores/${storeId}`;
144
145 const toServing = (
146 config:
147 | aiplatform.GoogleCloudAiplatformV1FeaturestoreOnlineServingConfig
148 | FeaturestoreOnlineServingConfig
149 | undefined,
150 ): FeaturestoreOnlineServingConfig | undefined => {
151 if (config === undefined) return undefined;
152 return {
153 fixedNodeCount: config.fixedNodeCount,
154 scaling: config.scaling
155 ? {
156 minNodeCount: config.scaling.minNodeCount,
157 maxNodeCount: config.scaling.maxNodeCount,
158 cpuUtilizationTarget: config.scaling.cpuUtilizationTarget,
159 }
160 : undefined,
161 };
162 };
163
164 const toAttrs = (store: aiplatform.GoogleCloudAiplatformV1Featurestore, project: string) => {
165 const name = store.name ?? "";
166 const parsed = parseResourceName(name, "featurestores");
167 return {
168 name,
169 featurestoreId: parsed.id,
170 project: parsed.project || project,
171 location: parsed.location,
172 labels: userLabels(store.labels),
173 onlineServingConfig: toServing(store.onlineServingConfig),
174 onlineStorageTtlDays: store.onlineStorageTtlDays,
175 state: store.state,
176 encryptionSpec: store.encryptionSpec?.kmsKeyName
177 ? { kmsKeyName: store.encryptionSpec.kmsKeyName }
178 : undefined,
179 createTime: store.createTime,
180 updateTime: store.updateTime,
181 etag: store.etag,
182 };
183 };
184
185 const getByName = (name: string) =>
186 aiplatform
187 .getProjectsLocationsFeaturestores({ name })
188 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
189
190 const waitUntilExists = (name: string) =>
191 getByName(name).pipe(
192 Effect.flatMap((store) =>
193 store
194 ? Effect.succeed(store)
195 : Effect.fail(new FeaturestoreNotResolved({ resourceName: name })),
196 ),
197 Effect.retry({
198 while: (error) => error._tag === "GCP.AIPlatform.FeaturestoreNotResolved",
199 times: 8,
200 schedule: Schedule.spaced("1 second"),
201 }),
202 );
203
204 const waitUntilGone = (name: string) =>
205 getByName(name).pipe(
206 Effect.flatMap((store) =>
207 store === undefined
208 ? Effect.void
209 : Effect.fail(new FeaturestoreStillExists({ resourceName: name })),
210 ),
211 Effect.retry({
212 while: (error) => error._tag === "GCP.AIPlatform.FeaturestoreStillExists",
213 times: 10,
214 schedule: Schedule.spaced("2 seconds"),
215 }),
216 );
217
218 const isReady = (state: string | undefined) => {
219 const value = (state ?? "").toUpperCase();
220 return value === "STABLE" || value === "";
221 };
222
223 const waitUntilReady = (name: string) =>
224 getByName(name).pipe(
225 Effect.filterOrFail(
226 (store): store is aiplatform.GoogleCloudAiplatformV1Featurestore => store !== undefined,
227 () => new FeaturestoreNotResolved({ resourceName: name }),
228 ),
229 Effect.filterOrFail(
230 (store) => isReady(store.state),
231 () => new FeaturestoreNotResolved({ resourceName: name }),
232 ),
233 Effect.retry({
234 while: (error) => error._tag === "GCP.AIPlatform.FeaturestoreNotResolved",
235 times: 10,
236 schedule: Schedule.spaced("8 seconds"),
237 }),
238 );
239
240 export const FeaturestoreProvider = () =>
241 Provider.succeed(Featurestore, {
242 stables: ["name", "featurestoreId", "project", "location", "createTime"],
243
244 diff: Effect.fn(function* ({ news, olds, output }) {
245 const env = yield* GcpEnvironment.current;
246 if (!isResolved(news)) return undefined;
247 const previousId = olds?.featurestoreId ?? output?.featurestoreId;
248 const nextId = news.featurestoreId ?? previousId;
249 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
250 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
251 const previousKey =
252 olds?.encryptionSpec?.kmsKeyName ?? output?.encryptionSpec?.kmsKeyName ?? "";
253 const nextKey = news.encryptionSpec?.kmsKeyName ?? previousKey;
254 const replace =
255 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
256 previousLocation !== nextLocation ||
257 previousKey !== nextKey;
258 if (!replace) return undefined;
259 return {
260 action: "replace" as const,
261 deleteFirst:
262 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
263 };
264 }),
265
266 read: Effect.fn(function* ({ id, olds, output }) {
267 const env = yield* GcpEnvironment.current;
268 const storeId = yield* toPhysicalSnake(
269 id,
270 olds?.featurestoreId,
271 output?.featurestoreId,
272 MAX_NAME_LENGTH,
273 );
274 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
275 const name = output?.name ?? resourceName(env.project, location, storeId);
276 const existing = yield* getByName(name);
277 if (existing === undefined) return undefined;
278 const attrs = toAttrs(existing, env.project);
279 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
280 }),
281
282 list: () =>
283 Effect.gen(function* () {
284 const env = yield* GcpEnvironment.current;
285 return yield* Stream.fromIterable(listLocations(env.region)).pipe(
286 Stream.flatMap((location) =>
287 aiplatform.listProjectsLocationsFeaturestores.pages({
288 parent: `projects/${env.project}/locations/${location}`,
289 pageSize: 100,
290 }),
291 ),
292 Stream.flatMap((page) => Stream.fromIterable(page.featurestores ?? [])),
293 Stream.filter((store) => hasAlchemyLabelMap(store.labels)),
294 Stream.map((store) => toAttrs(store, env.project)),
295 Stream.runCollect,
296 Effect.map((chunk) => Array.from(chunk)),
297 Effect.catchTag("NotFound", () => Effect.succeed([])),
298 );
299 }),
300
301 reconcile: Effect.fn(function* ({ id, news, output }) {
302 const env = yield* GcpEnvironment.current;
303 const storeId = yield* toPhysicalSnake(
304 id,
305 news.featurestoreId,
306 output?.featurestoreId,
307 MAX_NAME_LENGTH,
308 );
309 const location = normalizeLocation(news.location ?? output?.location, env.region);
310 const name = resourceName(env.project, location, storeId);
311 const desiredLabels = {
312 ...toLabels(news.labels),
313 ...(yield* createInternalLabels(id)),
314 };
315
316 let current = yield* getByName(output?.name ?? name);
317
318 if (current === undefined) {
319 const created = yield* aiplatform
320 .createProjectsLocationsFeaturestores({
321 parent: `projects/${env.project}/locations/${location}`,
322 featurestoreId: storeId,
323 body: {
324 labels: desiredLabels,
325 onlineServingConfig: news.onlineServingConfig,
326 onlineStorageTtlDays: news.onlineStorageTtlDays,
327 encryptionSpec: news.encryptionSpec,
328 },
329 })
330 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
331 if (created !== undefined) {
332 yield* waitForOperation(created);
333 }
334 current = yield* waitUntilExists(name);
335 }
336
337 if (current === undefined) {
338 return yield* new FeaturestoreNotResolved({ resourceName: name });
339 }
340
341 if (!isReady(current.state)) {
342 current = yield* waitUntilReady(name);
343 }
344
345 const observedLabels = tagRecord(current.labels);
346 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
347 const labelsChanged = upsert.length > 0 || removed.length > 0;
348 const servingChanged =
349 news.onlineServingConfig !== undefined &&
350 !specifiedEquals(news.onlineServingConfig, toServing(current.onlineServingConfig));
351 const ttlChanged =
352 news.onlineStorageTtlDays !== undefined &&
353 (current.onlineStorageTtlDays ?? 0) !== news.onlineStorageTtlDays;
354
355 if (labelsChanged || servingChanged || ttlChanged) {
356 const updateMask = [
357 labelsChanged ? "labels" : undefined,
358 servingChanged && news.onlineServingConfig?.scaling
359 ? "online_serving_config.scaling"
360 : servingChanged
361 ? "online_serving_config.fixed_node_count"
362 : undefined,
363 ttlChanged ? "online_storage_ttl_days" : undefined,
364 ].filter((field): field is string => field !== undefined);
365
366 const patched = yield* aiplatform
367 .patchProjectsLocationsFeaturestores({
368 name,
369 updateMask: updateMask.join(","),
370 body: {
371 name,
372 labels: desiredLabels,
373 onlineServingConfig: news.onlineServingConfig,
374 onlineStorageTtlDays: news.onlineStorageTtlDays,
375 },
376 })
377 .pipe(
378 Effect.retry({
379 while: (error) => error._tag === "Conflict",
380 times: 8,
381 schedule: Schedule.spaced("5 seconds"),
382 }),
383 );
384 yield* waitForOperation(patched);
385 current = yield* waitUntilReady(name);
386 }
387
388 return toAttrs(current, env.project);
389 }),
390
391 delete: Effect.fn(function* ({ output }) {
392 const operation = yield* aiplatform
393 .deleteProjectsLocationsFeaturestores({
394 name: output.name,
395 force: true,
396 })
397 .pipe(
398 Effect.catchTag("NotFound", () => Effect.undefined),
399 Effect.retry({
400 while: (error) => error._tag === "Conflict",
401 times: 8,
402 schedule: Schedule.spaced("5 seconds"),
403 }),
404 );
405 if (operation !== undefined) {
406 yield* waitForOperation(operation, { notFoundOk: true });
407 }
408 yield* waitUntilGone(output.name);
409 }),
410 });