The best way to control Claude Code, Codex and more from any computer or phone

main
ts 500 lines 17.1 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 expandParent,
16 fingerprint,
17 hasAlchemyLabelMap,
18 lastSegment,
19 normalizeLocation,
20 parseResourceName,
21 specifiedEquals,
22 toPhysicalSnake,
23 userLabels,
24 } from "./helpers.ts";
25 import { listLocations } from "./names.ts";
26 import { waitForOperation } from "./operations.ts";
27
28 const MAX_NAME_LENGTH = 60;
29
30 export type FeatureViewBigQuerySource = {
31 /** BigQuery view URI materialized on each sync. */
32 uri?: string;
33 /** Columns used as entity ids. */
34 entityIdColumns?: string[];
35 };
36
37 export type FeatureViewFeatureRegistrySource = {
38 /** Feature groups and feature ids to sync. */
39 featureGroups?: Array<{
40 featureGroupId?: string;
41 featureIds?: string[];
42 }>;
43 /** Parent project number of the Feature Groups. */
44 projectNumber?: string;
45 };
46
47 export type FeatureViewVertexRagSource = {
48 /** BigQuery view/table URI. */
49 uri?: string;
50 /** RAG corpus id. */
51 ragCorpusId?: string;
52 };
53
54 export type FeatureViewSyncConfig = {
55 /** Cron schedule. */
56 cron?: string;
57 /** Continuous sync. */
58 continuous?: boolean;
59 };
60
61 export type FeatureViewIndexConfig = {
62 embeddingColumn?: string;
63 embeddingDimension?: number;
64 crowdingColumn?: string;
65 filterColumns?: string[];
66 distanceMeasureType?:
67 | aiplatform.GoogleCloudAiplatformV1FeatureViewIndexConfigDistanceMeasureTypeEnum
68 | (string & {});
69 treeAhConfig?: { leafNodeEmbeddingCount?: string };
70 bruteForceConfig?: Record<string, never>;
71 };
72
73 export type FeatureOnlineStoresFeatureViewProps = {
74 /**
75 * Parent Feature Online Store resource name or id. Immutable.
76 */
77 featureOnlineStore: string;
78 /**
79 * Feature View id. Valid characters `[a-z0-9_]`, must start with a
80 * letter, max 60. Immutable.
81 */
82 featureViewId?: string;
83 /**
84 * Region used when `featureOnlineStore` is a bare id.
85 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
86 */
87 location?: string;
88 /**
89 * User labels. Alchemy ownership labels are merged in automatically.
90 */
91 labels?: Record<string, string>;
92 /**
93 * BigQuery source. Mutually exclusive with `featureRegistrySource`
94 * and `vertexRagSource`.
95 */
96 bigQuerySource?: FeatureViewBigQuerySource;
97 /**
98 * Feature Registry source.
99 */
100 featureRegistrySource?: FeatureViewFeatureRegistrySource;
101 /**
102 * Vertex RAG source.
103 */
104 vertexRagSource?: FeatureViewVertexRagSource;
105 /**
106 * Sync schedule.
107 */
108 syncConfig?: FeatureViewSyncConfig;
109 /**
110 * Vector index config.
111 */
112 indexConfig?: FeatureViewIndexConfig;
113 /**
114 * Optimized-store replica config.
115 */
116 optimizedConfig?: {
117 automaticResources?: { minReplicaCount?: number; maxReplicaCount?: number };
118 };
119 /**
120 * Service agent type used during data sync.
121 */
122 serviceAgentType?:
123 | aiplatform.GoogleCloudAiplatformV1FeatureViewServiceAgentTypeEnum
124 | (string & {});
125 /**
126 * Run an on-demand sync immediately at create.
127 * @default false
128 */
129 runSyncImmediately?: boolean;
130 };
131
132 export type FeatureOnlineStoresFeatureView = Resource<
133 "GCP.AIPlatform.FeatureOnlineStoresFeatureView",
134 FeatureOnlineStoresFeatureViewProps,
135 {
136 /** Full resource name. */
137 name: string;
138 /** Feature view id. */
139 featureViewId: string;
140 /** Parent online store resource name. */
141 featureOnlineStore: string;
142 /** Parent online store id. */
143 featureOnlineStoreId: string;
144 /** Project id. */
145 project: string;
146 /** Location id. */
147 location: string;
148 /** User labels (Alchemy ownership labels stripped). */
149 labels: Record<string, string>;
150 /** BigQuery source. */
151 bigQuerySource: aiplatform.GoogleCloudAiplatformV1FeatureViewBigQuerySource | undefined;
152 /** Feature Registry source. */
153 featureRegistrySource:
154 | aiplatform.GoogleCloudAiplatformV1FeatureViewFeatureRegistrySource
155 | undefined;
156 /** Vertex RAG source. */
157 vertexRagSource: aiplatform.GoogleCloudAiplatformV1FeatureViewVertexRagSource | undefined;
158 /** Sync config. */
159 syncConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewSyncConfig | undefined;
160 /** Vector index config. */
161 indexConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewIndexConfig | undefined;
162 /** Optimized replica config. */
163 optimizedConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewOptimizedConfig | undefined;
164 /** Service agent type. */
165 serviceAgentType:
166 | aiplatform.GoogleCloudAiplatformV1FeatureViewServiceAgentTypeEnum
167 | (string & {})
168 | undefined;
169 /** Unique service account email. */
170 serviceAccountEmail: string | undefined;
171 /** RFC3339 creation timestamp. */
172 createTime: string | undefined;
173 /** RFC3339 last-update timestamp. */
174 updateTime: string | undefined;
175 /** Server etag. */
176 etag: string | undefined;
177 },
178 never,
179 Providers
180 >;
181
182 /**
183 * A Feature View — the serving projection of a Feature Online Store.
184 *
185 * Parent store, view id, and location are identity. Labels, sources, sync
186 * schedule, and optimized replica counts update in place.
187 *
188 * ### Creating a Feature View
189 * **Example:** Registry-backed view
190 * ```typescript
191 * const view = yield* GCP.AIPlatform.FeatureOnlineStoresFeatureView("Users", {
192 * featureOnlineStore: store.name,
193 * featureRegistrySource: {
194 * featureGroups: [{ featureGroupId: group.featureGroupId, featureIds: ["age"] }],
195 * },
196 * syncConfig: { cron: "0 * * * *" },
197 * });
198 * ```
199 *
200 * @resource
201 * @category AIPlatform
202 */
203 export const FeatureOnlineStoresFeatureView = Resource<FeatureOnlineStoresFeatureView>(
204 "GCP.AIPlatform.FeatureOnlineStoresFeatureView",
205 );
206
207 export class FeatureOnlineStoresFeatureViewNotResolved extends Data.TaggedError(
208 "GCP.AIPlatform.FeatureOnlineStoresFeatureViewNotResolved",
209 )<{
210 resourceName: string;
211 }> {}
212
213 export class FeatureOnlineStoresFeatureViewStillExists extends Data.TaggedError(
214 "GCP.AIPlatform.FeatureOnlineStoresFeatureViewStillExists",
215 )<{
216 resourceName: string;
217 }> {}
218
219 const parentOf = (project: string, location: string, store: string) =>
220 expandParent(store, project, location, "featureOnlineStores");
221
222 const resourceName = (parent: string, viewId: string) => `${parent}/featureViews/${viewId}`;
223
224 const toAttrs = (view: aiplatform.GoogleCloudAiplatformV1FeatureView, project: string) => {
225 const name = view.name ?? "";
226 const parsed = parseResourceName(name, "featureViews");
227 const store = parseResourceName(parsed.parent, "featureOnlineStores");
228 return {
229 name,
230 featureViewId: parsed.id,
231 featureOnlineStore: parsed.parent,
232 featureOnlineStoreId: store.id,
233 project: parsed.project || project,
234 location: parsed.location,
235 labels: userLabels(view.labels),
236 bigQuerySource: view.bigQuerySource,
237 featureRegistrySource: view.featureRegistrySource,
238 vertexRagSource: view.vertexRagSource,
239 syncConfig: view.syncConfig,
240 indexConfig: view.indexConfig,
241 optimizedConfig: view.optimizedConfig,
242 serviceAgentType: view.serviceAgentType,
243 serviceAccountEmail: view.serviceAccountEmail,
244 createTime: view.createTime,
245 updateTime: view.updateTime,
246 etag: view.etag,
247 };
248 };
249
250 const getByName = (name: string) =>
251 aiplatform
252 .getProjectsLocationsFeatureOnlineStoresFeatureViews({ name })
253 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
254
255 const waitUntilExists = (name: string) =>
256 getByName(name).pipe(
257 Effect.flatMap((view) =>
258 view
259 ? Effect.succeed(view)
260 : Effect.fail(new FeatureOnlineStoresFeatureViewNotResolved({ resourceName: name })),
261 ),
262 Effect.retry({
263 while: (error) => error._tag === "GCP.AIPlatform.FeatureOnlineStoresFeatureViewNotResolved",
264 times: 8,
265 schedule: Schedule.spaced("1 second"),
266 }),
267 );
268
269 const waitUntilGone = (name: string) =>
270 getByName(name).pipe(
271 Effect.flatMap((view) =>
272 view === undefined
273 ? Effect.void
274 : Effect.fail(new FeatureOnlineStoresFeatureViewStillExists({ resourceName: name })),
275 ),
276 Effect.retry({
277 while: (error) => error._tag === "GCP.AIPlatform.FeatureOnlineStoresFeatureViewStillExists",
278 times: 10,
279 schedule: Schedule.spaced("2 seconds"),
280 }),
281 );
282
283 const listViewsUnder = (parent: string, project: string) =>
284 aiplatform.listProjectsLocationsFeatureOnlineStoresFeatureViews
285 .pages({ parent, pageSize: 100 })
286 .pipe(
287 Stream.flatMap((page) => Stream.fromIterable(page.featureViews ?? [])),
288 Stream.filter((view) => hasAlchemyLabelMap(view.labels)),
289 Stream.map((view) => toAttrs(view, project)),
290 Stream.runCollect,
291 Effect.map((chunk) => Array.from(chunk)),
292 Effect.catchTag("NotFound", () => Effect.succeed([])),
293 );
294
295 export const FeatureOnlineStoresFeatureViewProvider = () =>
296 Provider.succeed(FeatureOnlineStoresFeatureView, {
297 stables: [
298 "name",
299 "featureViewId",
300 "featureOnlineStore",
301 "featureOnlineStoreId",
302 "project",
303 "location",
304 "createTime",
305 ],
306
307 diff: Effect.fn(function* ({ news, olds, output }) {
308 const env = yield* GcpEnvironment.current;
309 if (!isResolved(news)) return undefined;
310 const previousId = olds?.featureViewId ?? output?.featureViewId;
311 const nextId = news.featureViewId ?? previousId;
312 const previousParent = olds?.featureOnlineStore ?? output?.featureOnlineStore ?? "";
313 const nextParent = news.featureOnlineStore ?? previousParent;
314 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
315 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
316 const parentChanged =
317 previousParent.length > 0 && lastSegment(nextParent) !== lastSegment(previousParent);
318 const replace =
319 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
320 parentChanged ||
321 previousLocation !== nextLocation;
322 if (!replace) return undefined;
323 return { action: "replace" as const, deleteFirst: false };
324 }),
325
326 read: Effect.fn(function* ({ id, olds, output }) {
327 const env = yield* GcpEnvironment.current;
328 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
329 const parentRef = olds?.featureOnlineStore ?? output?.featureOnlineStore;
330 // A create interrupted before its parent resolved has nothing to find.
331 if (output?.name === undefined && typeof parentRef !== "string") {
332 return undefined;
333 }
334 const parent = parentOf(env.project, location, parentRef ?? "");
335 const viewId = yield* toPhysicalSnake(
336 id,
337 olds?.featureViewId,
338 output?.featureViewId,
339 MAX_NAME_LENGTH,
340 );
341 const name = output?.name ?? resourceName(parent, viewId);
342 const existing = yield* getByName(name);
343 if (existing === undefined) return undefined;
344 const attrs = toAttrs(existing, env.project);
345 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
346 }),
347
348 list: () =>
349 Effect.gen(function* () {
350 const env = yield* GcpEnvironment.current;
351 const stores = yield* Stream.fromIterable(listLocations(env.region)).pipe(
352 Stream.flatMap((location) =>
353 aiplatform.listProjectsLocationsFeatureOnlineStores.pages({
354 parent: `projects/${env.project}/locations/${location}`,
355 pageSize: 100,
356 }),
357 ),
358 Stream.flatMap((page) => Stream.fromIterable(page.featureOnlineStores ?? [])),
359 Stream.filter((store) => hasAlchemyLabelMap(store.labels)),
360 Stream.runCollect,
361 Effect.map((chunk) => Array.from(chunk)),
362 Effect.catchTag("NotFound", () => Effect.succeed([])),
363 );
364 const nested = yield* Effect.forEach(
365 stores,
366 (store) => (store.name ? listViewsUnder(store.name, env.project) : Effect.succeed([])),
367 { concurrency: 4 },
368 );
369 return nested.flat();
370 }),
371
372 reconcile: Effect.fn(function* ({ id, news, output }) {
373 const env = yield* GcpEnvironment.current;
374 const location = normalizeLocation(news.location ?? output?.location, env.region);
375 const parent = parentOf(env.project, location, news.featureOnlineStore);
376 const viewId = yield* toPhysicalSnake(
377 id,
378 news.featureViewId,
379 output?.featureViewId,
380 MAX_NAME_LENGTH,
381 );
382 const name = resourceName(parent, viewId);
383 const desiredLabels = {
384 ...toLabels(news.labels),
385 ...(yield* createInternalLabels(id)),
386 };
387
388 let current = yield* getByName(output?.name ?? name);
389
390 if (current === undefined) {
391 const created = yield* aiplatform
392 .createProjectsLocationsFeatureOnlineStoresFeatureViews({
393 parent,
394 featureViewId: viewId,
395 runSyncImmediately: news.runSyncImmediately === true,
396 body: {
397 labels: desiredLabels,
398 bigQuerySource: news.bigQuerySource,
399 featureRegistrySource: news.featureRegistrySource,
400 vertexRagSource: news.vertexRagSource,
401 syncConfig: news.syncConfig,
402 indexConfig: news.indexConfig,
403 optimizedConfig: news.optimizedConfig,
404 serviceAgentType: news.serviceAgentType,
405 },
406 })
407 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
408 if (created !== undefined) {
409 yield* waitForOperation(created);
410 }
411 current = yield* waitUntilExists(name);
412 }
413
414 if (current === undefined) {
415 return yield* new FeatureOnlineStoresFeatureViewNotResolved({ resourceName: name });
416 }
417
418 const observedLabels = tagRecord(current.labels);
419 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
420 const labelsChanged = upsert.length > 0 || removed.length > 0;
421 const bqChanged =
422 news.bigQuerySource !== undefined &&
423 !specifiedEquals(news.bigQuerySource, current.bigQuerySource);
424 const registryChanged =
425 news.featureRegistrySource !== undefined &&
426 !specifiedEquals(news.featureRegistrySource, current.featureRegistrySource);
427 const syncChanged =
428 news.syncConfig !== undefined &&
429 fingerprint(current.syncConfig) !== fingerprint(news.syncConfig);
430 const optimizedChanged =
431 news.optimizedConfig !== undefined &&
432 !specifiedEquals(news.optimizedConfig, current.optimizedConfig);
433 const agentChanged =
434 news.serviceAgentType !== undefined &&
435 (current.serviceAgentType ?? "") !== news.serviceAgentType;
436
437 if (
438 labelsChanged ||
439 bqChanged ||
440 registryChanged ||
441 syncChanged ||
442 optimizedChanged ||
443 agentChanged
444 ) {
445 const updateMask = [
446 labelsChanged ? "labels" : undefined,
447 bqChanged ? "big_query_source" : undefined,
448 registryChanged ? "feature_registry_source" : undefined,
449 syncChanged ? "sync_config" : undefined,
450 optimizedChanged ? "optimized_config.automatic_resources" : undefined,
451 agentChanged ? "service_agent_type" : undefined,
452 ].filter((field): field is string => field !== undefined);
453
454 const patched = yield* aiplatform
455 .patchProjectsLocationsFeatureOnlineStoresFeatureViews({
456 name,
457 updateMask: updateMask.join(","),
458 body: {
459 name,
460 labels: desiredLabels,
461 bigQuerySource: news.bigQuerySource,
462 featureRegistrySource: news.featureRegistrySource,
463 syncConfig: news.syncConfig,
464 optimizedConfig: news.optimizedConfig,
465 serviceAgentType: news.serviceAgentType,
466 },
467 })
468 .pipe(
469 Effect.retry({
470 while: (error) => error._tag === "Conflict",
471 times: 8,
472 schedule: Schedule.spaced("5 seconds"),
473 }),
474 );
475 yield* waitForOperation(patched);
476 current = yield* waitUntilExists(name);
477 }
478
479 return toAttrs(current, env.project);
480 }),
481
482 delete: Effect.fn(function* ({ output }) {
483 const operation = yield* aiplatform
484 .deleteProjectsLocationsFeatureOnlineStoresFeatureViews({
485 name: output.name,
486 })
487 .pipe(
488 Effect.catchTag("NotFound", () => Effect.undefined),
489 Effect.retry({
490 while: (error) => error._tag === "Conflict",
491 times: 8,
492 schedule: Schedule.spaced("5 seconds"),
493 }),
494 );
495 if (operation !== undefined) {
496 yield* waitForOperation(operation, { notFoundOk: true });
497 }
498 yield* waitUntilGone(output.name);
499 }),
500 });