The best way to control Claude Code, Codex and more from any computer or phone

main
ts 429 lines 14.5 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 fingerprint,
16 hasAlchemyLabelMap,
17 normalizeLocation,
18 parseResourceName,
19 specifiedEquals,
20 toPhysicalSnake,
21 userLabels,
22 } from "./helpers.ts";
23 import { listLocations } from "./names.ts";
24 import { waitForOperation } from "./operations.ts";
25
26 const MAX_NAME_LENGTH = 60;
27
28 export type FeatureGroupBigQueryTimeSeries = {
29 /** Timestamp column. Defaults to `feature_timestamp`. */
30 timestampColumn?: string;
31 };
32
33 export type FeatureGroupBigQuery = {
34 /**
35 * BigQuery table or view URI (`bq://project.dataset.table`). Immutable.
36 */
37 inputUri?: string;
38 /** Convenience alias for `bigQuerySource.inputUri`. Immutable. */
39 bigQuerySource?: { inputUri?: string };
40 /** Treat the source as a single snapshot rather than a time series. */
41 staticDataSource?: boolean;
42 /** Collapse rows per entity including nulls. */
43 dense?: boolean;
44 /**
45 * Columns used as entity ids. Defaults to `entity_id`.
46 */
47 entityIdColumns?: string[];
48 /** Time-series options. */
49 timeSeries?: FeatureGroupBigQueryTimeSeries;
50 };
51
52 export type FeatureGroupProps = {
53 /**
54 * Feature group id. Valid characters `[a-z0-9_]`, must start with a
55 * letter, max 128. Immutable — changing it replaces the group.
56 */
57 featureGroupId?: string;
58 /**
59 * Region. Immutable — changing it replaces the group.
60 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
61 */
62 location?: string;
63 /**
64 * Description of the feature group.
65 */
66 description?: string;
67 /**
68 * User labels. Alchemy ownership labels are merged in automatically.
69 */
70 labels?: Record<string, string>;
71 /**
72 * BigQuery source. The table or view must include entity-id columns
73 * and (unless `staticDataSource`) a `feature_timestamp` column.
74 */
75 bigQuery?: FeatureGroupBigQuery;
76 /**
77 * Service agent used by jobs under this group.
78 */
79 serviceAgentType?:
80 | aiplatform.GoogleCloudAiplatformV1FeatureGroupServiceAgentTypeEnum
81 | (string & {});
82 };
83
84 export type FeatureGroup = Resource<
85 "GCP.AIPlatform.FeatureGroup",
86 FeatureGroupProps,
87 {
88 /** Full resource name. */
89 name: string;
90 /** Feature group id. */
91 featureGroupId: string;
92 /** Project id. */
93 project: string;
94 /** Location id. */
95 location: string;
96 /** Description. */
97 description: string | undefined;
98 /** User labels (Alchemy ownership labels stripped). */
99 labels: Record<string, string>;
100 /** BigQuery source. */
101 bigQuery: FeatureGroupBigQuery | undefined;
102 /** Service agent type. */
103 serviceAgentType: string | undefined;
104 /** Unique service account email. */
105 serviceAccountEmail: string | undefined;
106 /** RFC3339 creation timestamp. */
107 createTime: string | undefined;
108 /** RFC3339 last-update timestamp. */
109 updateTime: string | undefined;
110 /** Server etag. */
111 etag: string | undefined;
112 },
113 never,
114 Providers
115 >;
116
117 /**
118 * A Vertex AI Feature Registry Feature Group backed by a BigQuery table
119 * or view.
120 *
121 * Id, location, and BigQuery URI are identity. Description, labels,
122 * entity-id columns, and service-agent type update in place.
123 *
124 * ### Creating a Feature Group
125 * **Example:** Point at a BigQuery table
126 * ```typescript
127 * const group = yield* GCP.AIPlatform.FeatureGroup("Users", {
128 * bigQuery: {
129 * inputUri: "bq://my-project.features.users",
130 * entityIdColumns: ["entity_id"],
131 * },
132 * labels: { env: "prod" },
133 * });
134 * ```
135 *
136 * @resource
137 * @category AIPlatform
138 */
139 export const FeatureGroup = Resource<FeatureGroup>("GCP.AIPlatform.FeatureGroup");
140
141 export class FeatureGroupNotResolved extends Data.TaggedError(
142 "GCP.AIPlatform.FeatureGroupNotResolved",
143 )<{
144 resourceName: string;
145 }> {}
146
147 export class FeatureGroupStillExists extends Data.TaggedError(
148 "GCP.AIPlatform.FeatureGroupStillExists",
149 )<{
150 resourceName: string;
151 }> {}
152
153 const resourceName = (project: string, location: string, groupId: string) =>
154 `projects/${project}/locations/${location}/featureGroups/${groupId}`;
155
156 const inputUriOf = (bigQuery: FeatureGroupBigQuery | undefined) =>
157 bigQuery?.inputUri ?? bigQuery?.bigQuerySource?.inputUri;
158
159 const toBigQuery = (
160 source: aiplatform.GoogleCloudAiplatformV1FeatureGroupBigQuery | FeatureGroupBigQuery | undefined,
161 ): FeatureGroupBigQuery | undefined => {
162 if (source === undefined) return undefined;
163 const inputUri =
164 "inputUri" in source
165 ? (source.inputUri ?? (source as FeatureGroupBigQuery).bigQuerySource?.inputUri)
166 : source.bigQuerySource?.inputUri;
167 return {
168 inputUri,
169 bigQuerySource: inputUri ? { inputUri } : undefined,
170 staticDataSource: source.staticDataSource,
171 dense: source.dense,
172 entityIdColumns: source.entityIdColumns,
173 timeSeries: source.timeSeries,
174 };
175 };
176
177 const toBigQueryBody = (
178 source: FeatureGroupBigQuery | undefined,
179 ): aiplatform.GoogleCloudAiplatformV1FeatureGroupBigQuery | undefined => {
180 if (source === undefined) return undefined;
181 const inputUri = inputUriOf(source);
182 return {
183 staticDataSource: source.staticDataSource,
184 dense: source.dense,
185 entityIdColumns: source.entityIdColumns,
186 timeSeries: source.timeSeries,
187 bigQuerySource: inputUri ? { inputUri } : source.bigQuerySource,
188 };
189 };
190
191 const toAttrs = (group: aiplatform.GoogleCloudAiplatformV1FeatureGroup, project: string) => {
192 const name = group.name ?? "";
193 const parsed = parseResourceName(name, "featureGroups");
194 return {
195 name,
196 featureGroupId: parsed.id,
197 project: parsed.project || project,
198 location: parsed.location,
199 description: group.description,
200 labels: userLabels(group.labels),
201 bigQuery: toBigQuery(group.bigQuery),
202 serviceAgentType: group.serviceAgentType,
203 serviceAccountEmail: group.serviceAccountEmail,
204 createTime: group.createTime,
205 updateTime: group.updateTime,
206 etag: group.etag,
207 };
208 };
209
210 const getByName = (name: string) =>
211 aiplatform
212 .getProjectsLocationsFeatureGroups({ name })
213 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
214
215 const waitUntilExists = (name: string) =>
216 getByName(name).pipe(
217 Effect.flatMap((group) =>
218 group
219 ? Effect.succeed(group)
220 : Effect.fail(new FeatureGroupNotResolved({ resourceName: name })),
221 ),
222 Effect.retry({
223 while: (error) => error._tag === "GCP.AIPlatform.FeatureGroupNotResolved",
224 times: 8,
225 schedule: Schedule.spaced("1 second"),
226 }),
227 );
228
229 const waitUntilGone = (name: string) =>
230 getByName(name).pipe(
231 Effect.flatMap((group) =>
232 group === undefined
233 ? Effect.void
234 : Effect.fail(new FeatureGroupStillExists({ resourceName: name })),
235 ),
236 Effect.retry({
237 while: (error) => error._tag === "GCP.AIPlatform.FeatureGroupStillExists",
238 times: 10,
239 schedule: Schedule.spaced("2 seconds"),
240 }),
241 );
242
243 export const FeatureGroupProvider = () =>
244 Provider.succeed(FeatureGroup, {
245 stables: ["name", "featureGroupId", "project", "location", "createTime"],
246
247 diff: Effect.fn(function* ({ news, olds, output }) {
248 const env = yield* GcpEnvironment.current;
249 if (!isResolved(news)) return undefined;
250 const previousId = olds?.featureGroupId ?? output?.featureGroupId;
251 const nextId = news.featureGroupId ?? previousId;
252 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
253 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
254 const previousUri = inputUriOf(olds?.bigQuery) ?? inputUriOf(output?.bigQuery) ?? "";
255 const nextUri = inputUriOf(news.bigQuery) ?? previousUri;
256 const replace =
257 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
258 previousLocation !== nextLocation ||
259 (news.bigQuery !== undefined && nextUri !== previousUri);
260 if (!replace) return undefined;
261 return {
262 action: "replace" as const,
263 deleteFirst:
264 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
265 };
266 }),
267
268 read: Effect.fn(function* ({ id, olds, output }) {
269 const env = yield* GcpEnvironment.current;
270 const featureGroupId = yield* toPhysicalSnake(
271 id,
272 olds?.featureGroupId,
273 output?.featureGroupId,
274 MAX_NAME_LENGTH,
275 );
276 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
277 const name = output?.name ?? resourceName(env.project, location, featureGroupId);
278 const existing = yield* getByName(name);
279 if (existing === undefined) return undefined;
280 const attrs = toAttrs(existing, env.project);
281 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
282 }),
283
284 list: () =>
285 Effect.gen(function* () {
286 const env = yield* GcpEnvironment.current;
287 return yield* Stream.fromIterable(listLocations(env.region)).pipe(
288 Stream.flatMap((location) =>
289 aiplatform.listProjectsLocationsFeatureGroups.pages({
290 parent: `projects/${env.project}/locations/${location}`,
291 pageSize: 100,
292 }),
293 ),
294 Stream.flatMap((page) => Stream.fromIterable(page.featureGroups ?? [])),
295 Stream.filter((group) => hasAlchemyLabelMap(group.labels)),
296 Stream.map((group) => toAttrs(group, env.project)),
297 Stream.runCollect,
298 Effect.map((chunk) => Array.from(chunk)),
299 Effect.catchTag("NotFound", () => Effect.succeed([])),
300 );
301 }),
302
303 reconcile: Effect.fn(function* ({ id, news, output }) {
304 const env = yield* GcpEnvironment.current;
305 const featureGroupId = yield* toPhysicalSnake(
306 id,
307 news.featureGroupId,
308 output?.featureGroupId,
309 MAX_NAME_LENGTH,
310 );
311 const location = normalizeLocation(news.location ?? output?.location, env.region);
312 const name = resourceName(env.project, location, featureGroupId);
313 const desiredLabels = {
314 ...toLabels(news.labels),
315 ...(yield* createInternalLabels(id)),
316 };
317 const bigQuery = toBigQueryBody(news.bigQuery);
318
319 let current = yield* getByName(output?.name ?? name);
320
321 if (current === undefined) {
322 const created = yield* aiplatform
323 .createProjectsLocationsFeatureGroups({
324 parent: `projects/${env.project}/locations/${location}`,
325 featureGroupId,
326 body: {
327 description: news.description,
328 labels: desiredLabels,
329 bigQuery,
330 serviceAgentType: news.serviceAgentType,
331 },
332 })
333 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
334 if (created !== undefined) {
335 yield* waitForOperation(created);
336 }
337 current = yield* waitUntilExists(name);
338 }
339
340 if (current === undefined) {
341 return yield* new FeatureGroupNotResolved({ resourceName: name });
342 }
343
344 const observedLabels = tagRecord(current.labels);
345 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
346 const labelsChanged = upsert.length > 0 || removed.length > 0;
347 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
348 const agentChanged =
349 news.serviceAgentType !== undefined &&
350 (current.serviceAgentType ?? "") !== news.serviceAgentType;
351 const entityColumnsChanged =
352 news.bigQuery?.entityIdColumns !== undefined &&
353 fingerprint(current.bigQuery?.entityIdColumns ?? []) !==
354 fingerprint(news.bigQuery.entityIdColumns);
355 const staticChanged =
356 news.bigQuery?.staticDataSource !== undefined &&
357 (current.bigQuery?.staticDataSource === true) !== (news.bigQuery.staticDataSource === true);
358 const denseChanged =
359 news.bigQuery?.dense !== undefined &&
360 (current.bigQuery?.dense === true) !== (news.bigQuery.dense === true);
361 const timeSeriesChanged =
362 news.bigQuery?.timeSeries !== undefined &&
363 !specifiedEquals(news.bigQuery.timeSeries, current.bigQuery?.timeSeries);
364
365 if (
366 labelsChanged ||
367 descriptionChanged ||
368 agentChanged ||
369 entityColumnsChanged ||
370 staticChanged ||
371 denseChanged ||
372 timeSeriesChanged
373 ) {
374 const updateMask = [
375 labelsChanged ? "labels" : undefined,
376 descriptionChanged ? "description" : undefined,
377 agentChanged ? "service_agent_type" : undefined,
378 entityColumnsChanged ? "big_query.entity_id_columns" : undefined,
379 staticChanged ? "big_query.static_data_source" : undefined,
380 denseChanged ? "big_query.dense" : undefined,
381 timeSeriesChanged ? "big_query.time_series" : undefined,
382 ].filter((field): field is string => field !== undefined);
383
384 const patched = yield* aiplatform
385 .patchProjectsLocationsFeatureGroups({
386 name,
387 updateMask: updateMask.join(","),
388 body: {
389 name,
390 labels: desiredLabels,
391 description: news.description,
392 serviceAgentType: news.serviceAgentType,
393 bigQuery,
394 },
395 })
396 .pipe(
397 Effect.retry({
398 while: (error) => error._tag === "Conflict",
399 times: 8,
400 schedule: Schedule.spaced("5 seconds"),
401 }),
402 );
403 yield* waitForOperation(patched);
404 current = yield* waitUntilExists(name);
405 }
406
407 return toAttrs(current, env.project);
408 }),
409
410 delete: Effect.fn(function* ({ output }) {
411 const operation = yield* aiplatform
412 .deleteProjectsLocationsFeatureGroups({
413 name: output.name,
414 force: true,
415 })
416 .pipe(
417 Effect.catchTag("NotFound", () => Effect.undefined),
418 Effect.retry({
419 while: (error) => error._tag === "Conflict",
420 times: 8,
421 schedule: Schedule.spaced("5 seconds"),
422 }),
423 );
424 if (operation !== undefined) {
425 yield* waitForOperation(operation, { notFoundOk: true });
426 }
427 yield* waitUntilGone(output.name);
428 }),
429 });