The best way to control Claude Code, Codex and more from any computer or phone

main
ts 388 lines 13.2 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { hasAlchemyLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 alchemyIdFilter,
16 createInternalLabels,
17 hasAlchemyPrefix,
18 labelsDiffer,
19 listLocations,
20 locationOf,
21 lastSegment,
22 normalizeLocation,
23 projectOf,
24 resourceNameFromOperation,
25 stableJson,
26 toDisplayName,
27 toLabels,
28 userLabels,
29 } from "./names.ts";
30 import { settlePendingCreates, waitForOperation } from "./operations.ts";
31
32 export type IndexProps = {
33 /**
34 * Vertex AI location (`us-central1`, …). Immutable — changing it
35 * replaces the index.
36 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
37 */
38 location?: string;
39 /**
40 * Display name (max 128 Unicode characters). Generated from the stack,
41 * stage, and logical id when omitted.
42 */
43 displayName?: string;
44 /**
45 * Human-readable description.
46 */
47 description?: string;
48 /**
49 * User labels. Alchemy ownership labels are merged in automatically.
50 */
51 labels?: Record<string, string>;
52 /**
53 * How the index is updated. Immutable.
54 * @default "STREAM_UPDATE"
55 */
56 indexUpdateMethod?: aiplatform.GoogleCloudAiplatformV1IndexIndexUpdateMethodEnum | (string & {});
57 /**
58 * Matching Engine metadata (`contentsDeltaUri`, `config.dimensions`,
59 * algorithm config).
60 */
61 metadata?: unknown;
62 /**
63 * GCS URI of the OpenAPI schema for `metadata`. Immutable.
64 */
65 metadataSchemaUri?: string;
66 /**
67 * Customer-managed encryption key. Immutable.
68 */
69 encryptionSpec?: aiplatform.GoogleCloudAiplatformV1EncryptionSpec;
70 };
71
72 export type Index = Resource<
73 "GCP.AIPlatform.Index",
74 IndexProps,
75 {
76 /** Full resource name `projects/{project}/locations/{location}/indexes/{index}`. */
77 name: string;
78 /** Index id (last path segment). */
79 indexId: string;
80 /** Project id. */
81 project: string;
82 /** Location id. */
83 location: string;
84 /** Display name. */
85 displayName: string | undefined;
86 /** User-provided description. */
87 description: string | undefined;
88 /** User labels (Alchemy ownership labels stripped). */
89 labels: Record<string, string>;
90 /** Update method (`BATCH_UPDATE` or `STREAM_UPDATE`). */
91 indexUpdateMethod: string | undefined;
92 /** Matching Engine metadata. */
93 metadata: unknown;
94 /** Schema URI for metadata, if set. */
95 metadataSchemaUri: string | undefined;
96 /** Dense vector count. */
97 vectorsCount: string | undefined;
98 /** RFC3339 creation timestamp. */
99 createTime: string | undefined;
100 /** RFC3339 last-update timestamp. */
101 updateTime: string | undefined;
102 },
103 never,
104 Providers
105 >;
106
107 /**
108 * A Vertex AI Matching Engine Index of embedding vectors.
109 *
110 * Changing `location`, `indexUpdateMethod`, encryption, or
111 * `metadataSchemaUri` replaces the index. Display name, description,
112 * labels, and metadata update in place (metadata updates are long-running).
113 *
114 * ### Creating an Index
115 * **Example:** Empty streaming index
116 * ```typescript
117 * const index = yield* GCP.AIPlatform.Index("Embeddings", {
118 * displayName: "product-embeddings",
119 * indexUpdateMethod: "STREAM_UPDATE",
120 * metadata: {
121 * config: {
122 * dimensions: 768,
123 * approximateNeighborsCount: 150,
124 * distanceMeasureType: "DOT_PRODUCT_DISTANCE",
125 * shardSize: "SHARD_SIZE_SMALL",
126 * algorithmConfig: { bruteForceConfig: {} },
127 * },
128 * },
129 * labels: { env: "prod" },
130 * });
131 * ```
132 *
133 * @resource
134 * @category AIPlatform
135 */
136 export const Index = Resource<Index>("GCP.AIPlatform.Index");
137
138 export class IndexNotResolved extends Data.TaggedError("GCP.AIPlatform.IndexNotResolved")<{
139 resourceName: string;
140 }> {}
141
142 export class IndexStillExists extends Data.TaggedError("GCP.AIPlatform.IndexStillExists")<{
143 resourceName: string;
144 }> {}
145
146 const DEFAULT_UPDATE_METHOD = "STREAM_UPDATE";
147
148 const toAttrs = (index: aiplatform.GoogleCloudAiplatformV1Index, project: string) => {
149 const name = index.name ?? "";
150 return {
151 name,
152 indexId: lastSegment(name),
153 project: projectOf(name, project),
154 location: locationOf(name),
155 displayName: index.displayName,
156 description: index.description,
157 labels: userLabels(index.labels),
158 indexUpdateMethod: index.indexUpdateMethod,
159 metadata: index.metadata,
160 metadataSchemaUri: index.metadataSchemaUri,
161 vectorsCount: index.indexStats?.vectorsCount,
162 createTime: index.createTime,
163 updateTime: index.updateTime,
164 };
165 };
166
167 const getByName = (name: string) =>
168 aiplatform
169 .getProjectsLocationsIndexes({ name })
170 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
171
172 const listPage = (parent: string, filter?: string) =>
173 aiplatform.listProjectsLocationsIndexes.pages({ parent, pageSize: 100, filter }).pipe(
174 Stream.runCollect,
175 Effect.map((pages) => Array.from(pages).flatMap((page) => page.indexes ?? [])),
176 Effect.catchTag("NotFound", () =>
177 Effect.succeed([] as aiplatform.GoogleCloudAiplatformV1Index[]),
178 ),
179 );
180
181 const findVisible = (project: string, location: string, labels: Record<string, string>) =>
182 listPage(`projects/${project}/locations/${location}`, alchemyIdFilter(labels)).pipe(
183 Effect.map((items) => items.find((item) => hasAlchemyPrefix(item.labels)) ?? undefined),
184 );
185
186 /**
187 * Find our resource by its ownership labels. Ids are server-assigned and the
188 * resource is invisible to `list` while its create operation is pending
189 * (minutes on a fresh project), so on a miss wait for pending creates in
190 * this location and look again — otherwise an interrupted create's
191 * recovery read reports it gone and leaks it.
192 */
193 const findOwned = (project: string, location: string, labels: Record<string, string>) =>
194 Effect.gen(function* () {
195 const visible = yield* findVisible(project, location, labels);
196 if (visible !== undefined) return visible;
197 const settled = yield* settlePendingCreates({
198 parent: `projects/${project}/locations/${location}`,
199 collection: "indexes",
200 });
201 return settled ? yield* findVisible(project, location, labels) : undefined;
202 });
203
204 const waitUntilExists = (name: string) =>
205 getByName(name).pipe(
206 Effect.filterOrFail(
207 (index): index is aiplatform.GoogleCloudAiplatformV1Index => index !== undefined,
208 () => new IndexNotResolved({ resourceName: name }),
209 ),
210 Effect.retry({
211 while: (error) => error._tag === "GCP.AIPlatform.IndexNotResolved",
212 times: 8,
213 schedule: Schedule.spaced("2 seconds"),
214 }),
215 );
216
217 const waitUntilGone = (name: string) =>
218 getByName(name).pipe(
219 Effect.filterOrFail(
220 (index) => index === undefined,
221 () => new IndexStillExists({ resourceName: name }),
222 ),
223 Effect.asVoid,
224 Effect.retry({
225 while: (error) => error._tag === "GCP.AIPlatform.IndexStillExists",
226 times: 10,
227 schedule: Schedule.spaced("2 seconds"),
228 }),
229 );
230
231 export const IndexProvider = () =>
232 Provider.succeed(Index, {
233 stables: ["name", "indexId", "project", "location", "indexUpdateMethod", "createTime"],
234
235 diff: Effect.fn(function* ({ news, olds, output }) {
236 const env = yield* GcpEnvironment.current;
237 if (!isResolved(news)) return undefined;
238 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
239 const nextLocation = normalizeLocation(
240 news.location ?? olds?.location ?? output?.location,
241 env.region,
242 );
243 const previousMethod = (
244 olds?.indexUpdateMethod ??
245 output?.indexUpdateMethod ??
246 DEFAULT_UPDATE_METHOD
247 ).toUpperCase();
248 const nextMethod = (news.indexUpdateMethod ?? previousMethod).toUpperCase();
249 const previousSchema = olds?.metadataSchemaUri ?? output?.metadataSchemaUri ?? "";
250 const nextSchema = news.metadataSchemaUri ?? previousSchema;
251 if (
252 previousLocation !== nextLocation ||
253 previousMethod !== nextMethod ||
254 nextSchema !== previousSchema
255 ) {
256 return { action: "replace" as const, deleteFirst: false };
257 }
258 return undefined;
259 }),
260
261 read: Effect.fn(function* ({ id, olds, output }) {
262 const env = yield* GcpEnvironment.current;
263 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
264 const ownership = yield* createInternalLabels(id);
265 const existing =
266 (output?.name !== undefined ? yield* getByName(output.name) : undefined) ??
267 (yield* findOwned(env.project, location, ownership));
268 if (existing === undefined) return undefined;
269 const attrs = toAttrs(existing, env.project);
270 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
271 }),
272
273 list: () =>
274 Effect.gen(function* () {
275 const env = yield* GcpEnvironment.current;
276 const pages = yield* Effect.forEach(
277 listLocations(env.region),
278 (location) => listPage(`projects/${env.project}/locations/${location}`),
279 { concurrency: 4 },
280 );
281 return pages
282 .flat()
283 .filter((index) => hasAlchemyPrefix(index.labels))
284 .map((index) => toAttrs(index, env.project));
285 }),
286
287 reconcile: Effect.fn(function* ({ id, news, output }) {
288 const env = yield* GcpEnvironment.current;
289 const location = normalizeLocation(news.location ?? output?.location, env.region);
290 const displayName = yield* toDisplayName(id, news.displayName, output?.displayName);
291 const desiredLabels = {
292 ...toLabels(news.labels),
293 ...(yield* createInternalLabels(id)),
294 };
295 const indexUpdateMethod = (
296 news.indexUpdateMethod ??
297 output?.indexUpdateMethod ??
298 DEFAULT_UPDATE_METHOD
299 ).toUpperCase();
300
301 let current =
302 (output?.name !== undefined ? yield* getByName(output.name) : undefined) ??
303 (yield* findOwned(env.project, location, desiredLabels));
304
305 if (current === undefined) {
306 const created = yield* aiplatform
307 .createProjectsLocationsIndexes({
308 parent: `projects/${env.project}/locations/${location}`,
309 body: {
310 displayName,
311 description: news.description,
312 labels: desiredLabels,
313 indexUpdateMethod,
314 metadata: news.metadata,
315 metadataSchemaUri: news.metadataSchemaUri,
316 encryptionSpec: news.encryptionSpec,
317 },
318 })
319 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
320 if (created !== undefined) {
321 const done = yield* waitForOperation(created);
322 const name = resourceNameFromOperation(done);
323 current =
324 name !== undefined
325 ? yield* waitUntilExists(name)
326 : yield* findOwned(env.project, location, desiredLabels);
327 }
328 if (current === undefined) {
329 current = yield* findOwned(env.project, location, desiredLabels);
330 }
331 }
332
333 if (current === undefined || current.name === undefined) {
334 return yield* new IndexNotResolved({
335 resourceName: output?.name ?? `${location}/indexes`,
336 });
337 }
338
339 const name = current.name;
340 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
341 const displayChanged = (current.displayName ?? "") !== displayName;
342 const labelsChanged = labelsDiffer(current.labels, desiredLabels);
343 const metadataChanged =
344 news.metadata !== undefined && stableJson(current.metadata) !== stableJson(news.metadata);
345
346 if (descriptionChanged || displayChanged || labelsChanged || metadataChanged) {
347 const operation = yield* aiplatform.patchProjectsLocationsIndexes({
348 name,
349 updateMask: [
350 descriptionChanged ? "description" : undefined,
351 displayChanged ? "display_name" : undefined,
352 labelsChanged ? "labels" : undefined,
353 metadataChanged ? "metadata" : undefined,
354 ]
355 .filter((field): field is string => field !== undefined)
356 .join(","),
357 body: {
358 name,
359 displayName,
360 description: news.description,
361 labels: desiredLabels,
362 metadata: news.metadata,
363 },
364 });
365 yield* waitForOperation(operation);
366 current = yield* waitUntilExists(name);
367 }
368
369 return toAttrs(current, env.project);
370 }),
371
372 delete: Effect.fn(function* ({ output }) {
373 const operation = yield* aiplatform
374 .deleteProjectsLocationsIndexes({ name: output.name })
375 .pipe(
376 Effect.retry({
377 while: (error) => error._tag === "Conflict",
378 times: 8,
379 schedule: Schedule.spaced("2 seconds"),
380 }),
381 Effect.catchTag("NotFound", () => Effect.undefined),
382 );
383 if (operation !== undefined) {
384 yield* waitForOperation(operation, { notFoundOk: true });
385 }
386 yield* waitUntilGone(output.name);
387 }),
388 });