The best way to control Claude Code, Codex and more from any computer or phone

main
ts 445 lines 15.5 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { GcpEnvironment } from "../Environment.ts";
13 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
14 import type { Providers } from "../Providers.ts";
15 import {
16 DEFAULT_TABULAR_METADATA_SCHEMA_URI,
17 hasAlchemyLabelKeys,
18 normalizeLocation,
19 parentOf,
20 parseResourceName,
21 rfc1035,
22 resourceNameFromOperation,
23 userLabels,
24 waitForOperation,
25 } from "./internal.ts";
26 import { listLocations } from "./names.ts";
27 import type { EncryptionSpec } from "./shared.ts";
28
29 export type DatasetProps = {
30 /**
31 * Dataset id (the `{dataset}` segment of
32 * `projects/{project}/locations/{location}/datasets/{dataset}`). If
33 * omitted, Vertex AI assigns an id. Immutable — changing it replaces
34 * the dataset.
35 */
36 datasetId?: string;
37 /**
38 * Region (`us-central1`, `us-east1`, …). Immutable — changing it
39 * replaces the dataset. `US-CENTRAL1` is accepted and normalized to
40 * `us-central1`.
41 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
42 */
43 location?: string;
44 /**
45 * User-facing display name (max 128 characters). Defaults to the
46 * dataset id.
47 */
48 displayName?: string;
49 /**
50 * Human-readable description.
51 */
52 description?: string;
53 /**
54 * User labels. Alchemy ownership labels are merged in automatically.
55 */
56 labels?: Record<string, string>;
57 /**
58 * GCS URI of the dataset metadata schema YAML (OpenAPI 3.0.2 Schema
59 * Object). Immutable — changing it replaces the dataset.
60 * @default "gs://google-cloud-aiplatform/schema/dataset/metadata/tabular_1.0.0.yaml"
61 */
62 metadataSchemaUri?: string;
63 /**
64 * Additional schema-specific metadata. For an empty tabular dataset,
65 * `{}` is sufficient.
66 */
67 metadata?: unknown;
68 /**
69 * Customer-managed encryption. Immutable — changing it replaces the
70 * dataset.
71 */
72 encryptionSpec?: EncryptionSpec;
73 /**
74 * Public base model last used by a prompt dataset.
75 */
76 modelReference?: string;
77 /**
78 * Saved queries created with the dataset (max one on create). Required
79 * for DatasetVersion — Vertex rejects versioning an empty dataset with
80 * no SavedQuery (`Unsupported dataset version creation with empty set
81 * of saved queries`).
82 */
83 savedQueries?: Array<{
84 /** User-facing saved query name. */
85 displayName?: string;
86 /**
87 * Problem type (`IMAGE_CLASSIFICATION_SINGLE_LABEL`,
88 * `TEXT_CLASSIFICATION_SINGLE_LABEL`, …).
89 */
90 problemType?: string;
91 /** Additional saved-query metadata. */
92 metadata?: unknown;
93 }>;
94 };
95
96 export type Dataset = Resource<
97 "GCP.AIPlatform.Dataset",
98 DatasetProps,
99 {
100 /** Full resource name `projects/{project}/locations/{location}/datasets/{dataset}`. */
101 name: string;
102 /** Dataset id (last path segment). */
103 datasetId: string;
104 /** Project id. */
105 project: string;
106 /** Location id (`us-central1`, …). */
107 location: string;
108 /** User-facing display name. */
109 displayName: string | undefined;
110 /** Description. */
111 description: string | undefined;
112 /** User labels (Alchemy ownership labels stripped). */
113 labels: Record<string, string>;
114 /** Metadata schema URI. */
115 metadataSchemaUri: string | undefined;
116 /** Schema-specific metadata. */
117 metadata: unknown;
118 /** Number of DataItems (non-structured datasets). */
119 dataItemCount: string | undefined;
120 /** MetadataStore artifact created with the dataset. */
121 metadataArtifact: string | undefined;
122 /** Public base model reference, if any. */
123 modelReference: string | undefined;
124 /** Customer-managed KMS key, if any. */
125 kmsKeyName: string | undefined;
126 /** RFC3339 creation timestamp. */
127 createTime: string | undefined;
128 /** RFC3339 last-update timestamp. */
129 updateTime: string | undefined;
130 },
131 never,
132 Providers
133 >;
134
135 /**
136 * A Vertex AI Dataset — a collection of DataItems and Annotations.
137 *
138 * Location, dataset id, metadata schema, and CMEK are immutable. Display
139 * name, description, and labels update in place. Alchemy ownership labels
140 * are merged into `labels` so `list` / `pnpm nuke:gcp` can find the
141 * dataset.
142 *
143 * ### Creating a Dataset
144 * **Example:** Generated id, empty tabular dataset
145 * ```typescript
146 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
147 * displayName: "sample rows",
148 * labels: { env: "test" },
149 * });
150 * ```
151 *
152 * **Example:** Explicit id and image metadata schema
153 * ```typescript
154 * const dataset = yield* GCP.AIPlatform.Dataset("Images", {
155 * datasetId: "app-images",
156 * location: "us-central1",
157 * metadataSchemaUri:
158 * "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml",
159 * metadata: {},
160 * });
161 * ```
162 *
163 * ### Updating a Dataset
164 * **Example:** Rename and relabel
165 * ```typescript
166 * // Same logical id, changed props: the engine updates it in place.
167 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
168 * displayName: "prod rows",
169 * labels: { env: "prod" },
170 * });
171 * ```
172 *
173 * @resource
174 * @category AIPlatform
175 */
176 export const Dataset = Resource<Dataset>("GCP.AIPlatform.Dataset");
177
178 export class DatasetNotResolved extends Data.TaggedError("GCP.AIPlatform.DatasetNotResolved")<{
179 resourceName: string;
180 }> {}
181
182 export class DatasetStillExists extends Data.TaggedError("GCP.AIPlatform.DatasetStillExists")<{
183 resourceName: string;
184 }> {}
185
186 const resourceName = (project: string, location: string, datasetId: string) =>
187 `projects/${project}/locations/${location}/datasets/${datasetId}`;
188
189 const toId = (id: string, datasetId: string | undefined, existing?: string) =>
190 Effect.gen(function* () {
191 if (datasetId !== undefined) return rfc1035(datasetId);
192 if (existing !== undefined) return existing;
193 return rfc1035(
194 yield* createPhysicalName({
195 id,
196 maxLength: 63,
197 lowercase: true,
198 }),
199 );
200 });
201
202 const schemaUriOf = (uri: string | undefined) =>
203 uri && uri.length > 0 ? uri : DEFAULT_TABULAR_METADATA_SCHEMA_URI;
204
205 const toAttrs = (dataset: aiplatform.GoogleCloudAiplatformV1Dataset, project: string) => {
206 const name = dataset.name ?? "";
207 const parsed = parseResourceName(name, "datasets");
208 return {
209 name,
210 datasetId: parsed.id,
211 project: parsed.project || project,
212 location: parsed.location,
213 displayName: dataset.displayName,
214 description: dataset.description,
215 labels: userLabels(dataset.labels),
216 metadataSchemaUri: dataset.metadataSchemaUri,
217 metadata: dataset.metadata,
218 dataItemCount: dataset.dataItemCount,
219 metadataArtifact: dataset.metadataArtifact,
220 modelReference: dataset.modelReference,
221 kmsKeyName: dataset.encryptionSpec?.kmsKeyName,
222 createTime: dataset.createTime,
223 updateTime: dataset.updateTime,
224 };
225 };
226
227 const getByName = (name: string) =>
228 name.length === 0
229 ? Effect.undefined
230 : aiplatform.getDatasets({ name }).pipe(Effect.catchTag("NotFound", () => Effect.undefined));
231
232 const listDatasets = (project: string, region: string) =>
233 Effect.forEach(listLocations(region), (location) =>
234 listDatasetsAt(parentOf(project, location)),
235 ).pipe(Effect.map((pages) => pages.flat()));
236
237 const listDatasetsAt = (parent: string) =>
238 aiplatform.listProjectsLocationsDatasets
239 .pages({
240 parent,
241 pageSize: 100,
242 })
243 .pipe(
244 Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
245 Stream.filter((dataset) => hasAlchemyLabelKeys(dataset.labels)),
246 Stream.runCollect,
247 Effect.map((chunk) => Array.from(chunk)),
248 Effect.catchTag("NotFound", () => Effect.succeed([])),
249 );
250
251 const findOwned = (id: string, project: string, region: string, hinted?: string) =>
252 Effect.gen(function* () {
253 if (hinted !== undefined && hinted.length > 0) {
254 const existing = yield* getByName(hinted);
255 if (existing !== undefined) return existing;
256 }
257 const owned = yield* listDatasets(project, region);
258 for (const dataset of owned) {
259 if (yield* hasAlchemyLabels(id, tagRecord(dataset.labels))) {
260 return dataset;
261 }
262 }
263 return undefined as aiplatform.GoogleCloudAiplatformV1Dataset | undefined;
264 });
265
266 const waitUntilExists = (name: string) =>
267 getByName(name).pipe(
268 Effect.flatMap((dataset) =>
269 dataset
270 ? Effect.succeed(dataset)
271 : Effect.fail(new DatasetNotResolved({ resourceName: name })),
272 ),
273 Effect.retry({
274 while: (error) => error._tag === "GCP.AIPlatform.DatasetNotResolved",
275 times: 8,
276 schedule: Schedule.spaced("1 second"),
277 }),
278 );
279
280 const waitUntilGone = (name: string) =>
281 getByName(name).pipe(
282 Effect.flatMap((dataset) =>
283 dataset === undefined
284 ? Effect.void
285 : Effect.fail(new DatasetStillExists({ resourceName: name })),
286 ),
287 Effect.retry({
288 while: (error) => error._tag === "GCP.AIPlatform.DatasetStillExists",
289 times: 10,
290 schedule: Schedule.spaced("2 seconds"),
291 }),
292 );
293
294 export const DatasetProvider = () =>
295 Provider.succeed(Dataset, {
296 stables: ["name", "datasetId", "project", "location", "metadataSchemaUri", "createTime"],
297
298 diff: Effect.fn(function* ({ news, olds, output }) {
299 const env = yield* GcpEnvironment.current;
300 if (!isResolved(news)) return undefined;
301 const previousId = olds?.datasetId ?? output?.datasetId;
302 const nextId = news.datasetId ?? previousId;
303 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
304 const nextLocation = normalizeLocation(
305 news.location ?? olds?.location ?? output?.location,
306 env.region,
307 );
308 const previousSchema = schemaUriOf(olds?.metadataSchemaUri ?? output?.metadataSchemaUri);
309 const nextSchema = schemaUriOf(
310 news.metadataSchemaUri ?? olds?.metadataSchemaUri ?? output?.metadataSchemaUri,
311 );
312 const previousKey = olds?.encryptionSpec?.kmsKeyName ?? output?.kmsKeyName ?? "";
313 const nextKey = news.encryptionSpec?.kmsKeyName ?? previousKey;
314 const replace =
315 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
316 previousLocation !== nextLocation ||
317 previousSchema !== nextSchema ||
318 previousKey !== nextKey;
319 if (!replace) return undefined;
320 return {
321 action: "replace" as const,
322 deleteFirst:
323 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
324 };
325 }),
326
327 read: Effect.fn(function* ({ id, olds, output }) {
328 const env = yield* GcpEnvironment.current;
329 const existing = yield* findOwned(id, env.project, env.region, output?.name);
330 if (existing === undefined) {
331 if (output?.name) return undefined;
332 // Dataset ids are server-assigned numbers; only a caller-supplied
333 // id can be looked up by name.
334 const datasetId = olds?.datasetId;
335 if (datasetId === undefined) return undefined;
336 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
337 const named = yield* getByName(resourceName(env.project, location, datasetId));
338 if (named === undefined) return undefined;
339 const attrs = toAttrs(named, env.project);
340 return (yield* hasAlchemyLabels(id, tagRecord(named.labels))) ? attrs : Unowned(attrs);
341 }
342 const attrs = toAttrs(existing, env.project);
343 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
344 }),
345
346 list: () =>
347 Effect.gen(function* () {
348 const env = yield* GcpEnvironment.current;
349 const datasets = yield* listDatasets(env.project, env.region);
350 return datasets.map((dataset) => toAttrs(dataset, env.project));
351 }),
352
353 reconcile: Effect.fn(function* ({ id, news, output }) {
354 const env = yield* GcpEnvironment.current;
355 const location = normalizeLocation(news.location ?? output?.location, env.region);
356 const datasetId = yield* toId(id, news.datasetId, output?.datasetId);
357 const desiredLabels = {
358 ...toLabels(news.labels),
359 ...(yield* createInternalLabels(id)),
360 };
361 const displayName = news.displayName ?? datasetId;
362 const metadataSchemaUri = schemaUriOf(news.metadataSchemaUri);
363
364 let current = yield* findOwned(id, env.project, env.region, output?.name);
365 if (current === undefined && news.datasetId !== undefined) {
366 current = yield* getByName(resourceName(env.project, location, news.datasetId));
367 }
368
369 if (current === undefined) {
370 const created = yield* aiplatform
371 .createProjectsLocationsDatasets({
372 parent: parentOf(env.project, location),
373 body: {
374 displayName,
375 description: news.description,
376 labels: desiredLabels,
377 metadataSchemaUri,
378 metadata: news.metadata ?? {},
379 encryptionSpec: news.encryptionSpec,
380 modelReference: news.modelReference,
381 savedQueries: news.savedQueries,
382 },
383 })
384 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
385 if (created !== undefined) {
386 const done = yield* waitForOperation(created);
387 const createdName =
388 resourceNameFromOperation(done) ??
389 (yield* findOwned(id, env.project, env.region))?.name;
390 if (createdName !== undefined && createdName.length > 0) {
391 current = yield* waitUntilExists(createdName);
392 }
393 }
394 if (current === undefined) {
395 current = yield* findOwned(id, env.project, env.region);
396 }
397 }
398
399 if (current === undefined) {
400 return yield* new DatasetNotResolved({
401 resourceName: output?.name ?? resourceName(env.project, location, datasetId),
402 });
403 }
404
405 const name = current.name ?? "";
406 const observedLabels = tagRecord(current.labels);
407 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
408 const labelsChanged = upsert.length > 0 || removed.length > 0;
409 const displayNameChanged = (current.displayName ?? "") !== displayName;
410 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
411
412 if (labelsChanged || displayNameChanged || descriptionChanged) {
413 current = yield* aiplatform.patchProjectsLocationsDatasets({
414 name,
415 updateMask: [
416 labelsChanged ? "labels" : undefined,
417 displayNameChanged ? "display_name" : undefined,
418 descriptionChanged ? "description" : undefined,
419 ]
420 .filter((field): field is string => field !== undefined)
421 .join(","),
422 body: {
423 name,
424 labels: desiredLabels,
425 displayName,
426 description: news.description,
427 },
428 });
429 }
430
431 return toAttrs(current, env.project);
432 }),
433
434 delete: Effect.fn(function* ({ output }) {
435 const existing = yield* getByName(output.name);
436 if (existing === undefined) return;
437 const operation = yield* aiplatform
438 .deleteDatasets({ name: output.name })
439 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
440 if (operation !== undefined) {
441 yield* waitForOperation(operation, { notFoundOk: true });
442 }
443 yield* waitUntilGone(output.name);
444 }),
445 });