@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 357 lines 11.8 KB
Raw
1 import { createHash } from "node:crypto";
2 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { isResolved } from "alchemy/Diff";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 hasAlchemyLabelKeys,
16 parseOwnership,
17 parseResourceName,
18 resourceNameFromOperation,
19 waitForOperation,
20 } from "./internal.ts";
21 import { listLocations } from "./names.ts";
22
23 export type DatasetsDatasetVersionProps = {
24 /**
25 * Parent Dataset resource name
26 * `projects/{project}/locations/{location}/datasets/{dataset}`.
27 * Immutable — changing it replaces the version.
28 */
29 dataset: string;
30 /**
31 * User-facing display name (max 128 characters). Alchemy stamps
32 * ownership into this field (versions have no labels) so `list` /
33 * nuke can find the version.
34 */
35 displayName?: string;
36 };
37
38 export type DatasetsDatasetVersion = Resource<
39 "GCP.AIPlatform.DatasetsDatasetVersion",
40 DatasetsDatasetVersionProps,
41 {
42 /** Full resource name `.../datasets/{dataset}/datasetVersions/{version}`. */
43 name: string;
44 /** Dataset version id (last path segment). */
45 datasetVersionId: string;
46 /** Parent dataset resource name. */
47 dataset: string;
48 /** Project id. */
49 project: string;
50 /** Location id. */
51 location: string;
52 /** User display name with the Alchemy ownership prefix stripped. */
53 displayName: string | undefined;
54 /** Schema-specific metadata. */
55 metadata: unknown;
56 /** Associated BigQuery dataset name, if any. */
57 bigQueryDatasetName: string | undefined;
58 /** Public base model reference, if any. */
59 modelReference: string | undefined;
60 /** RFC3339 creation timestamp. */
61 createTime: string | undefined;
62 /** RFC3339 last-update timestamp. */
63 updateTime: string | undefined;
64 },
65 never,
66 Providers
67 >;
68
69 /**
70 * A Vertex AI DatasetVersion — a snapshot of a Dataset.
71 *
72 * Versions have no labels field, so Alchemy stamps ownership into
73 * `displayName` for `list` / nuke. The parent dataset is immutable;
74 * display name updates in place.
75 *
76 * ### Creating a Dataset Version
77 * **Example:** Snapshot a dataset
78 * ```typescript
79 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
80 * metadataSchemaUri:
81 * "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml",
82 * });
83 * const version = yield* GCP.AIPlatform.DatasetsDatasetVersion("V1", {
84 * dataset: dataset.name,
85 * displayName: "v1",
86 * });
87 * ```
88 *
89 * ### Updating a Dataset Version
90 * **Example:** Rename the snapshot
91 * ```typescript
92 * // Same logical id, changed props: the engine updates it in place.
93 * const version = yield* GCP.AIPlatform.DatasetsDatasetVersion("V1", {
94 * dataset: dataset.name,
95 * displayName: "v1-final",
96 * });
97 * ```
98 *
99 * @resource
100 * @category AIPlatform
101 */
102 export const DatasetsDatasetVersion = Resource<DatasetsDatasetVersion>(
103 "GCP.AIPlatform.DatasetsDatasetVersion",
104 );
105
106 export class DatasetsDatasetVersionNotResolved extends Data.TaggedError(
107 "GCP.AIPlatform.DatasetsDatasetVersionNotResolved",
108 )<{
109 resourceName: string;
110 }> {}
111
112 export class DatasetsDatasetVersionStillExists extends Data.TaggedError(
113 "GCP.AIPlatform.DatasetsDatasetVersionStillExists",
114 )<{
115 resourceName: string;
116 }> {}
117
118 const parentDataset = (name: string) => {
119 const marker = "/datasetVersions/";
120 const index = name.indexOf(marker);
121 return index >= 0 ? name.slice(0, index) : name;
122 };
123
124 const parseVersionName = (name: string) => {
125 const parsed = parseResourceName(name, "datasetVersions");
126 const datasets = parseResourceName(name, "datasets");
127 return {
128 ...parsed,
129 datasetId: datasets.id,
130 dataset: parentDataset(name),
131 };
132 };
133
134 const getByName = (name: string) =>
135 name.length === 0
136 ? Effect.undefined
137 : aiplatform
138 .getDatasetsDatasetVersions({ name })
139 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
140
141 /**
142 * Dataset versions carry ownership only in their display name, which
143 * Vertex AI caps at 128 characters — too short for the full
144 * `[alchemy stack=… stage=… id=…]` marker of long stack names. Stamp a
145 * fixed-width hash of the ownership labels instead.
146 */
147 const ownerTag = (id: string) =>
148 createInternalLabels(id).pipe(
149 Effect.flatMap((labels) =>
150 Effect.sync(
151 () =>
152 `[alchemy #${createHash("sha256")
153 .update(JSON.stringify(labels))
154 .digest("hex")
155 .slice(0, 16)}]`,
156 ),
157 ),
158 );
159
160 const encodeDisplayName = (tag: string, displayName: string | undefined) =>
161 displayName ? `${tag} ${displayName}` : tag;
162
163 const ownedByAlchemy = (id: string, displayName: string | undefined) =>
164 Effect.gen(function* () {
165 if (displayName?.startsWith(yield* ownerTag(id))) return true;
166 return yield* hasAlchemyLabels(id, parseOwnership(displayName).labels);
167 });
168
169 const toAttrs = (version: aiplatform.GoogleCloudAiplatformV1DatasetVersion, project: string) => {
170 const name = version.name ?? "";
171 const parsed = parseVersionName(name);
172 const ownership = parseOwnership(version.displayName);
173 return {
174 name,
175 datasetVersionId: parsed.id,
176 dataset: parsed.dataset,
177 project: parsed.project || project,
178 location: parsed.location,
179 displayName: ownership.text,
180 metadata: version.metadata,
181 bigQueryDatasetName: version.bigQueryDatasetName,
182 modelReference: version.modelReference,
183 createTime: version.createTime,
184 updateTime: version.updateTime,
185 };
186 };
187
188 const listVersions = (dataset: string) =>
189 aiplatform.listProjectsLocationsDatasetsDatasetVersions
190 .pages({ parent: dataset, pageSize: 100 })
191 .pipe(
192 Stream.flatMap((page) => Stream.fromIterable(page.datasetVersions ?? [])),
193 Stream.runCollect,
194 Effect.map((chunk) => Array.from(chunk)),
195 Effect.catchTag("NotFound", () => Effect.succeed([])),
196 );
197
198 const listDatasets = (project: string, region: string) =>
199 Effect.forEach(listLocations(region), (location) =>
200 listDatasetsAt(`projects/${project}/locations/${location}`),
201 ).pipe(Effect.map((pages) => pages.flat()));
202
203 const listDatasetsAt = (parent: string) =>
204 aiplatform.listProjectsLocationsDatasets
205 .pages({
206 parent,
207 pageSize: 100,
208 })
209 .pipe(
210 Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
211 Stream.filter((dataset) => hasAlchemyLabelKeys(dataset.labels)),
212 Stream.runCollect,
213 Effect.map((chunk) => Array.from(chunk)),
214 Effect.catchTag("NotFound", () => Effect.succeed([])),
215 );
216
217 const findOwned = (id: string, dataset: string, hinted?: string) =>
218 Effect.gen(function* () {
219 if (hinted !== undefined && hinted.length > 0) {
220 const existing = yield* getByName(hinted);
221 if (existing !== undefined) return existing;
222 }
223 const versions = yield* listVersions(dataset);
224 for (const version of versions) {
225 if (yield* ownedByAlchemy(id, version.displayName)) return version;
226 }
227 return undefined as aiplatform.GoogleCloudAiplatformV1DatasetVersion | undefined;
228 });
229
230 const waitUntilExists = (name: string) =>
231 getByName(name).pipe(
232 Effect.flatMap((version) =>
233 version
234 ? Effect.succeed(version)
235 : Effect.fail(new DatasetsDatasetVersionNotResolved({ resourceName: name })),
236 ),
237 Effect.retry({
238 while: (error) => error._tag === "GCP.AIPlatform.DatasetsDatasetVersionNotResolved",
239 times: 8,
240 schedule: Schedule.spaced("1 second"),
241 }),
242 );
243
244 const waitUntilGone = (name: string) =>
245 getByName(name).pipe(
246 Effect.flatMap((version) =>
247 version === undefined
248 ? Effect.void
249 : Effect.fail(new DatasetsDatasetVersionStillExists({ resourceName: name })),
250 ),
251 Effect.retry({
252 while: (error) => error._tag === "GCP.AIPlatform.DatasetsDatasetVersionStillExists",
253 times: 10,
254 schedule: Schedule.spaced("2 seconds"),
255 }),
256 );
257
258 export const DatasetsDatasetVersionProvider = () =>
259 Provider.succeed(DatasetsDatasetVersion, {
260 stables: ["name", "datasetVersionId", "dataset", "project", "location", "createTime"],
261
262 diff: Effect.fn(function* ({ news, olds, output }) {
263 if (!isResolved(news)) return undefined;
264 const previous = olds?.dataset ?? output?.dataset;
265 if (previous !== undefined && news.dataset !== previous) {
266 return { action: "replace" as const, deleteFirst: false };
267 }
268 return undefined;
269 }),
270
271 read: Effect.fn(function* ({ id, olds, output }) {
272 const env = yield* GcpEnvironment.current;
273 const dataset = olds?.dataset ?? output?.dataset;
274 const existing =
275 output?.name !== undefined
276 ? yield* getByName(output.name)
277 : dataset !== undefined
278 ? yield* findOwned(id, dataset)
279 : undefined;
280 if (existing === undefined) return undefined;
281 const attrs = toAttrs(existing, env.project);
282 return (yield* ownedByAlchemy(id, existing.displayName)) ? attrs : Unowned(attrs);
283 }),
284
285 list: () =>
286 Effect.gen(function* () {
287 const env = yield* GcpEnvironment.current;
288 const datasets = yield* listDatasets(env.project, env.region);
289 const versions: ReturnType<typeof toAttrs>[] = [];
290 for (const dataset of datasets) {
291 if (dataset.name === undefined) continue;
292 const page = yield* listVersions(dataset.name);
293 for (const version of page) {
294 if (version.displayName?.startsWith("[alchemy ")) {
295 versions.push(toAttrs(version, env.project));
296 }
297 }
298 }
299 return versions;
300 }),
301
302 reconcile: Effect.fn(function* ({ id, news, output }) {
303 const env = yield* GcpEnvironment.current;
304 const desiredDisplayName = encodeDisplayName(yield* ownerTag(id), news.displayName);
305
306 let current = yield* findOwned(id, news.dataset, output?.name);
307
308 if (current === undefined) {
309 const created = yield* aiplatform
310 .createDatasetsDatasetVersions({
311 parent: news.dataset,
312 body: { displayName: desiredDisplayName },
313 })
314 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
315 if (created !== undefined) {
316 const done = yield* waitForOperation(created);
317 const createdName =
318 resourceNameFromOperation(done) ?? (yield* findOwned(id, news.dataset))?.name;
319 if (createdName !== undefined && createdName.length > 0) {
320 current = yield* waitUntilExists(createdName);
321 }
322 }
323 if (current === undefined) {
324 current = yield* findOwned(id, news.dataset);
325 }
326 }
327
328 if (current === undefined) {
329 return yield* new DatasetsDatasetVersionNotResolved({
330 resourceName: output?.name ?? `${news.dataset}/datasetVersions/-`,
331 });
332 }
333
334 const name = current.name ?? "";
335 if ((current.displayName ?? "") !== desiredDisplayName) {
336 current = yield* aiplatform.patchProjectsLocationsDatasetsDatasetVersions({
337 name,
338 updateMask: "display_name",
339 body: { name, displayName: desiredDisplayName },
340 });
341 }
342
343 return toAttrs(current, env.project);
344 }),
345
346 delete: Effect.fn(function* ({ output }) {
347 const existing = yield* getByName(output.name);
348 if (existing === undefined) return;
349 const operation = yield* aiplatform
350 .deleteDatasetsDatasetVersions({ name: output.name })
351 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
352 if (operation !== undefined) {
353 yield* waitForOperation(operation, { notFoundOk: true });
354 }
355 yield* waitUntilGone(output.name);
356 }),
357 });