The best way to control Claude Code, Codex and more from any computer or phone

main
ts 358 lines 12.2 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import * as Stream from "effect/Stream";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import {
14 fingerprint,
15 normalizeLocation,
16 parseResourceName,
17 specifiedEquals,
18 toPhysicalRfc1035,
19 } from "./helpers.ts";
20 import { waitForOperation } from "./operations.ts";
21
22 const MAX_NAME_LENGTH = 63;
23
24 export type EvaluationSetProps = {
25 /**
26 * Evaluation set id (the `{evaluation_set}` segment). Assigned by
27 * Vertex on create. Provide to target an existing set.
28 */
29 evaluationSetId?: string;
30 /**
31 * Region. Immutable — changing it replaces the set.
32 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
33 */
34 location?: string;
35 /**
36 * Display name. Defaults to a generated id when omitted.
37 */
38 displayName?: string;
39 /**
40 * Evaluation item resource names that belong to this set.
41 */
42 evaluationItems: string[];
43 /**
44 * Static agent configs keyed by agent id.
45 */
46 agentConfigs?: Record<string, aiplatform.GoogleCloudAiplatformV1AgentConfig>;
47 /**
48 * Caller metadata. Evaluation Sets have no labels field, so Alchemy
49 * ownership keys (`alchemy-stack`, `alchemy-stage`, `alchemy-id`) are
50 * merged into this object for `list` / nuke.
51 */
52 metadata?: unknown;
53 };
54
55 export type EvaluationSet = Resource<
56 "GCP.AIPlatform.EvaluationSet",
57 EvaluationSetProps,
58 {
59 /** Full resource name. */
60 name: string;
61 /** Evaluation set id (last path segment). */
62 evaluationSetId: string;
63 /** Project id. */
64 project: string;
65 /** Location id. */
66 location: string;
67 /** Display name. */
68 displayName: string | undefined;
69 /** Member evaluation item names. */
70 evaluationItems: string[];
71 /** Agent configs. */
72 agentConfigs: aiplatform.GoogleCloudAiplatformV1AgentConfigMap | undefined;
73 /** Caller metadata with Alchemy ownership keys stripped. */
74 metadata: unknown;
75 /** RFC3339 creation timestamp. */
76 createTime: string | undefined;
77 /** RFC3339 last-update timestamp. */
78 updateTime: string | undefined;
79 },
80 never,
81 Providers
82 >;
83
84 /**
85 * A Vertex AI Evaluation Set — a collection of Evaluation Items evaluated
86 * together.
87 *
88 * Evaluation Sets have no labels field, so Alchemy stamps ownership into
89 * `metadata`. Member items and agent configs update in place; display
90 * name, metadata, and location changes replace the set.
91 *
92 * ### Creating an Evaluation Set
93 * **Example:** Group request items
94 * ```typescript
95 * const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
96 * evaluationItems: [item.name],
97 * });
98 * ```
99 *
100 * ### Updating an Evaluation Set
101 * **Example:** Add items
102 * ```typescript
103 * // Same logical id, changed props: the engine updates it in place.
104 * const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
105 * evaluationItems: [item.name, extra.name],
106 * });
107 * ```
108 *
109 * @resource
110 * @category AIPlatform
111 */
112 export const EvaluationSet = Resource<EvaluationSet>("GCP.AIPlatform.EvaluationSet");
113
114 export class EvaluationSetNotResolved extends Data.TaggedError(
115 "GCP.AIPlatform.EvaluationSetNotResolved",
116 )<{
117 resourceName: string;
118 }> {}
119
120 export class EvaluationSetStillExists extends Data.TaggedError(
121 "GCP.AIPlatform.EvaluationSetStillExists",
122 )<{
123 resourceName: string;
124 }> {}
125
126 const resourceName = (project: string, location: string, setId: string) =>
127 `projects/${project}/locations/${location}/evaluationSets/${setId}`;
128
129 const metadataObject = (value: unknown): Record<string, unknown> => {
130 if (value !== null && typeof value === "object" && !Array.isArray(value)) {
131 return { ...(value as Record<string, unknown>) };
132 }
133 if (value === undefined) return {};
134 return { value };
135 };
136
137 const encodeMetadata = (
138 ownership: Record<string, string>,
139 metadata: unknown,
140 ): Record<string, unknown> => ({
141 ...metadataObject(metadata),
142 ...ownership,
143 });
144
145 const stripOwnershipMeta = (metadata: unknown): unknown => {
146 const rest = Object.fromEntries(
147 Object.entries(metadataObject(metadata)).filter(([key]) => !key.startsWith("alchemy-")),
148 );
149 return Object.keys(rest).length === 0 ? undefined : rest;
150 };
151
152 const metadataLabels = (metadata: unknown): Record<string, string> => {
153 const out: Record<string, string> = {};
154 for (const [key, value] of Object.entries(metadataObject(metadata))) {
155 if (key.startsWith("alchemy-") && typeof value === "string") {
156 out[key] = value;
157 }
158 }
159 return out;
160 };
161
162 const hasOwnershipMarker = (metadata: unknown) => Object.keys(metadataLabels(metadata)).length > 0;
163
164 const sortedItems = (items: readonly string[] | undefined) =>
165 [...(items ?? [])].sort((left, right) => left.localeCompare(right));
166
167 const toAttrs = (set: aiplatform.GoogleCloudAiplatformV1EvaluationSet, project: string) => {
168 const name = set.name ?? "";
169 const parsed = parseResourceName(name, "evaluationSets");
170 return {
171 name,
172 evaluationSetId: parsed.id,
173 project: parsed.project || project,
174 location: parsed.location,
175 displayName: set.displayName,
176 evaluationItems: sortedItems(set.evaluationItems),
177 agentConfigs: set.agentConfigs,
178 metadata: stripOwnershipMeta(set.metadata),
179 createTime: set.createTime,
180 updateTime: set.updateTime,
181 };
182 };
183
184 const getByName = (name: string) =>
185 aiplatform
186 .getProjectsLocationsEvaluationSets({ name })
187 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
188
189 const listOwned = (project: string, location = "-") =>
190 aiplatform.listProjectsLocationsEvaluationSets
191 .pages({
192 parent: `projects/${project}/locations/${location}`,
193 pageSize: 100,
194 })
195 .pipe(
196 Stream.flatMap((page) => Stream.fromIterable(page.evaluationSets ?? [])),
197 Stream.filter((set) => hasOwnershipMarker(set.metadata)),
198 Stream.runCollect,
199 Effect.map((chunk) => Array.from(chunk)),
200 Effect.catchTag("NotFound", () => Effect.succeed([])),
201 );
202
203 const findOwned = (id: string, project: string, location?: string) =>
204 Effect.gen(function* () {
205 const sets = yield* listOwned(project, location);
206 for (const set of sets) {
207 if (yield* hasAlchemyLabels(id, metadataLabels(set.metadata))) {
208 return set;
209 }
210 }
211 return undefined;
212 });
213
214 const waitUntilGone = (name: string) =>
215 getByName(name).pipe(
216 Effect.flatMap((set) =>
217 set === undefined
218 ? Effect.void
219 : Effect.fail(new EvaluationSetStillExists({ resourceName: name })),
220 ),
221 Effect.retry({
222 while: (error) => error._tag === "GCP.AIPlatform.EvaluationSetStillExists",
223 times: 10,
224 schedule: Schedule.spaced("2 seconds"),
225 }),
226 );
227
228 export const EvaluationSetProvider = () =>
229 Provider.succeed(EvaluationSet, {
230 stables: ["name", "evaluationSetId", "project", "location", "createTime"],
231
232 diff: Effect.fn(function* ({ news, olds, output }) {
233 const env = yield* GcpEnvironment.current;
234 if (!isResolved(news)) return undefined;
235 const previousId = olds?.evaluationSetId ?? output?.evaluationSetId;
236 const nextId = news.evaluationSetId ?? previousId;
237 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
238 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
239 const displayNameChanged = olds !== undefined && news.displayName !== olds.displayName;
240 const metadataChanged =
241 olds !== undefined &&
242 fingerprint(news.metadata ?? null) !== fingerprint(olds.metadata ?? null);
243 const replace =
244 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
245 previousLocation !== nextLocation ||
246 displayNameChanged ||
247 metadataChanged;
248 if (!replace) return undefined;
249 return { action: "replace" as const, deleteFirst: false };
250 }),
251
252 read: Effect.fn(function* ({ id, olds, output }) {
253 const env = yield* GcpEnvironment.current;
254 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
255 const setId = olds?.evaluationSetId ?? output?.evaluationSetId;
256 const name = output?.name ?? (setId ? resourceName(env.project, location, setId) : undefined);
257 const existing = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
258 if (existing === undefined) return undefined;
259 const attrs = toAttrs(existing, env.project);
260 return (yield* hasAlchemyLabels(id, metadataLabels(existing.metadata)))
261 ? attrs
262 : Unowned(attrs);
263 }),
264
265 list: () =>
266 Effect.gen(function* () {
267 const env = yield* GcpEnvironment.current;
268 const sets = yield* listOwned(env.project);
269 return sets.map((set) => toAttrs(set, env.project));
270 }),
271
272 reconcile: Effect.fn(function* ({ id, news, output }) {
273 const env = yield* GcpEnvironment.current;
274 const location = normalizeLocation(news.location ?? output?.location, env.region);
275 const setId = news.evaluationSetId ?? output?.evaluationSetId;
276 const name = output?.name ?? (setId ? resourceName(env.project, location, setId) : undefined);
277 const ownership = yield* createInternalLabels(id);
278 const desiredMetadata = encodeMetadata(ownership, news.metadata);
279 const displayName =
280 news.displayName ?? (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
281 const evaluationItems = sortedItems(news.evaluationItems);
282
283 let current = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
284
285 if (current === undefined) {
286 current = yield* aiplatform
287 .createProjectsLocationsEvaluationSets({
288 parent: `projects/${env.project}/locations/${location}`,
289 body: {
290 displayName,
291 evaluationItems,
292 agentConfigs: news.agentConfigs,
293 metadata: desiredMetadata,
294 },
295 })
296 .pipe(Effect.catchTag("Conflict", () => findOwned(id, env.project, location)));
297 }
298
299 if (current === undefined) {
300 return yield* new EvaluationSetNotResolved({
301 resourceName: name ?? displayName,
302 });
303 }
304
305 const resolvedName = current.name ?? name ?? "";
306 // Only `evaluation_items` and `agent_configs` are mutable; display
307 // name and metadata changes replace the set (see `diff`).
308 const itemsChanged =
309 fingerprint(sortedItems(current.evaluationItems)) !== fingerprint(evaluationItems);
310 const agentChanged =
311 news.agentConfigs !== undefined &&
312 !specifiedEquals(news.agentConfigs, current.agentConfigs);
313
314 if (itemsChanged || agentChanged) {
315 const updateMask = [
316 itemsChanged ? "evaluation_items" : undefined,
317 agentChanged ? "agent_configs" : undefined,
318 ].filter((field): field is string => field !== undefined);
319
320 current = yield* aiplatform
321 .patchProjectsLocationsEvaluationSets({
322 name: resolvedName,
323 updateMask: updateMask.join(","),
324 body: {
325 name: resolvedName,
326 evaluationItems,
327 agentConfigs: news.agentConfigs,
328 },
329 })
330 .pipe(
331 Effect.retry({
332 while: (error) => error._tag === "Conflict",
333 times: 8,
334 schedule: Schedule.spaced("5 seconds"),
335 }),
336 );
337 }
338
339 return toAttrs(current, env.project);
340 }),
341
342 delete: Effect.fn(function* ({ output }) {
343 const operation = yield* aiplatform
344 .deleteProjectsLocationsEvaluationSets({ name: output.name })
345 .pipe(
346 Effect.catchTag("NotFound", () => Effect.undefined),
347 Effect.retry({
348 while: (error) => error._tag === "Conflict",
349 times: 8,
350 schedule: Schedule.spaced("5 seconds"),
351 }),
352 );
353 if (operation !== undefined) {
354 yield* waitForOperation(operation, { notFoundOk: true });
355 }
356 yield* waitUntilGone(output.name);
357 }),
358 });