The best way to control Claude Code, Codex and more from any computer or phone

main
ts 347 lines 11.9 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import { listLocations } from "./names.ts";
13 import { resourceNameFromOperation, waitForOperation } from "./operations.ts";
14 import {
15 AiPlatformNotResolved,
16 AiPlatformStillExists,
17 collectPages,
18 encodeDisplayName,
19 hasDisplayNameOwnership,
20 jsonEqual,
21 locationParent,
22 normalizeLocation,
23 ownedById,
24 parseDisplayName,
25 parseResourceName,
26 } from "./shared.ts";
27
28 const COLLECTION = "onlineEvaluators";
29
30 export type OnlineEvaluatorRandomSampling = {
31 /** Percentage of traces to sample (`1`–`100`). */
32 percentage?: number;
33 };
34
35 export type OnlineEvaluatorConfig = {
36 /** Random sampling method. */
37 randomSampling?: OnlineEvaluatorRandomSampling;
38 /** Max evaluations per 10-minute run (`"0"` is unbounded). */
39 maxEvaluatedSamplesPerRun?: string;
40 };
41
42 export type MetricSource = {
43 /** Registered metric resource name. */
44 metricResourceName?: string;
45 };
46
47 export type CloudObservability = {
48 /** OpenTelemetry semantic convention version (e.g. `"1.39.0"`). */
49 openTelemetry?: { semconvVersion?: string };
50 /** Log view used to query logs. */
51 logView?: string;
52 /** Trace view used to query traces. */
53 traceView?: string;
54 };
55
56 export type OnlineEvaluatorProps = {
57 /**
58 * Vertex AI location. Immutable — changing it replaces the evaluator.
59 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
60 */
61 location?: string;
62 /**
63 * Human-readable name (max 63 characters). OnlineEvaluator has no
64 * labels field, so Alchemy ownership is stored in a compact `[alc …]`
65 * displayName prefix for `list` / nuke.
66 */
67 displayName?: string;
68 /**
69 * Agent resource the evaluator scores. Immutable.
70 */
71 agentResource: string;
72 /**
73 * Metric sources (max 25). At least one is required.
74 */
75 metricSources: MetricSource[];
76 /**
77 * Sampling configuration. Defaults to 10% random sampling.
78 */
79 config?: OnlineEvaluatorConfig;
80 /**
81 * Cloud Observability (Trace / Logging) data source.
82 */
83 cloudObservability?: CloudObservability;
84 };
85
86 export type OnlineEvaluator = Resource<
87 "GCP.AIPlatform.OnlineEvaluator",
88 OnlineEvaluatorProps,
89 {
90 /** Full resource name. */
91 name: string;
92 /** Evaluator id (last path segment). */
93 onlineEvaluatorId: string;
94 /** Project id. */
95 project: string;
96 /** Location id. */
97 location: string;
98 /** User display name with the Alchemy ownership prefix stripped. */
99 displayName: string | undefined;
100 /** Agent resource being evaluated. */
101 agentResource: string | undefined;
102 /** Server-reported state (`ACTIVE`, `SUSPENDED`, `FAILED`, …). */
103 state: string | undefined;
104 /** RFC3339 creation timestamp. */
105 createTime: string | undefined;
106 /** RFC3339 last-update timestamp. */
107 updateTime: string | undefined;
108 },
109 never,
110 Providers
111 >;
112
113 /**
114 * A Vertex AI Online Evaluator — periodically scores agent traces against
115 * registered metrics.
116 *
117 * OnlineEvaluator has no labels, so Alchemy stamps ownership into the
118 * display name. Changing `location` or `agentResource` replaces the
119 * evaluator. Display name, config, and metric sources update in place.
120 *
121 * ### Creating an Online Evaluator
122 * **Example:** Sample 10% of traces
123 * ```typescript
124 * const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
125 * agentResource: engine.name,
126 * metricSources: [{ metricResourceName: metric.name }],
127 * config: { randomSampling: { percentage: 10 } },
128 * });
129 * ```
130 *
131 * @resource
132 * @category AIPlatform
133 */
134 export const OnlineEvaluator = Resource<OnlineEvaluator>("GCP.AIPlatform.OnlineEvaluator");
135
136 export class OnlineEvaluatorNotResolved extends Data.TaggedError(
137 "GCP.AIPlatform.OnlineEvaluatorNotResolved",
138 )<{
139 resourceName: string;
140 }> {}
141
142 const defaultConfig = (
143 config: OnlineEvaluatorConfig | undefined,
144 ): aiplatform.GoogleCloudAiplatformV1OnlineEvaluatorConfig => ({
145 randomSampling: {
146 percentage: config?.randomSampling?.percentage ?? 10,
147 },
148 maxEvaluatedSamplesPerRun: config?.maxEvaluatedSamplesPerRun,
149 });
150
151 const toAttrs = (evaluator: aiplatform.GoogleCloudAiplatformV1OnlineEvaluator, project: string) => {
152 const name = evaluator.name ?? "";
153 const parsed = parseResourceName(name, COLLECTION);
154 const display = parseDisplayName(evaluator.displayName);
155 return {
156 name,
157 onlineEvaluatorId: parsed.id,
158 project: parsed.project || project,
159 location: parsed.location,
160 displayName: display.displayName,
161 agentResource: evaluator.agentResource,
162 state: evaluator.state,
163 createTime: evaluator.createTime,
164 updateTime: evaluator.updateTime,
165 };
166 };
167
168 const getByName = (name: string) =>
169 aiplatform
170 .getProjectsLocationsOnlineEvaluators({ name })
171 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
172
173 const waitUntilExists = (name: string) =>
174 getByName(name).pipe(
175 Effect.filterOrFail(
176 (evaluator): evaluator is aiplatform.GoogleCloudAiplatformV1OnlineEvaluator =>
177 evaluator !== undefined,
178 () => new AiPlatformNotResolved({ resourceName: name }),
179 ),
180 Effect.retry({
181 while: (error) => error._tag === "GCP.AIPlatform.NotResolved",
182 times: 8,
183 schedule: Schedule.spaced("2 seconds"),
184 }),
185 );
186
187 const waitUntilGone = (name: string) =>
188 getByName(name).pipe(
189 Effect.filterOrFail(
190 (evaluator) => evaluator === undefined,
191 () => new AiPlatformStillExists({ resourceName: name }),
192 ),
193 Effect.asVoid,
194 Effect.retry({
195 while: (error) => error._tag === "GCP.AIPlatform.StillExists",
196 times: 10,
197 schedule: Schedule.spaced("2 seconds"),
198 }),
199 );
200
201 const findOwned = (parent: string, id: string) =>
202 Effect.gen(function* () {
203 const pages = yield* collectPages(
204 aiplatform.listProjectsLocationsOnlineEvaluators.pages({
205 parent,
206 pageSize: 100,
207 }),
208 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
209 const evaluators = pages.flatMap((page) => page.onlineEvaluators ?? []);
210 for (const evaluator of evaluators) {
211 const parsed = parseDisplayName(evaluator.displayName);
212 if (yield* ownedById(id, parsed.labels)) {
213 return evaluator;
214 }
215 }
216 return undefined;
217 });
218
219 export const OnlineEvaluatorProvider = () =>
220 Provider.succeed(OnlineEvaluator, {
221 stables: ["name", "onlineEvaluatorId", "project", "location", "agentResource", "createTime"],
222
223 diff: Effect.fn(function* ({ news, olds, output }) {
224 const env = yield* GcpEnvironment.current;
225 if (!isResolved(news)) return undefined;
226 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
227 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
228 const agentChanged =
229 news.agentResource !== (olds?.agentResource ?? output?.agentResource ?? news.agentResource);
230 const replace = previousLocation !== nextLocation || (olds !== undefined && agentChanged);
231 if (!replace) return undefined;
232 return {
233 action: "replace" as const,
234 deleteFirst: previousLocation === nextLocation,
235 };
236 }),
237
238 read: Effect.fn(function* ({ id, olds, output }) {
239 const env = yield* GcpEnvironment.current;
240 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
241 const parent = locationParent(env.project, location);
242 const existing =
243 output?.name !== undefined ? yield* getByName(output.name) : yield* findOwned(parent, id);
244 if (existing === undefined) return undefined;
245 const attrs = toAttrs(existing, env.project);
246 const parsed = parseDisplayName(existing.displayName);
247 return (yield* hasAlchemyLabels(id, parsed.labels)) || (yield* ownedById(id, parsed.labels))
248 ? attrs
249 : Unowned(attrs);
250 }),
251
252 list: () =>
253 Effect.gen(function* () {
254 const env = yield* GcpEnvironment.current;
255 const pages = (yield* Effect.forEach(listLocations(env.region), (location) =>
256 collectPages(
257 aiplatform.listProjectsLocationsOnlineEvaluators.pages({
258 parent: locationParent(env.project, location),
259 pageSize: 100,
260 }),
261 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
262 )).flat();
263 return pages.flatMap((page) =>
264 (page.onlineEvaluators ?? [])
265 .filter((evaluator) => hasDisplayNameOwnership(evaluator.displayName))
266 .map((evaluator) => toAttrs(evaluator, env.project)),
267 );
268 }),
269
270 reconcile: Effect.fn(function* ({ id, news, output }) {
271 const env = yield* GcpEnvironment.current;
272 const location = normalizeLocation(news.location ?? output?.location, env.region);
273 const parent = locationParent(env.project, location);
274 const internal = yield* createInternalLabels(id);
275 const displayName = encodeDisplayName(internal, news.displayName);
276 const config = defaultConfig(news.config);
277
278 let current =
279 output?.name !== undefined ? yield* getByName(output.name) : yield* findOwned(parent, id);
280
281 if (current === undefined) {
282 const created = yield* aiplatform.createProjectsLocationsOnlineEvaluators({
283 parent,
284 body: {
285 displayName,
286 agentResource: news.agentResource,
287 metricSources: news.metricSources,
288 config,
289 cloudObservability: news.cloudObservability,
290 },
291 });
292 yield* waitForOperation(created, { alreadyExistsOk: true });
293 const createdName = resourceNameFromOperation(created);
294 current =
295 createdName !== undefined
296 ? yield* waitUntilExists(createdName)
297 : yield* findOwned(parent, id);
298 }
299
300 if (current === undefined || current.name === undefined) {
301 return yield* new OnlineEvaluatorNotResolved({
302 resourceName: output?.name ?? parent,
303 });
304 }
305
306 const observedName = current.name;
307 const observedDisplay = parseDisplayName(current.displayName);
308 const displayChanged = (observedDisplay.displayName ?? "") !== (news.displayName ?? "");
309 const configChanged = !jsonEqual(current.config, config);
310 const metricsChanged = !jsonEqual(current.metricSources, news.metricSources);
311
312 if (displayChanged || configChanged || metricsChanged) {
313 const updateMask = [
314 displayChanged ? "display_name" : undefined,
315 configChanged ? "config" : undefined,
316 metricsChanged ? "metric_sources" : undefined,
317 ].filter((field): field is string => field !== undefined);
318 const patched = yield* aiplatform.patchProjectsLocationsOnlineEvaluators({
319 name: observedName,
320 updateMask: updateMask.join(","),
321 body: {
322 name: observedName,
323 displayName,
324 config,
325 metricSources: news.metricSources,
326 },
327 });
328 yield* waitForOperation(patched);
329 current = yield* getByName(observedName);
330 }
331
332 if (current === undefined) {
333 return yield* new OnlineEvaluatorNotResolved({ resourceName: observedName });
334 }
335 return toAttrs(current, env.project);
336 }),
337
338 delete: Effect.fn(function* ({ output }) {
339 const operation = yield* aiplatform
340 .deleteProjectsLocationsOnlineEvaluators({ name: output.name })
341 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
342 if (operation !== undefined) {
343 yield* waitForOperation(operation, { notFoundOk: true });
344 }
345 yield* waitUntilGone(output.name);
346 }),
347 });