@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 313 lines 11.1 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 hasAlchemyLabelMap,
16 normalizeLocation,
17 parseResourceName,
18 toPhysicalRfc1035,
19 userLabels,
20 } from "./helpers.ts";
21 import { waitForOperation } from "./operations.ts";
22
23 const MAX_NAME_LENGTH = 63;
24
25 export type EvaluationRunProps = {
26 /**
27 * Evaluation run id (the `{evaluation_run}` segment). Assigned by
28 * Vertex on create. Provide to target an existing run.
29 */
30 evaluationRunId?: string;
31 /**
32 * Region. Immutable — changing it replaces the run.
33 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
34 */
35 location?: string;
36 /**
37 * Display name. Defaults to a generated id when omitted. Immutable.
38 */
39 displayName?: string;
40 /**
41 * User labels. Alchemy ownership labels are merged in automatically.
42 */
43 labels?: Record<string, string>;
44 /**
45 * Arbitrary caller metadata.
46 */
47 metadata?: unknown;
48 /**
49 * Data source (an Evaluation Set name or a BigQuery request set).
50 * Immutable.
51 */
52 dataSource: aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource;
53 /**
54 * Candidate-to-inference config map. Immutable.
55 */
56 inferenceConfigs?: Record<string, aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfig>;
57 /**
58 * Metrics, rubric, and output configuration. Immutable.
59 */
60 evaluationConfig?: aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig;
61 };
62
63 export type EvaluationRun = Resource<
64 "GCP.AIPlatform.EvaluationRun",
65 EvaluationRunProps,
66 {
67 /** Full resource name. */
68 name: string;
69 /** Evaluation run id (last path segment). */
70 evaluationRunId: string;
71 /** Project id. */
72 project: string;
73 /** Location id. */
74 location: string;
75 /** Display name. */
76 displayName: string | undefined;
77 /** Run state. */
78 state: aiplatform.GoogleCloudAiplatformV1EvaluationRunStateEnum | (string & {}) | undefined;
79 /** Data source. */
80 dataSource: aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource | undefined;
81 /** Inference configs. */
82 inferenceConfigs: aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfigMap | undefined;
83 /** Evaluation config. */
84 evaluationConfig: aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig | undefined;
85 /** Snapshot evaluation set. */
86 evaluationSetSnapshot: string | undefined;
87 /** Results when the run succeeded. */
88 evaluationResults: aiplatform.GoogleCloudAiplatformV1EvaluationResults | undefined;
89 /** Caller metadata. */
90 metadata: unknown;
91 /** User labels (Alchemy ownership labels stripped). */
92 labels: Record<string, string>;
93 /** Error when FAILED or CANCELLED. */
94 error: aiplatform.GoogleRpcStatus | undefined;
95 /** RFC3339 creation timestamp. */
96 createTime: string | undefined;
97 /** RFC3339 completion timestamp. */
98 completionTime: string | undefined;
99 },
100 never,
101 Providers
102 >;
103
104 /**
105 * A Vertex AI Evaluation Run — one execution of metrics over a set of
106 * evaluation items or a BigQuery request set.
107 *
108 * Vertex assigns the resource id. Creating a run starts evaluation. There
109 * is no update API: changing location, display name, data source, or
110 * config replaces the run.
111 *
112 * ### Creating an Evaluation Run
113 * **Example:** Run metrics over an evaluation set
114 * ```typescript
115 * const run = yield* GCP.AIPlatform.EvaluationRun("Quality", {
116 * dataSource: { evaluationSet: evaluationSet.name },
117 * evaluationConfig: {
118 * metrics: [
119 * {
120 * metric: "instruction_following_v1",
121 * predefinedMetricSpec: { metricSpecName: "instruction_following_v1" },
122 * },
123 * ],
124 * },
125 * labels: { env: "test" },
126 * });
127 * ```
128 *
129 * @resource
130 * @category AIPlatform
131 */
132 export const EvaluationRun = Resource<EvaluationRun>("GCP.AIPlatform.EvaluationRun");
133
134 export class EvaluationRunNotResolved extends Data.TaggedError(
135 "GCP.AIPlatform.EvaluationRunNotResolved",
136 )<{
137 resourceName: string;
138 }> {}
139
140 export class EvaluationRunStillExists extends Data.TaggedError(
141 "GCP.AIPlatform.EvaluationRunStillExists",
142 )<{
143 resourceName: string;
144 }> {}
145
146 const resourceName = (project: string, location: string, runId: string) =>
147 `projects/${project}/locations/${location}/evaluationRuns/${runId}`;
148
149 const toAttrs = (run: aiplatform.GoogleCloudAiplatformV1EvaluationRun, project: string) => {
150 const name = run.name ?? "";
151 const parsed = parseResourceName(name, "evaluationRuns");
152 return {
153 name,
154 evaluationRunId: parsed.id,
155 project: parsed.project || project,
156 location: parsed.location,
157 displayName: run.displayName,
158 state: run.state,
159 dataSource: run.dataSource,
160 inferenceConfigs: run.inferenceConfigs,
161 evaluationConfig: run.evaluationConfig,
162 evaluationSetSnapshot: run.evaluationSetSnapshot,
163 evaluationResults: run.evaluationResults,
164 metadata: run.metadata,
165 labels: userLabels(run.labels),
166 error: run.error,
167 createTime: run.createTime,
168 completionTime: run.completionTime,
169 };
170 };
171
172 const getByName = (name: string) =>
173 aiplatform
174 .getProjectsLocationsEvaluationRuns({ name })
175 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
176
177 const listOwned = (project: string, location = "-") =>
178 aiplatform.listProjectsLocationsEvaluationRuns
179 .pages({
180 parent: `projects/${project}/locations/${location}`,
181 pageSize: 100,
182 })
183 .pipe(
184 Stream.flatMap((page) => Stream.fromIterable(page.evaluationRuns ?? [])),
185 Stream.filter((run) => hasAlchemyLabelMap(run.labels)),
186 Stream.runCollect,
187 Effect.map((chunk) => Array.from(chunk)),
188 Effect.catchTag("NotFound", () => Effect.succeed([])),
189 );
190
191 const findOwned = (id: string, project: string, location?: string) =>
192 Effect.gen(function* () {
193 const runs = yield* listOwned(project, location);
194 for (const run of runs) {
195 if (yield* hasAlchemyLabels(id, tagRecord(run.labels))) {
196 return run;
197 }
198 }
199 return undefined;
200 });
201
202 const waitUntilGone = (name: string) =>
203 getByName(name).pipe(
204 Effect.flatMap((run) =>
205 run === undefined
206 ? Effect.void
207 : Effect.fail(new EvaluationRunStillExists({ resourceName: name })),
208 ),
209 Effect.retry({
210 while: (error) => error._tag === "GCP.AIPlatform.EvaluationRunStillExists",
211 times: 10,
212 schedule: Schedule.spaced("2 seconds"),
213 }),
214 );
215
216 export const EvaluationRunProvider = () =>
217 Provider.succeed(EvaluationRun, {
218 stables: ["name", "evaluationRunId", "project", "location", "createTime"],
219
220 diff: Effect.fn(function* ({ news, olds, output }) {
221 const env = yield* GcpEnvironment.current;
222 if (!isResolved(news)) return undefined;
223 const previousId = olds?.evaluationRunId ?? output?.evaluationRunId;
224 const nextId = news.evaluationRunId ?? previousId;
225 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
226 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
227 const previousDisplay = olds?.displayName ?? output?.displayName ?? "";
228 const nextDisplay = news.displayName ?? previousDisplay;
229 const replace =
230 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
231 previousLocation !== nextLocation ||
232 (news.displayName !== undefined && nextDisplay !== previousDisplay);
233 if (!replace) return undefined;
234 return { action: "replace" as const, deleteFirst: false };
235 }),
236
237 read: Effect.fn(function* ({ id, olds, output }) {
238 const env = yield* GcpEnvironment.current;
239 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
240 const runId = olds?.evaluationRunId ?? output?.evaluationRunId;
241 const name = output?.name ?? (runId ? resourceName(env.project, location, runId) : undefined);
242 const existing = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
243 if (existing === undefined) return undefined;
244 const attrs = toAttrs(existing, env.project);
245 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
246 }),
247
248 list: () =>
249 Effect.gen(function* () {
250 const env = yield* GcpEnvironment.current;
251 const runs = yield* listOwned(env.project);
252 return runs.map((run) => toAttrs(run, env.project));
253 }),
254
255 reconcile: Effect.fn(function* ({ id, news, output }) {
256 const env = yield* GcpEnvironment.current;
257 const location = normalizeLocation(news.location ?? output?.location, env.region);
258 const runId = news.evaluationRunId ?? output?.evaluationRunId;
259 const name = output?.name ?? (runId ? resourceName(env.project, location, runId) : undefined);
260 const desiredLabels = {
261 ...toLabels(news.labels),
262 ...(yield* createInternalLabels(id)),
263 };
264 const displayName =
265 news.displayName ?? (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
266
267 let current = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
268
269 if (current === undefined) {
270 current = yield* aiplatform
271 .createProjectsLocationsEvaluationRuns({
272 parent: `projects/${env.project}/locations/${location}`,
273 body: {
274 displayName,
275 dataSource: news.dataSource,
276 inferenceConfigs: news.inferenceConfigs,
277 evaluationConfig: news.evaluationConfig,
278 metadata: news.metadata,
279 labels: desiredLabels,
280 },
281 })
282 .pipe(Effect.catchTag("Conflict", () => findOwned(id, env.project, location)));
283 }
284
285 if (current === undefined) {
286 return yield* new EvaluationRunNotResolved({
287 resourceName: name ?? displayName,
288 });
289 }
290
291 return toAttrs(current, env.project);
292 }),
293
294 delete: Effect.fn(function* ({ output }) {
295 // A running evaluation blocks the delete with BadRequest "There are
296 // other operations running on the EvaluationRun" until it finishes
297 // (cancel is unimplemented: HTTP 501).
298 const operation = yield* aiplatform
299 .deleteProjectsLocationsEvaluationRuns({ name: output.name })
300 .pipe(
301 Effect.catchTag("NotFound", () => Effect.undefined),
302 Effect.retry({
303 while: (error) => error._tag === "Conflict" || error._tag === "BadRequest",
304 times: 24,
305 schedule: Schedule.spaced("5 seconds"),
306 }),
307 );
308 if (operation !== undefined) {
309 yield* waitForOperation(operation, { notFoundOk: true });
310 }
311 yield* waitUntilGone(output.name);
312 }),
313 });