@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 307 lines 10.4 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13 import type { Providers } from "../Providers.ts";
14 import {
15 hasAlchemyLabelMap,
16 normalizeLocation,
17 parseResourceName,
18 toPhysicalRfc1035,
19 userLabels,
20 } from "./helpers.ts";
21 import { waitForOperation } from "./operations.ts";
22
23 const MAX_NAME_LENGTH = 63;
24
25 export type EvaluationItemProps = {
26 /**
27 * Evaluation item id (the `{evaluation_item}` segment). Assigned by
28 * Vertex on create. Provide to target an existing item.
29 */
30 evaluationItemId?: string;
31 /**
32 * Region. Immutable — changing it replaces the item.
33 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
34 */
35 location?: string;
36 /**
37 * Display name. Defaults to a generated id when omitted.
38 */
39 displayName?: string;
40 /**
41 * Item type (`REQUEST` or `RESULT`). Immutable.
42 */
43 evaluationItemType:
44 | aiplatform.GoogleCloudAiplatformV1EvaluationItemEvaluationItemTypeEnum
45 | (string & {});
46 /**
47 * Cloud Storage object holding the request or response.
48 */
49 gcsUri?: string;
50 /**
51 * Inlined evaluation request (prompt, golden response, candidates).
52 */
53 evaluationRequest?: aiplatform.GoogleCloudAiplatformV1EvaluationRequest;
54 /**
55 * Arbitrary caller metadata.
56 */
57 metadata?: unknown;
58 /**
59 * User labels. Alchemy ownership labels are merged in automatically.
60 */
61 labels?: Record<string, string>;
62 };
63
64 export type EvaluationItem = Resource<
65 "GCP.AIPlatform.EvaluationItem",
66 EvaluationItemProps,
67 {
68 /** Full resource name. */
69 name: string;
70 /** Evaluation item id (last path segment). */
71 evaluationItemId: string;
72 /** Project id. */
73 project: string;
74 /** Location id. */
75 location: string;
76 /** Display name. */
77 displayName: string | undefined;
78 /** Item type. */
79 evaluationItemType: string | undefined;
80 /** GCS object URI, if set. */
81 gcsUri: string | undefined;
82 /** Inlined evaluation request, if set. */
83 evaluationRequest: aiplatform.GoogleCloudAiplatformV1EvaluationRequest | undefined;
84 /** Evaluation result (RESULT items). */
85 evaluationResponse: aiplatform.GoogleCloudAiplatformV1EvaluationResult | undefined;
86 /** Caller metadata. */
87 metadata: unknown;
88 /** User labels (Alchemy ownership labels stripped). */
89 labels: Record<string, string>;
90 /** Error, if any. */
91 error: aiplatform.GoogleRpcStatus | undefined;
92 /** RFC3339 creation timestamp. */
93 createTime: string | undefined;
94 },
95 never,
96 Providers
97 >;
98
99 /**
100 * A Vertex AI Evaluation Item — one request or result row used by
101 * Evaluation Sets and Evaluation Runs.
102 *
103 * Vertex assigns the resource id. There is no update API: changing type,
104 * location, display name, request, or GCS URI replaces the item. Labels
105 * are stamped at create time.
106 *
107 * ### Creating an Evaluation Item
108 * **Example:** Request item with an inlined prompt
109 * ```typescript
110 * const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
111 * evaluationItemType: "REQUEST",
112 * evaluationRequest: { prompt: { text: "What is 2+2?" } },
113 * labels: { env: "test" },
114 * });
115 * ```
116 *
117 * @resource
118 * @category AIPlatform
119 */
120 export const EvaluationItem = Resource<EvaluationItem>("GCP.AIPlatform.EvaluationItem");
121
122 export class EvaluationItemNotResolved extends Data.TaggedError(
123 "GCP.AIPlatform.EvaluationItemNotResolved",
124 )<{
125 resourceName: string;
126 }> {}
127
128 export class EvaluationItemStillExists extends Data.TaggedError(
129 "GCP.AIPlatform.EvaluationItemStillExists",
130 )<{
131 resourceName: string;
132 }> {}
133
134 const resourceName = (project: string, location: string, itemId: string) =>
135 `projects/${project}/locations/${location}/evaluationItems/${itemId}`;
136
137 const toAttrs = (item: aiplatform.GoogleCloudAiplatformV1EvaluationItem, project: string) => {
138 const name = item.name ?? "";
139 const parsed = parseResourceName(name, "evaluationItems");
140 return {
141 name,
142 evaluationItemId: parsed.id,
143 project: parsed.project || project,
144 location: parsed.location,
145 displayName: item.displayName,
146 evaluationItemType: item.evaluationItemType,
147 gcsUri: item.gcsUri,
148 evaluationRequest: item.evaluationRequest,
149 evaluationResponse: item.evaluationResponse,
150 metadata: item.metadata,
151 labels: userLabels(item.labels),
152 error: item.error,
153 createTime: item.createTime,
154 };
155 };
156
157 const getByName = (name: string) =>
158 aiplatform
159 .getProjectsLocationsEvaluationItems({ name })
160 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
161
162 const listOwned = (project: string, location = "-") =>
163 aiplatform.listProjectsLocationsEvaluationItems
164 .pages({
165 parent: `projects/${project}/locations/${location}`,
166 pageSize: 100,
167 })
168 .pipe(
169 Stream.flatMap((page) => Stream.fromIterable(page.evaluationItems ?? [])),
170 Stream.filter((item) => hasAlchemyLabelMap(item.labels)),
171 Stream.runCollect,
172 Effect.map((chunk) => Array.from(chunk)),
173 Effect.catchTag("NotFound", () => Effect.succeed([])),
174 );
175
176 const findOwned = (id: string, project: string, location?: string) =>
177 Effect.gen(function* () {
178 const items = yield* listOwned(project, location);
179 for (const item of items) {
180 if (yield* hasAlchemyLabels(id, tagRecord(item.labels))) {
181 return item;
182 }
183 }
184 return undefined;
185 });
186
187 const waitUntilGone = (name: string) =>
188 getByName(name).pipe(
189 Effect.flatMap((item) =>
190 item === undefined
191 ? Effect.void
192 : Effect.fail(new EvaluationItemStillExists({ resourceName: name })),
193 ),
194 Effect.retry({
195 while: (error) => error._tag === "GCP.AIPlatform.EvaluationItemStillExists",
196 times: 10,
197 schedule: Schedule.spaced("2 seconds"),
198 }),
199 );
200
201 export const EvaluationItemProvider = () =>
202 Provider.succeed(EvaluationItem, {
203 stables: [
204 "name",
205 "evaluationItemId",
206 "project",
207 "location",
208 "createTime",
209 "evaluationItemType",
210 ],
211
212 diff: Effect.fn(function* ({ news, olds, output }) {
213 const env = yield* GcpEnvironment.current;
214 if (!isResolved(news)) return undefined;
215 const previousId = olds?.evaluationItemId ?? output?.evaluationItemId;
216 const nextId = news.evaluationItemId ?? previousId;
217 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
218 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
219 const previousType = olds?.evaluationItemType ?? output?.evaluationItemType ?? "";
220 const nextType = news.evaluationItemType ?? previousType;
221 const previousDisplay = olds?.displayName ?? output?.displayName ?? "";
222 const nextDisplay = news.displayName ?? previousDisplay;
223 const replace =
224 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
225 previousLocation !== nextLocation ||
226 previousType !== nextType ||
227 (news.displayName !== undefined && nextDisplay !== previousDisplay);
228 if (!replace) return undefined;
229 return { action: "replace" as const, deleteFirst: false };
230 }),
231
232 read: Effect.fn(function* ({ id, olds, output }) {
233 const env = yield* GcpEnvironment.current;
234 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
235 const itemId = olds?.evaluationItemId ?? output?.evaluationItemId;
236 const name =
237 output?.name ?? (itemId ? resourceName(env.project, location, itemId) : undefined);
238 const existing = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
239 if (existing === undefined) return undefined;
240 const attrs = toAttrs(existing, env.project);
241 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
242 }),
243
244 list: () =>
245 Effect.gen(function* () {
246 const env = yield* GcpEnvironment.current;
247 const items = yield* listOwned(env.project);
248 return items.map((item) => toAttrs(item, env.project));
249 }),
250
251 reconcile: Effect.fn(function* ({ id, news, output }) {
252 const env = yield* GcpEnvironment.current;
253 const location = normalizeLocation(news.location ?? output?.location, env.region);
254 const itemId = news.evaluationItemId ?? output?.evaluationItemId;
255 const name =
256 output?.name ?? (itemId ? resourceName(env.project, location, itemId) : undefined);
257 const desiredLabels = {
258 ...toLabels(news.labels),
259 ...(yield* createInternalLabels(id)),
260 };
261 const displayName =
262 news.displayName ?? (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
263
264 let current = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
265
266 if (current === undefined) {
267 current = yield* aiplatform
268 .createProjectsLocationsEvaluationItems({
269 parent: `projects/${env.project}/locations/${location}`,
270 body: {
271 displayName,
272 evaluationItemType: news.evaluationItemType,
273 gcsUri: news.gcsUri,
274 evaluationRequest: news.evaluationRequest,
275 metadata: news.metadata,
276 labels: desiredLabels,
277 },
278 })
279 .pipe(Effect.catchTag("Conflict", () => findOwned(id, env.project, location)));
280 }
281
282 if (current === undefined) {
283 return yield* new EvaluationItemNotResolved({
284 resourceName: name ?? displayName,
285 });
286 }
287
288 return toAttrs(current, env.project);
289 }),
290
291 delete: Effect.fn(function* ({ output }) {
292 const operation = yield* aiplatform
293 .deleteProjectsLocationsEvaluationItems({ name: output.name })
294 .pipe(
295 Effect.catchTag("NotFound", () => Effect.undefined),
296 Effect.retry({
297 while: (error) => error._tag === "Conflict",
298 times: 8,
299 schedule: Schedule.spaced("5 seconds"),
300 }),
301 );
302 if (operation !== undefined) {
303 yield* waitForOperation(operation, { notFoundOk: true });
304 }
305 yield* waitUntilGone(output.name);
306 }),
307 });