@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 335 lines 12.8 KB
Raw
1 import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
2 import { deepEqual, isResolved } from "alchemy/Diff";
3 import { createPhysicalName } from "alchemy/PhysicalName";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Effect from "effect/Effect";
7 import * as Predicate from "effect/Predicate";
8 import * as Stream from "effect/Stream";
9 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10 import type { Providers } from "../Providers.ts";
11
12 const TypeId = "Cloudflare.AI.Evaluation" as const;
13 type TypeId = typeof TypeId;
14
15 export type EvaluationProps = {
16 /**
17 * The AI Gateway the evaluation runs against. Changing the gateway
18 * triggers a replacement.
19 */
20 gatewayId: string;
21 /**
22 * Human readable evaluation name. If omitted, a unique name is generated
23 * from the app, stage, and logical ID. Evaluations are immutable —
24 * changing the name triggers a replacement.
25 * @default ${app}-${stage}-${id}
26 */
27 name?: string;
28 /**
29 * The datasets (saved log filters) the evaluation processes. Usually the
30 * `datasetId` attributes of `Dataset` resources on the same
31 * gateway. Changing the datasets triggers a replacement.
32 */
33 datasetIds: string[];
34 /**
35 * The evaluation types to run (e.g. speed, cost). Discover ids with
36 * `listEvaluationTypes` — the mandatory `speed` and `cost` types are
37 * account-global constants. Changing the types triggers a replacement.
38 */
39 evaluationTypeIds: string[];
40 };
41
42 export type EvaluationAttributes = {
43 /**
44 * Server-generated evaluation identifier. Stable for the lifetime of the
45 * evaluation.
46 */
47 evaluationId: string;
48 /**
49 * The Cloudflare account the evaluation belongs to.
50 */
51 accountId: string;
52 /**
53 * The AI Gateway the evaluation runs against.
54 */
55 gatewayId: string;
56 /**
57 * Human readable evaluation name.
58 */
59 name: string;
60 /**
61 * The dataset ids the evaluation processes.
62 */
63 datasetIds: string[];
64 /**
65 * The evaluation type ids the evaluation runs.
66 */
67 evaluationTypeIds: string[];
68 /**
69 * Whether Cloudflare has finished processing the evaluation. Evaluations
70 * are asynchronous jobs over logged traffic — this starts `false`.
71 */
72 processed: boolean;
73 /**
74 * Number of logs the evaluation covers.
75 */
76 totalLogs: number;
77 /**
78 * When the evaluation was created.
79 */
80 createdAt: string;
81 /**
82 * When the evaluation was last modified.
83 */
84 modifiedAt: string;
85 };
86
87 export type Evaluation = Resource<TypeId, EvaluationProps, EvaluationAttributes, never, Providers>;
88
89 /**
90 * An evaluation job on a Cloudflare.AI. Gateway.
91 *
92 * Evaluations measure performance (speed, cost, feedback) of the logged
93 * traffic captured by one or more datasets on a gateway. They are
94 * create-only on Cloudflare's side: any prop change replaces the
95 * evaluation with a fresh job.
96 * ### Creating an Evaluation
97 * **Example:** Evaluate a dataset for speed and cost
98 * ```typescript
99 * const gateway = yield* Cloudflare.AI.Gateway("Gateway");
100 *
101 * const dataset = yield* Cloudflare.AI.Dataset("SuccessLogs", {
102 * gatewayId: gateway.gatewayId,
103 * filters: [{ key: "success", operator: "eq", value: [true] }],
104 * });
105 *
106 * const types = yield* listEvaluationTypes(gateway.accountId);
107 * const evaluation = yield* Cloudflare.AI.Evaluation("Baseline", {
108 * gatewayId: gateway.gatewayId,
109 * datasetIds: [dataset.datasetId],
110 * evaluationTypeIds: types
111 * .filter((t) => t.mandatory)
112 * .map((t) => t.id),
113 * });
114 * ```
115 *
116 * @see https://developers.cloudflare.com/ai-gateway/evaluations/
117 *
118 * @resource
119 * @product AI Gateway
120 * @category AI
121 */
122 export const Evaluation = Resource<Evaluation>(TypeId, {
123 aliases: ["Cloudflare.AiGateway.Evaluation"],
124 });
125
126 /**
127 * Returns true if the given value is an Evaluation resource.
128 */
129 export const isEvaluation = (value: unknown): value is Evaluation =>
130 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
131
132 /**
133 * List the evaluation types available on the account (speed, cost,
134 * feedback, ...). The `mandatory` types must be included in every
135 * evaluation.
136 */
137 export const listEvaluationTypes = (accountId: string) =>
138 aiGateway.listEvaluationTypes({ accountId, perPage: 50 }).pipe(Effect.map((page) => page.result));
139
140 export const EvaluationProvider = () =>
141 Provider.succeed(Evaluation, {
142 stables: ["evaluationId", "accountId", "gatewayId", "createdAt"],
143 diff: Effect.fn(function* ({ news, output }) {
144 if (!isResolved(news)) return undefined;
145 const { accountId } = yield* yield* CloudflareEnvironment;
146 if ((output?.accountId ?? accountId) !== accountId) {
147 return { action: "replace" } as const;
148 }
149 if (output === undefined) return undefined;
150 // Evaluations have no update API — any change is a replacement.
151 // Auto-generated names are engine-owned: the deployed name stays
152 // authoritative even if the generator would name this id differently
153 // today. Only an explicit user-provided name can force a replace.
154 const newName = news.name ?? output.name;
155 if (
156 output.gatewayId !== news.gatewayId ||
157 output.name !== newName ||
158 !deepEqual(output.datasetIds, news.datasetIds) ||
159 !deepEqual(output.evaluationTypeIds, news.evaluationTypeIds)
160 ) {
161 return { action: "replace" } as const;
162 }
163 return undefined;
164 }),
165 read: Effect.fn(function* ({ id, olds, output }) {
166 const { accountId } = yield* yield* CloudflareEnvironment;
167 const acct = output?.accountId ?? accountId;
168 const gatewayId = output?.gatewayId ?? (olds?.gatewayId as string | undefined);
169 if (gatewayId === undefined) return undefined;
170 const known = {
171 datasetIds: output?.datasetIds ?? (olds?.datasetIds as string[] | undefined) ?? [],
172 evaluationTypeIds:
173 output?.evaluationTypeIds ?? (olds?.evaluationTypeIds as string[] | undefined) ?? [],
174 };
175
176 if (output?.evaluationId) {
177 const observed = yield* getEvaluation(acct, gatewayId, output.evaluationId);
178 return observed ? toAttributes(observed, acct, known) : undefined;
179 }
180 // Cold read — recover from lost state by matching the deterministic
181 // physical name. Names are not unique on Cloudflare's side; an exact
182 // match on our generated/explicit name is the best identity we have.
183 const name = yield* createEvaluationName(id, olds?.name);
184 const match = yield* findByName(acct, gatewayId, name);
185 return match ? toAttributes(match, acct, known) : undefined;
186 }),
187 reconcile: Effect.fn(function* ({ id, news, output }) {
188 const { accountId } = yield* yield* CloudflareEnvironment;
189 const gatewayId = news.gatewayId as string;
190 // Prefer the deployed name: regenerating would target a different
191 // resource if the generator's output for this id ever drifts.
192 const name = output?.name ?? (yield* createEvaluationName(id, news.name));
193 const datasetIds = news.datasetIds as string[];
194 const evaluationTypeIds = news.evaluationTypeIds;
195
196 // Observe — the evaluationId cached on `output` is a hint, not a
197 // guarantee: a missing evaluation falls through and we recreate.
198 const observed = output?.evaluationId
199 ? yield* getEvaluation(output.accountId ?? accountId, gatewayId, output.evaluationId)
200 : undefined;
201
202 if (observed) {
203 // Evaluations are immutable; diff routes every prop change to a
204 // replacement, so an observed evaluation is already converged.
205 return toAttributes(observed, accountId, {
206 datasetIds: output?.datasetIds ?? datasetIds,
207 evaluationTypeIds: output?.evaluationTypeIds ?? evaluationTypeIds,
208 });
209 }
210
211 // Ensure — greenfield (or out-of-band delete). Cloudflare enforces
212 // name uniqueness, so a conflict (e.g. a leftover from an interrupted
213 // run or a create race) is converged by adopting the existing
214 // evaluation by name instead of failing — evaluations are immutable,
215 // and diff routes any prop change to a replacement, so a name match
216 // is already the desired evaluation.
217 const created = yield* aiGateway
218 .createEvaluation({ accountId, gatewayId, name, datasetIds, evaluationTypeIds })
219 .pipe(
220 Effect.catchTag("EvaluationNameAlreadyExists", (originalError) =>
221 findByName(accountId, gatewayId, name).pipe(
222 Effect.flatMap((match) =>
223 match ? Effect.succeed(match) : Effect.fail(originalError),
224 ),
225 ),
226 ),
227 );
228 return toAttributes(created, accountId, { datasetIds, evaluationTypeIds });
229 }),
230 delete: Effect.fn(function* ({ output }) {
231 yield* aiGateway
232 .deleteEvaluation({
233 accountId: output.accountId,
234 gatewayId: output.gatewayId,
235 id: output.evaluationId,
236 })
237 // Cloudflare reports both a missing evaluation and a missing parent
238 // gateway with code 7002 — either way it's already gone.
239 .pipe(Effect.catchTag("EvaluationNotFound", () => Effect.void));
240 }),
241 // Evaluations are scoped under an AI Gateway, which itself is
242 // account-scoped. Fan out: exhaustively paginate every gateway in the
243 // account, then exhaustively paginate the evaluations under each gateway
244 // (bounded concurrency) and hydrate each into the same Attributes shape
245 // `read` produces (via `toAttributes`). A gateway deleted mid-enumeration
246 // returns an empty evaluation list, so no per-item not-found handling is
247 // required. We pass `[]` for known type ids: the listed evaluation only
248 // echoes type ids back once results exist, and `toAttributes` falls back
249 // to the result-derived ids otherwise.
250 list: Effect.fn(function* () {
251 const { accountId } = yield* yield* CloudflareEnvironment;
252 const gatewayIds = yield* aiGateway.listAiGateways.pages({ accountId }).pipe(
253 Stream.runCollect,
254 Effect.map((chunk) =>
255 Array.from(chunk).flatMap((page) => (page.result ?? []).map((gateway) => gateway.id)),
256 ),
257 );
258 const rows = yield* Effect.forEach(
259 gatewayIds,
260 (gatewayId) =>
261 aiGateway.listEvaluations.pages({ accountId, gatewayId }).pipe(
262 Stream.runCollect,
263 Effect.map((chunk) =>
264 Array.from(chunk).flatMap((page) =>
265 (page.result ?? []).map((evaluation) =>
266 toAttributes(evaluation, accountId, { datasetIds: [], evaluationTypeIds: [] }),
267 ),
268 ),
269 ),
270 ),
271 { concurrency: 10 },
272 );
273 return rows.flat();
274 }),
275 });
276
277 /**
278 * Read an evaluation by id, mapping "gone" (`EvaluationNotFound`,
279 * Cloudflare error code 7002 — which also covers a deleted parent gateway)
280 * to `undefined`.
281 */
282 const getEvaluation = (accountId: string, gatewayId: string, id: string) =>
283 aiGateway
284 .getEvaluation({ accountId, gatewayId, id })
285 .pipe(Effect.catchTag("EvaluationNotFound", () => Effect.undefined));
286
287 /**
288 * Find an evaluation by exact name. Cloudflare's `name` query is a fuzzy
289 * match, so re-check exactly client-side. If several evaluations carry the
290 * same name, pick the oldest for determinism. A missing parent gateway
291 * returns an empty list on this endpoint.
292 */
293 const findByName = (accountId: string, gatewayId: string, name: string) =>
294 aiGateway.listEvaluations.items({ accountId, gatewayId, name, perPage: 50 }).pipe(
295 Stream.filter((e) => e.name === name),
296 Stream.runCollect,
297 Effect.map((chunk) =>
298 Array.from(chunk)
299 .sort((a, b) => a.createdAt.localeCompare(b.createdAt))
300 .at(0),
301 ),
302 );
303
304 const createEvaluationName = (id: string, name: string | undefined) =>
305 Effect.gen(function* () {
306 return name ?? (yield* createPhysicalName({ id, lowercase: true }));
307 });
308
309 const toAttributes = (
310 evaluation:
311 | aiGateway.GetEvaluationResponse
312 | aiGateway.CreateEvaluationResponse
313 | aiGateway.ListEvaluationsResponse["result"][number],
314 accountId: string,
315 known: { datasetIds: string[]; evaluationTypeIds: string[] },
316 ): EvaluationAttributes => ({
317 evaluationId: evaluation.id,
318 accountId,
319 gatewayId: evaluation.gatewayId,
320 name: evaluation.name,
321 // The API does not always echo the datasets back (the create response
322 // returns none), so fall back to what we asked for.
323 datasetIds:
324 evaluation.datasets.length > 0 ? evaluation.datasets.map((d) => d.id) : known.datasetIds,
325 // The API only echoes evaluation type ids back once results exist, so
326 // prefer the observed result types and fall back to what we asked for.
327 evaluationTypeIds:
328 evaluation.results.length > 0
329 ? [...new Set(evaluation.results.map((r) => r.evaluationTypeId))]
330 : known.evaluationTypeIds,
331 processed: evaluation.processed,
332 totalLogs: evaluation.totalLogs,
333 createdAt: evaluation.createdAt,
334 modifiedAt: evaluation.modifiedAt,
335 });