@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 374 lines 12 KB
Raw
1 import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
2 import { deepEqual, isResolved } from "alchemy/Diff";
3 import { createPhysicalName } from "alchemy/PhysicalName";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Effect from "effect/Effect";
7 import * as Predicate from "effect/Predicate";
8 import * as Stream from "effect/Stream";
9 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10 import type { Providers } from "../Providers.ts";
11
12 const TypeId = "Cloudflare.AI.Dataset" as const;
13 type TypeId = typeof TypeId;
14
15 /**
16 * Log property a dataset filter can match on.
17 */
18 export type DatasetFilterKey =
19 | "created_at"
20 | "request_content_type"
21 | "response_content_type"
22 | "success"
23 | "cached"
24 | "provider"
25 | "model"
26 | "cost"
27 | "tokens"
28 | "tokens_in"
29 | "tokens_out"
30 | "duration"
31 | "feedback";
32
33 /**
34 * Comparison operator applied to a dataset filter.
35 */
36 export type DatasetFilterOperator = "eq" | "contains" | "lt" | "gt";
37
38 /**
39 * A single saved log filter on an AI Gateway dataset.
40 */
41 export type DatasetFilter = {
42 /**
43 * Log property to match on (e.g. `success`, `model`, `provider`).
44 */
45 key: DatasetFilterKey;
46 /**
47 * Comparison operator.
48 */
49 operator: DatasetFilterOperator;
50 /**
51 * Values to compare against. Multiple values act as an OR.
52 */
53 value: (string | number | boolean)[];
54 };
55
56 export type DatasetProps = {
57 /**
58 * The AI Gateway the dataset belongs to. Changing the gateway triggers a
59 * replacement.
60 */
61 gatewayId: string;
62 /**
63 * Human readable dataset name. If omitted, a unique name is generated from
64 * the app, stage, and logical ID.
65 * @default ${app}-${stage}-${id}
66 */
67 name?: string;
68 /**
69 * Whether the dataset is enabled (actively collecting matching logs).
70 * @default true
71 */
72 enable?: boolean;
73 /**
74 * Saved log filters defining which gateway logs the dataset captures.
75 * An empty array captures all logs.
76 */
77 filters: DatasetFilter[];
78 };
79
80 export type DatasetAttributes = {
81 /**
82 * Server-generated dataset identifier. Stable across updates.
83 */
84 datasetId: string;
85 /**
86 * The Cloudflare account the dataset belongs to.
87 */
88 accountId: string;
89 /**
90 * The AI Gateway the dataset belongs to.
91 */
92 gatewayId: string;
93 /**
94 * Human readable dataset name.
95 */
96 name: string;
97 /**
98 * Whether the dataset is enabled.
99 */
100 enable: boolean;
101 /**
102 * Saved log filters.
103 */
104 filters: DatasetFilter[];
105 /**
106 * When the dataset was created.
107 */
108 createdAt: string;
109 /**
110 * When the dataset was last modified.
111 */
112 modifiedAt: string;
113 };
114
115 export type Dataset = Resource<TypeId, DatasetProps, DatasetAttributes, never, Providers>;
116
117 /**
118 * A saved log filter ("dataset") on a Cloudflare.AI. Gateway.
119 *
120 * Datasets capture a slice of the gateway's request logs (filtered by
121 * provider, model, success, cost, tokens, etc.) and serve as the input to AI
122 * Gateway evaluations. Name, enablement, and filters are all mutable in
123 * place; only moving the dataset to a different gateway forces a replacement.
124 * ### Creating a Dataset
125 * **Example:** Capture successful requests
126 * ```typescript
127 * const gateway = yield* Cloudflare.AI.Gateway("Gateway");
128 *
129 * const dataset = yield* Cloudflare.AI.Dataset("SuccessLogs", {
130 * gatewayId: gateway.gatewayId,
131 * filters: [{ key: "success", operator: "eq", value: [true] }],
132 * });
133 * ```
134 *
135 * **Example:** Capture logs for a specific model
136 * ```typescript
137 * const dataset = yield* Cloudflare.AI.Dataset("LlamaLogs", {
138 * gatewayId: gateway.gatewayId,
139 * name: "llama-traffic",
140 * filters: [
141 * { key: "provider", operator: "eq", value: ["workers-ai"] },
142 * { key: "model", operator: "contains", value: ["llama"] },
143 * ],
144 * });
145 * ```
146 *
147 * ### Updating a Dataset
148 * **Example:** Disable collection without deleting
149 * ```typescript
150 * const dataset = yield* Cloudflare.AI.Dataset("SuccessLogs", {
151 * gatewayId: gateway.gatewayId,
152 * enable: false,
153 * filters: [{ key: "success", operator: "eq", value: [true] }],
154 * });
155 * ```
156 *
157 * @see https://developers.cloudflare.com/ai-gateway/evaluations/set-up-evaluations/
158 *
159 * @resource
160 * @product AI Gateway
161 * @category AI
162 */
163 export const Dataset = Resource<Dataset>(TypeId, {
164 aliases: ["Cloudflare.AiGateway.Dataset"],
165 });
166
167 /**
168 * Returns true if the given value is a Dataset resource.
169 */
170 export const isDataset = (value: unknown): value is Dataset =>
171 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
172
173 export const DatasetProvider = () =>
174 Provider.succeed(Dataset, {
175 stables: ["datasetId", "accountId", "gatewayId", "createdAt"],
176 diff: Effect.fn(function* ({ olds, news, output }) {
177 if (!isResolved(news)) return undefined;
178 const { accountId } = yield* yield* CloudflareEnvironment;
179 if ((output?.accountId ?? accountId) !== accountId) {
180 return { action: "replace" } as const;
181 }
182 // The gateway is a path parameter — a dataset cannot move between
183 // gateways in place. By diff time both sides are resolved strings.
184 const oldGatewayId = output?.gatewayId ?? olds?.gatewayId;
185 if (
186 typeof oldGatewayId === "string" &&
187 typeof news.gatewayId === "string" &&
188 oldGatewayId !== news.gatewayId
189 ) {
190 return { action: "replace" } as const;
191 }
192 return undefined;
193 }),
194 read: Effect.fn(function* ({ id, olds, output }) {
195 const { accountId } = yield* yield* CloudflareEnvironment;
196 const acct = output?.accountId ?? accountId;
197 const gatewayId = output?.gatewayId ?? (olds?.gatewayId as string | undefined);
198 if (gatewayId === undefined) return undefined;
199
200 if (output?.datasetId) {
201 const observed = yield* getDataset(acct, gatewayId, output.datasetId);
202 return observed ? toAttributes(observed, acct) : undefined;
203 }
204 // Cold read — recover from lost state by matching the deterministic
205 // physical name. Names are not unique on Cloudflare's side; an exact
206 // match on our generated/explicit name is the best identity we have.
207 const name = yield* createDatasetName(id, olds?.name);
208 const match = yield* findByName(acct, gatewayId, name);
209 return match ? toAttributes(match, acct) : undefined;
210 }),
211 reconcile: Effect.fn(function* ({ id, news, output }) {
212 const { accountId } = yield* yield* CloudflareEnvironment;
213 const gatewayId = news.gatewayId as string;
214 const name = yield* createDatasetName(id, news.name);
215 const desired = {
216 name,
217 enable: news.enable ?? true,
218 filters: news.filters ?? [],
219 };
220
221 // Observe — the datasetId cached on `output` is a hint, not a
222 // guarantee: a missing dataset falls through and we recreate.
223 let observed = output?.datasetId
224 ? yield* getDataset(output.accountId ?? accountId, gatewayId, output.datasetId)
225 : undefined;
226
227 if (!observed) {
228 // Ensure — greenfield (or out-of-band delete). Cloudflare enforces
229 // name uniqueness per gateway, so a name conflict (e.g. a leftover
230 // from an interrupted run or a create race) is converged by
231 // adopting the existing dataset and syncing it to the desired
232 // shape below instead of failing.
233 const created = yield* aiGateway
234 .createDataset({
235 accountId,
236 gatewayId,
237 ...desired,
238 })
239 .pipe(
240 Effect.catchTag("DatasetNameAlreadyExists", (originalError) =>
241 findByName(accountId, gatewayId, name).pipe(
242 Effect.flatMap((match) =>
243 match ? Effect.succeed(match) : Effect.fail(originalError),
244 ),
245 ),
246 ),
247 );
248 observed = created;
249 }
250
251 // Sync — diff observed cloud state against desired; the update API is
252 // a PUT that takes the full body, so send everything, but skip the
253 // call entirely on a no-op.
254 const observedShape = {
255 name: observed.name,
256 enable: observed.enable,
257 filters: normalizeFilters(observed.filters),
258 };
259 if (deepEqual(observedShape, desired)) {
260 return toAttributes(observed, accountId);
261 }
262 const updated = yield* aiGateway.updateDataset({
263 accountId,
264 gatewayId,
265 id: observed.id,
266 ...desired,
267 });
268 return toAttributes(updated, accountId);
269 }),
270 delete: Effect.fn(function* ({ output }) {
271 yield* aiGateway
272 .deleteDataset({
273 accountId: output.accountId,
274 gatewayId: output.gatewayId,
275 id: output.datasetId,
276 })
277 // Cloudflare reports both a missing dataset and a missing gateway
278 // with code 7002 on this endpoint — either way it's already gone.
279 .pipe(Effect.catchTag("DatasetNotFound", () => Effect.void));
280 }),
281 list: Effect.fn(function* () {
282 const { accountId } = yield* yield* CloudflareEnvironment;
283 // Datasets are scoped under a gateway and there is no account-wide
284 // dataset list, so fan out: enumerate every account gateway, then
285 // exhaustively list each gateway's datasets.
286 const gateways = yield* aiGateway.listAiGateways.pages({ accountId }).pipe(
287 Stream.runCollect,
288 Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.result ?? [])),
289 );
290 const rows = yield* Effect.forEach(
291 gateways,
292 (gateway) =>
293 aiGateway.listDatasets.pages({ accountId, gatewayId: gateway.id }).pipe(
294 Stream.runCollect,
295 Effect.map((chunk) =>
296 Array.from(chunk).flatMap((page) =>
297 (page.result ?? []).map((d) => toAttributes(d, accountId)),
298 ),
299 ),
300 // A gateway removed between enumeration and its dataset list
301 // is gone — skip it rather than failing the whole listing.
302 Effect.catchTag("GatewayNotFound", () => Effect.succeed([] as DatasetAttributes[])),
303 ),
304 { concurrency: 10 },
305 );
306 return rows.flat();
307 }),
308 });
309
310 /**
311 * Read a dataset by id, mapping "gone" (`DatasetNotFound`, Cloudflare error
312 * code 7002 — which also covers a deleted parent gateway) to `undefined`.
313 */
314 const getDataset = (accountId: string, gatewayId: string, id: string) =>
315 aiGateway
316 .getDataset({ accountId, gatewayId, id })
317 .pipe(Effect.catchTag("DatasetNotFound", () => Effect.undefined));
318
319 /**
320 * Find a dataset by exact name. Cloudflare's `name` query is a fuzzy match,
321 * so re-check exactly client-side. If several datasets carry the same name,
322 * pick the oldest for determinism. A missing parent gateway means the
323 * dataset is gone too.
324 */
325 const findByName = (accountId: string, gatewayId: string, name: string) =>
326 aiGateway.listDatasets.items({ accountId, gatewayId, name, perPage: 50 }).pipe(
327 Stream.filter((d) => d.name === name),
328 Stream.runCollect,
329 Effect.map((chunk) =>
330 Array.from(chunk)
331 .sort((a, b) => a.createdAt.localeCompare(b.createdAt))
332 .at(0),
333 ),
334 Effect.catchTag("GatewayNotFound", () => Effect.undefined),
335 );
336
337 const createDatasetName = (id: string, name: string | undefined) =>
338 Effect.gen(function* () {
339 return name ?? (yield* createPhysicalName({ id, lowercase: true }));
340 });
341
342 /**
343 * Strip readonly markers / open-union widening from the wire filters so they
344 * diff structurally against the user's props.
345 */
346 const normalizeFilters = (
347 filters: readonly {
348 key: string;
349 operator: string;
350 value: readonly (string | number | boolean)[];
351 }[],
352 ): DatasetFilter[] =>
353 filters.map((f) => ({
354 key: f.key as DatasetFilterKey,
355 operator: f.operator as DatasetFilterOperator,
356 value: [...f.value],
357 }));
358
359 const toAttributes = (
360 dataset:
361 | aiGateway.GetDatasetResponse
362 | aiGateway.CreateDatasetResponse
363 | aiGateway.UpdateDatasetResponse,
364 accountId: string,
365 ): DatasetAttributes => ({
366 datasetId: dataset.id,
367 accountId,
368 gatewayId: dataset.gatewayId,
369 name: dataset.name,
370 enable: dataset.enable,
371 filters: normalizeFilters(dataset.filters),
372 createdAt: dataset.createdAt,
373 modifiedAt: dataset.modifiedAt,
374 });