The best way to control Claude Code, Codex and more from any computer or phone

main
ts 590 lines 18.1 KB
Raw
1 import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
2 import { deepEqual, isResolved } from "alchemy/Diff";
3 import { createPhysicalName } from "alchemy/PhysicalName";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Effect from "effect/Effect";
7 import * as Predicate from "effect/Predicate";
8 import * as Stream from "effect/Stream";
9 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10 import type { Providers } from "../Providers.ts";
11
12 const TypeId = "Cloudflare.AI.DynamicRouting" as const;
13 type TypeId = typeof TypeId;
14
15 /**
16 * Reference to another element in the route graph.
17 */
18 export type RouteEdge = {
19 /**
20 * The `id` of the element the edge points at.
21 */
22 elementId: string;
23 };
24
25 /**
26 * Entry point of the route graph. Every route has exactly one.
27 */
28 export type RouteStartElement = {
29 /**
30 * Unique element identifier within the route graph.
31 */
32 id: string;
33 type: "start";
34 /**
35 * The element executed first.
36 */
37 outputs: { next: RouteEdge };
38 };
39
40 /**
41 * Branches the request based on conditions over request metadata.
42 */
43 export type RouteConditionalElement = {
44 /**
45 * Unique element identifier within the route graph.
46 */
47 id: string;
48 type: "conditional";
49 /**
50 * Conditions evaluated against the request.
51 */
52 properties: { conditions?: unknown };
53 /**
54 * Edges taken when the conditions evaluate true / false.
55 */
56 outputs: { true: RouteEdge; false: RouteEdge };
57 };
58
59 /**
60 * Splits traffic across multiple edges by percentage.
61 */
62 export type RoutePercentageElement = {
63 /**
64 * Unique element identifier within the route graph.
65 */
66 id: string;
67 type: "percentage";
68 /**
69 * Percentage-weighted edges.
70 */
71 outputs: Record<string, unknown>;
72 };
73
74 /**
75 * Applies a rate limit; requests over the limit take the fallback edge.
76 */
77 export type RouteRateElement = {
78 /**
79 * Unique element identifier within the route graph.
80 */
81 id: string;
82 type: "rate";
83 properties: {
84 /**
85 * Key the limit is bucketed by.
86 */
87 key: string;
88 /**
89 * Maximum count/cost allowed inside the window.
90 */
91 limit: number;
92 /**
93 * Whether the limit counts requests or cost.
94 */
95 limitType: "count" | "cost";
96 /**
97 * Window size in seconds.
98 */
99 window: number;
100 };
101 /**
102 * Edges taken when under (success) or over (fallback) the limit.
103 */
104 outputs: { success: RouteEdge; fallback: RouteEdge };
105 };
106
107 /**
108 * Sends the request to a provider/model; failures take the fallback edge.
109 */
110 export type RouteModelElement = {
111 /**
112 * Unique element identifier within the route graph.
113 */
114 id: string;
115 type: "model";
116 properties: {
117 /**
118 * Provider slug (e.g. `workers-ai`, `openai`, `anthropic`).
119 */
120 provider: string;
121 /**
122 * Model identifier (e.g. `@cf/meta/llama-3.1-8b-instruct`).
123 */
124 model: string;
125 /**
126 * Number of retries before taking the fallback edge.
127 */
128 retries: number;
129 /**
130 * Request timeout in milliseconds.
131 */
132 timeout: number;
133 };
134 /**
135 * Edges taken on success / failure.
136 */
137 outputs: { success: RouteEdge; fallback: RouteEdge };
138 };
139
140 /**
141 * Terminal element of the route graph.
142 */
143 export type RouteEndElement = {
144 /**
145 * Unique element identifier within the route graph.
146 */
147 id: string;
148 type: "end";
149 /**
150 * Always empty.
151 */
152 outputs: Record<string, never>;
153 };
154
155 /**
156 * A node in an AI Gateway dynamic routing graph. A well-formed graph starts
157 * at a `start` element and every path terminates at an `end` element.
158 */
159 export type RouteElement =
160 | RouteStartElement
161 | RouteConditionalElement
162 | RoutePercentageElement
163 | RouteRateElement
164 | RouteModelElement
165 | RouteEndElement;
166
167 export type GatewayDynamicRoutingProps = {
168 /**
169 * The AI Gateway the route belongs to. Changing the gateway triggers a
170 * replacement.
171 */
172 gatewayId: string;
173 /**
174 * Route name, unique within the gateway. If omitted, a unique name is
175 * generated from the app, stage, and logical ID. Renames are applied in
176 * place.
177 * @default ${app}-${stage}-${id}
178 */
179 name?: string;
180 /**
181 * The element graph describing how requests are routed. Changing the
182 * graph creates a new route version and deploys it.
183 */
184 elements: RouteElement[];
185 };
186
187 export type GatewayDynamicRoutingAttributes = {
188 /**
189 * Server-generated route identifier. Stable across updates.
190 */
191 routeId: string;
192 /**
193 * The Cloudflare account the route belongs to.
194 */
195 accountId: string;
196 /**
197 * The AI Gateway the route belongs to.
198 */
199 gatewayId: string;
200 /**
201 * Route name.
202 */
203 name: string;
204 /**
205 * The element graph of the currently deployed route version.
206 */
207 elements: RouteElement[];
208 /**
209 * Identifier of the currently deployed route version.
210 */
211 versionId: string;
212 /**
213 * Identifier of the active deployment.
214 */
215 deploymentId: string;
216 /**
217 * When the route was created.
218 */
219 createdAt: string;
220 /**
221 * When the route was last modified.
222 */
223 modifiedAt: string;
224 };
225
226 export type GatewayDynamicRouting = Resource<
227 TypeId,
228 GatewayDynamicRoutingProps,
229 GatewayDynamicRoutingAttributes,
230 never,
231 Providers
232 >;
233
234 /**
235 * A dynamic routing configuration ("route") on a Cloudflare.AI. Gateway.
236 *
237 * Dynamic routing models request handling as a graph of elements — start,
238 * conditional, percentage split, rate limit, model, and end nodes — so a
239 * single gateway endpoint can A/B test models, enforce per-user budgets, and
240 * fall back between providers without app changes.
241 *
242 * Cloudflare versions route configurations: changing `elements` creates a
243 * new version and deploys it; the reconciler also re-deploys when the live
244 * deployed version drifts from the desired graph. Renames are applied in
245 * place; only moving the route to a different gateway forces a replacement.
246 * ### Creating a Route
247 * **Example:** Route all traffic to one model
248 * ```typescript
249 * const gateway = yield* Cloudflare.AI.Gateway("Gateway");
250 *
251 * const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
252 * gatewayId: gateway.gatewayId,
253 * elements: [
254 * { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
255 * {
256 * id: "model",
257 * type: "model",
258 * properties: {
259 * provider: "workers-ai",
260 * model: "@cf/meta/llama-3.1-8b-instruct",
261 * retries: 1,
262 * timeout: 30000,
263 * },
264 * outputs: {
265 * success: { elementId: "end" },
266 * fallback: { elementId: "end" },
267 * },
268 * },
269 * { id: "end", type: "end", outputs: {} },
270 * ],
271 * });
272 * ```
273 *
274 * ### Updating a Route
275 * **Example:** Change the model — creates and deploys a new version
276 * ```typescript
277 * const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
278 * gatewayId: gateway.gatewayId,
279 * elements: [
280 * { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
281 * {
282 * id: "model",
283 * type: "model",
284 * properties: {
285 * provider: "workers-ai",
286 * model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
287 * retries: 2,
288 * timeout: 60000,
289 * },
290 * outputs: {
291 * success: { elementId: "end" },
292 * fallback: { elementId: "end" },
293 * },
294 * },
295 * { id: "end", type: "end", outputs: {} },
296 * ],
297 * });
298 * ```
299 *
300 * @see https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/
301 *
302 * @resource
303 * @product AI Gateway
304 * @category AI
305 */
306 export const GatewayDynamicRouting = Resource<GatewayDynamicRouting>(TypeId, {
307 aliases: ["Cloudflare.AiGateway.DynamicRouting"],
308 });
309
310 /**
311 * Returns true if the given value is a GatewayDynamicRouting resource.
312 */
313 export const isDynamicRouting = (value: unknown): value is GatewayDynamicRouting =>
314 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
315
316 export const DynamicRoutingProvider = () =>
317 Provider.succeed(GatewayDynamicRouting, {
318 stables: ["routeId", "accountId", "gatewayId", "createdAt"],
319 diff: Effect.fn(function* ({ olds, news, output }) {
320 if (!isResolved(news)) return undefined;
321 const { accountId } = yield* yield* CloudflareEnvironment;
322 if ((output?.accountId ?? accountId) !== accountId) {
323 return { action: "replace" } as const;
324 }
325 // The gateway is a path parameter — a route cannot move between
326 // gateways in place. By diff time both sides are resolved strings.
327 const oldGatewayId = output?.gatewayId ?? olds?.gatewayId;
328 if (
329 typeof oldGatewayId === "string" &&
330 typeof news.gatewayId === "string" &&
331 oldGatewayId !== news.gatewayId
332 ) {
333 return { action: "replace" } as const;
334 }
335 return undefined;
336 }),
337 read: Effect.fn(function* ({ id, olds, output }) {
338 const { accountId } = yield* yield* CloudflareEnvironment;
339 const acct = output?.accountId ?? accountId;
340 const gatewayId = output?.gatewayId ?? (olds?.gatewayId as string | undefined);
341 if (gatewayId === undefined) return undefined;
342
343 if (output?.routeId) {
344 const observed = yield* getRoute(acct, gatewayId, output.routeId);
345 return observed ? toAttributes(observed, acct) : undefined;
346 }
347 // Cold read — recover from lost state by matching the deterministic
348 // physical name. Route names are unique within a gateway, so an exact
349 // match identifies the route.
350 const name = yield* createRouteName(id, olds?.name);
351 const match = yield* findByName(acct, gatewayId, name);
352 if (match) {
353 const observed = yield* getRoute(acct, gatewayId, match.id);
354 return observed ? toAttributes(observed, acct) : undefined;
355 }
356 return undefined;
357 }),
358 reconcile: Effect.fn(function* ({ id, news, output }) {
359 const { accountId } = yield* yield* CloudflareEnvironment;
360 const gatewayId = news.gatewayId as string;
361 const name = yield* createRouteName(id, news.name);
362 const desired = news.elements ?? [];
363
364 // Observe — the routeId cached on `output` is a hint, not a
365 // guarantee: a missing route falls through and we recreate.
366 const observed = output?.routeId
367 ? yield* getRoute(output.accountId ?? accountId, gatewayId, output.routeId)
368 : undefined;
369
370 // Ensure — create if missing. Route names are unique per gateway:
371 // tolerate the `RouteAlreadyExists` race (a peer reconciler created
372 // it concurrently, or state persistence failed after a previous
373 // create) by recovering the existing route by name.
374 const routeId =
375 observed?.id ??
376 (yield* aiGateway
377 .createDynamicRouting({
378 accountId,
379 gatewayId,
380 name,
381 elements: desired,
382 })
383 .pipe(
384 Effect.catchTag("RouteAlreadyExists", (error) =>
385 findByName(accountId, gatewayId, name).pipe(
386 Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
387 ),
388 ),
389 Effect.map((route) => route.id),
390 ));
391
392 // Sync — observed cloud state is the diff baseline. Creation above
393 // auto-deploys version 1, so a fresh `get` covers all paths.
394 const current = yield* aiGateway.getDynamicRouting({
395 accountId,
396 gatewayId,
397 id: routeId,
398 });
399
400 if (current.name !== name) {
401 yield* aiGateway
402 .patchDynamicRouting({
403 accountId,
404 gatewayId,
405 id: routeId,
406 name,
407 })
408 .pipe(
409 // Cloudflare does not cascade-delete a gateway's routes when the
410 // gateway itself is deleted: recreating a gateway with the same
411 // id resurrects its old route rows, and a rename onto one of
412 // those ghosts trips the backing `UNIQUE(routes.name,
413 // routes.gateway_id)` constraint (leaked as a 500, typed as
414 // `RouteAlreadyExists` via a distilled patch). Route names are
415 // unique and deterministic, so the holder is a stale duplicate
416 // of this logical resource — delete it and retry the rename.
417 Effect.catchTag("RouteAlreadyExists", (error) =>
418 Effect.gen(function* () {
419 const holder = yield* findByName(accountId, gatewayId, name);
420 if (holder === undefined || holder.id === routeId) {
421 return yield* error;
422 }
423 yield* aiGateway
424 .deleteDynamicRouting({
425 accountId,
426 gatewayId,
427 id: holder.id,
428 })
429 .pipe(Effect.catchTag("RouteNotFound", () => Effect.void));
430 yield* aiGateway.patchDynamicRouting({
431 accountId,
432 gatewayId,
433 id: routeId,
434 name,
435 });
436 }),
437 ),
438 );
439 }
440
441 // `get` returns the *deployed* version's graph, so an undeployed
442 // draft version naturally shows up as drift and gets re-deployed.
443 const observedElements = elementsOf(current);
444 if (!deepEqual(observedElements, desired)) {
445 const version = yield* aiGateway.createVersionDynamicRouting({
446 accountId,
447 gatewayId,
448 id: routeId,
449 elements: desired,
450 });
451 yield* aiGateway.createDeploymentDynamicRouting({
452 accountId,
453 gatewayId,
454 id: routeId,
455 versionId: version.versionId,
456 });
457 }
458
459 // Return — re-read the final state so attributes reflect the
460 // deployed version, not stale intermediate responses.
461 const final = yield* aiGateway.getDynamicRouting({
462 accountId,
463 gatewayId,
464 id: routeId,
465 });
466 return toAttributes(final, accountId);
467 }),
468 delete: Effect.fn(function* ({ output }) {
469 yield* aiGateway
470 .deleteDynamicRouting({
471 accountId: output.accountId,
472 gatewayId: output.gatewayId,
473 id: output.routeId,
474 })
475 .pipe(
476 // Route already gone — or the whole parent gateway is gone.
477 Effect.catchTags({
478 RouteNotFound: () => Effect.void,
479 GatewayNotFound: () => Effect.void,
480 }),
481 );
482 }),
483 list: Effect.fn(function* () {
484 const { accountId } = yield* yield* CloudflareEnvironment;
485 // Routes are scoped under a gateway and there is no account-wide
486 // route list, so fan out: enumerate every account gateway, then
487 // exhaustively list each gateway's routes.
488 const gateways = yield* aiGateway.listAiGateways.pages({ accountId }).pipe(
489 Stream.runCollect,
490 Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.result ?? [])),
491 );
492 const rows = yield* Effect.forEach(
493 gateways,
494 (gateway) =>
495 listRoutes(accountId, gateway.id).pipe(
496 // A gateway removed between enumeration and its route list is
497 // gone — skip it rather than failing the whole listing.
498 Effect.catchTag("GatewayNotFound", () =>
499 Effect.succeed([] as GatewayDynamicRoutingAttributes[]),
500 ),
501 ),
502 { concurrency: 10 },
503 );
504 return rows.flat();
505 }),
506 });
507
508 /**
509 * Exhaustively list every route in a gateway. `listDynamicRoutings` is not a
510 * paginated distilled operation (the Cloudflare response carries only
511 * `page`/`per_page`, no total), so page manually until a short page signals
512 * the end.
513 *
514 * The list endpoint omits each route's element graph — its items carry only
515 * deployment/version metadata, no `version.data` — so hydrate every route with
516 * a per-route `get` to populate `elements` into the exact `read` shape.
517 */
518 const listRoutes = (accountId: string, gatewayId: string) =>
519 Effect.gen(function* () {
520 const perPage = 50;
521 const ids: string[] = [];
522 for (let page = 1; ; page++) {
523 const response = yield* aiGateway.listDynamicRoutings({
524 accountId,
525 gatewayId,
526 page,
527 perPage,
528 });
529 const routes = response.data.routes;
530 for (const route of routes) {
531 ids.push(route.id);
532 }
533 if (routes.length < perPage) break;
534 }
535 const hydrated = yield* Effect.forEach(ids, (id) => getRoute(accountId, gatewayId, id), {
536 concurrency: 10,
537 });
538 return hydrated.filter((r) => r !== undefined).map((route) => toAttributes(route, accountId));
539 });
540
541 /**
542 * Read a route by id, mapping "gone" (`RouteNotFound`, Cloudflare error code
543 * 7005, or `GatewayNotFound`, code 7002, when the parent gateway was
544 * deleted) to `undefined`.
545 */
546 const getRoute = (accountId: string, gatewayId: string, id: string) =>
547 aiGateway.getDynamicRouting({ accountId, gatewayId, id }).pipe(
548 Effect.catchTags({
549 RouteNotFound: () => Effect.undefined,
550 GatewayNotFound: () => Effect.undefined,
551 }),
552 );
553
554 /**
555 * Find a route by exact name. Route names are unique within a gateway. A
556 * missing parent gateway means the route is gone too.
557 */
558 const findByName = (accountId: string, gatewayId: string, name: string) =>
559 aiGateway.listDynamicRoutings({ accountId, gatewayId, perPage: 50 }).pipe(
560 Effect.map((list) => list.data.routes.find((r) => r.name === name)),
561 Effect.catchTag("GatewayNotFound", () => Effect.undefined),
562 );
563
564 const createRouteName = (id: string, name: string | undefined) =>
565 Effect.gen(function* () {
566 return name ?? (yield* createPhysicalName({ id, lowercase: true }));
567 });
568
569 /**
570 * The deployed element graph. Cloudflare returns it as `version.data` (the
571 * distilled schema types it `unknown` because the public spec wrongly
572 * declares it a string).
573 */
574 const elementsOf = (route: aiGateway.GetDynamicRoutingResponse): RouteElement[] =>
575 (route.version.data ?? []) as RouteElement[];
576
577 const toAttributes = (
578 route: aiGateway.GetDynamicRoutingResponse,
579 accountId: string,
580 ): GatewayDynamicRoutingAttributes => ({
581 routeId: route.id,
582 accountId,
583 gatewayId: route.gatewayId,
584 name: route.name,
585 elements: elementsOf(route),
586 versionId: route.deployment.versionId,
587 deploymentId: route.deployment.deploymentId,
588 createdAt: route.createdAt,
589 modifiedAt: route.modifiedAt,
590 });