The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,208 lines 42.9 KB
Raw
1 /// <reference types="@cloudflare/workers-types" />
2
3 import type { RuntimeContext } from "alchemy/RuntimeContext";
4 import {
5 AiError,
6 LanguageModel as AiLanguageModel,
7 IdGenerator,
8 Prompt,
9 Response,
10 Tool,
11 } from "effect/ai";
12 import * as Effect from "effect/Effect";
13 import * as Sse from "effect/encoding/Sse";
14 import * as Layer from "effect/Layer";
15 import * as Option from "effect/Option";
16 import * as Schema from "effect/Schema";
17 import * as Stream from "effect/Stream";
18
19 /**
20 * The slice of an AI client the LanguageModel adapter needs: the raw Workers
21 * AI handle plus, when routed through an AI Gateway, the gateway id.
22 *
23 * Both `Cloudflare.AI.QueryGateway(gateway)` (gateway-routed) and
24 * `Cloudflare.Workers.AI()` (plain Workers AI binding) clients satisfy it.
25 */
26 export interface LanguageModelClient {
27 /** Effect resolving to the raw Workers AI runtime binding. */
28 readonly raw: Effect.Effect<Ai, never, RuntimeContext>;
29 /**
30 * Effect resolving to the AI Gateway id to route `ai.run` calls through.
31 * Omit for a plain Workers AI binding — requests then go directly to
32 * Workers AI without a gateway.
33 */
34 readonly id?: Effect.Effect<string, never, RuntimeContext>;
35 }
36
37 /**
38 * Options for constructing a Workers AI-backed LanguageModel.
39 */
40 export interface LanguageModelOptions {
41 /**
42 * Already-bound AI client — from `Cloudflare.AI.QueryGateway(gateway)`
43 * (routed through the gateway) or `Cloudflare.Workers.AI()` (direct).
44 */
45 readonly client: LanguageModelClient;
46 /** Workers AI model id, e.g. `@cf/meta/llama-3.3-70b-instruct-fp8-fast`. */
47 readonly model: string;
48 /** Optional per-call defaults; overridable per request via `providerOptions`. */
49 readonly parameters?: {
50 readonly temperature?: number;
51 readonly maxTokens?: number;
52 readonly topP?: number;
53 readonly topK?: number;
54 readonly seed?: number;
55 readonly frequencyPenalty?: number;
56 readonly presencePenalty?: number;
57 };
58 }
59
60 /**
61 * Provide a {@link AiLanguageModel.LanguageModel} layer backed by the supplied
62 * AI Gateway client and Workers AI model.
63 */
64 export const makeLanguageModelLayer = (
65 options: LanguageModelOptions,
66 ): Layer.Layer<AiLanguageModel.LanguageModel, never, RuntimeContext> =>
67 Layer.effect(AiLanguageModel.LanguageModel, makeLanguageModel(options));
68
69 /**
70 * Build a {@link AiLanguageModel.LanguageModel} that proxies generateText/streamText
71 * through the supplied AI Gateway client to a Workers AI model.
72 */
73 export const makeLanguageModel = ({
74 client,
75 model,
76 parameters,
77 }: LanguageModelOptions): Effect.Effect<AiLanguageModel.LanguageModel, never, RuntimeContext> =>
78 Effect.gen(function* () {
79 const ai = yield* client.raw;
80 const gatewayId = client.id === undefined ? undefined : yield* client.id;
81 const startsInThink = THINK_PREFILLED_MODELS.has(model);
82
83 const callRaw = (
84 body: WorkersAiInputs,
85 method: "generateText" | "streamText",
86 ): Effect.Effect<Response, AiError.AiError> =>
87 Effect.tryPromise({
88 try: () =>
89 ai.run(model as keyof AiModels, body as unknown as AiModels[keyof AiModels]["inputs"], {
90 ...(gatewayId === undefined ? {} : { gateway: { id: gatewayId } }),
91 returnRawResponse: true,
92 }),
93 catch: (cause) => toAiError(cause, method),
94 });
95
96 return yield* AiLanguageModel.make({
97 generateText: (options) =>
98 Effect.gen(function* () {
99 const body = toRequestBody({ options, parameters, stream: false });
100 const resp = yield* callRaw(body, "generateText");
101 const json = yield* Effect.tryPromise({
102 try: () => resp.json() as Promise<Record<string, unknown>>,
103 catch: (cause) => toAiError(cause, "generateText"),
104 });
105 return yield* parseGenerateText(json, startsInThink);
106 }),
107 streamText: (options) =>
108 Stream.unwrap(
109 Effect.gen(function* () {
110 const idGen = yield* IdGenerator.IdGenerator;
111 const body = toRequestBody({ options, parameters, stream: true });
112 const resp = yield* callRaw(body, "streamText");
113 const hasTools = options.tools.length > 0 && options.toolChoice !== "none";
114 return parseStreamText(resp, idGen, hasTools, startsInThink);
115 }),
116 ),
117 });
118 });
119
120 // ---------------------------------------------------------------------------
121 // Wire format types (Workers AI request)
122 //
123 // Workers AI returns two response shapes depending on the model:
124 // - Native: { response: "...", tool_calls: [...] , usage }
125 // - OpenAI: { choices: [{ message: { content, tool_calls, reasoning_content } }], usage }
126 //
127 // We accept both defensively — schemas would over-constrain.
128 // ---------------------------------------------------------------------------
129
130 interface WorkersAiMessage {
131 readonly role: "system" | "user" | "assistant" | "tool";
132 readonly content?: unknown;
133 readonly name?: string;
134 readonly tool_call_id?: string;
135 readonly tool_calls?: ReadonlyArray<{
136 readonly id: string;
137 readonly type: "function";
138 readonly function: { readonly name: string; readonly arguments: string };
139 }>;
140 readonly reasoning?: string;
141 }
142
143 interface WorkersAiToolDef {
144 readonly type: "function";
145 readonly function: {
146 readonly name: string;
147 readonly description?: string;
148 readonly parameters: unknown;
149 };
150 }
151
152 type WorkersAiToolChoice =
153 | "auto"
154 | "required"
155 | "none"
156 | { readonly type: "function"; readonly function: { readonly name: string } };
157
158 interface WorkersAiInputs {
159 readonly messages: ReadonlyArray<WorkersAiMessage>;
160 readonly tools?: ReadonlyArray<WorkersAiToolDef>;
161 readonly tool_choice?: WorkersAiToolChoice;
162 readonly stream?: boolean;
163 readonly stream_options?: { readonly include_usage: boolean };
164 readonly max_tokens?: number;
165 readonly temperature?: number;
166 readonly top_p?: number;
167 readonly top_k?: number;
168 readonly random_seed?: number;
169 readonly frequency_penalty?: number;
170 readonly presence_penalty?: number;
171 }
172
173 // ---------------------------------------------------------------------------
174 // Prompt → Workers AI messages (pure, no .push mutation)
175 // ---------------------------------------------------------------------------
176
177 const uint8ArrayToBase64 = (bytes: Uint8Array): string => {
178 // Keep older Workers and no_nodejs_compat deployments working without a
179 // Node import. The native web API encodes only this view's byte range.
180 const toBase64 = (bytes as Uint8Array & { toBase64?: () => string }).toBase64;
181 if (typeof toBase64 === "function") return toBase64.call(bytes);
182
183 let binary = "";
184 const chunkSize = 8192;
185 for (let i = 0; i < bytes.length; i += chunkSize) {
186 const chunk = bytes.subarray(i, Math.min(i + chunkSize, bytes.length));
187 binary += String.fromCharCode(...chunk);
188 }
189 return btoa(binary);
190 };
191
192 const fileToImageUrl = (data: string | Uint8Array | URL, mediaType: string): string => {
193 if (data instanceof URL) return data.toString();
194 if (data instanceof Uint8Array) {
195 return `data:${mediaType};base64,${uint8ArrayToBase64(data)}`;
196 }
197 if (data.startsWith("data:") || data.startsWith("http")) return data;
198 return `data:${mediaType};base64,${data}`;
199 };
200
201 const convertPromptToMessages = (prompt: Prompt.Prompt): ReadonlyArray<WorkersAiMessage> =>
202 prompt.content.flatMap((m): ReadonlyArray<WorkersAiMessage> => {
203 switch (m.role) {
204 case "system":
205 return [{ role: "system", content: m.content }];
206 case "user":
207 return [toUserMessage(m.content)];
208 case "assistant":
209 return [toAssistantMessage(m.content)];
210 case "tool":
211 return m.content.flatMap(toToolMessage);
212 }
213 });
214
215 const toUserMessage = (parts: Prompt.UserMessage["content"]): WorkersAiMessage => {
216 const text = parts.flatMap((p) => (p.type === "text" ? [p.text] : [])).join("\n");
217 const images = parts.flatMap((p) =>
218 p.type === "file"
219 ? [
220 {
221 type: "image_url" as const,
222 image_url: { url: fileToImageUrl(p.data, p.mediaType) },
223 },
224 ]
225 : [],
226 );
227 if (images.length === 0) return { role: "user", content: text };
228 return {
229 role: "user",
230 content: [...(text.length > 0 ? [{ type: "text" as const, text }] : []), ...images],
231 };
232 };
233
234 const toAssistantMessage = (parts: Prompt.AssistantMessage["content"]): WorkersAiMessage => {
235 const text = parts.flatMap((p) => (p.type === "text" ? [p.text] : [])).join("");
236 const reasoning = parts.flatMap((p) => (p.type === "reasoning" ? [p.text] : [])).join("");
237 const toolCalls = parts.flatMap((p) =>
238 p.type === "tool-call"
239 ? [
240 {
241 id: p.id,
242 type: "function" as const,
243 function: {
244 name: p.name,
245 arguments: typeof p.params === "string" ? p.params : JSON.stringify(p.params ?? {}),
246 },
247 },
248 ]
249 : [],
250 );
251 return {
252 role: "assistant",
253 content: text,
254 ...(reasoning ? { reasoning } : {}),
255 ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
256 };
257 };
258
259 const toToolMessage = (
260 part: Prompt.ToolMessage["content"][number],
261 ): ReadonlyArray<WorkersAiMessage> =>
262 part.type === "tool-result"
263 ? [
264 {
265 role: "tool",
266 name: part.name,
267 tool_call_id: part.id,
268 content: typeof part.result === "string" ? part.result : JSON.stringify(part.result),
269 },
270 ]
271 : [];
272
273 // ---------------------------------------------------------------------------
274 // Tools / tool_choice
275 // ---------------------------------------------------------------------------
276
277 type PreparedToolSchema = {
278 readonly schema: Tool.Any["parametersSchema"];
279 readonly ast: Tool.Any["parametersSchema"]["ast"];
280 readonly json: ReturnType<typeof Tool.getJsonSchema>;
281 };
282
283 // Tools and Effect schemas expose immutable update methods. Still check both
284 // identities: JavaScript callers can replace a tool's schema in place. Dynamic
285 // JSON schemas remain caller-owned and may change without an identity change.
286 const preparedToolSchemas = new WeakMap<Tool.Any, PreparedToolSchema>();
287
288 const prepareTool = (tool: Tool.Any): WorkersAiToolDef => {
289 let parameters: ReturnType<typeof Tool.getJsonSchema>;
290 if (Tool.isDynamic(tool) && tool.jsonSchema !== undefined) {
291 parameters = Tool.getJsonSchema(tool);
292 } else {
293 const schema = tool.parametersSchema;
294 let cached = preparedToolSchemas.get(tool);
295 if (cached === undefined || cached.schema !== schema || cached.ast !== schema.ast) {
296 cached = { schema, ast: schema.ast, json: Tool.getJsonSchema(tool) };
297 preparedToolSchemas.set(tool, cached);
298 }
299 // Each request previously owned its compiled schema. Keep that isolation
300 // if the runtime client mutates request input, without recompiling it.
301 parameters = structuredClone(cached.json);
302 }
303 return {
304 type: "function",
305 function: {
306 name: tool.name,
307 description: Tool.getDescription(tool),
308 parameters,
309 },
310 };
311 };
312
313 const prepareTools = (
314 tools: ReadonlyArray<Tool.Any>,
315 toolChoice: AiLanguageModel.ProviderOptions["toolChoice"],
316 ): {
317 tools?: ReadonlyArray<WorkersAiToolDef>;
318 tool_choice?: WorkersAiToolChoice;
319 } => {
320 if (tools.length === 0) return {};
321 if (toolChoice != null && typeof toolChoice === "object" && "tool" in toolChoice) {
322 return {
323 tools: tools.filter((tool) => tool.name === toolChoice.tool).map(prepareTool),
324 tool_choice: "required",
325 };
326 }
327 if (toolChoice != null && typeof toolChoice === "object" && "oneOf" in toolChoice) {
328 const allowed = new Set(toolChoice.oneOf);
329 return {
330 tools: tools.filter((tool) => allowed.has(tool.name)).map(prepareTool),
331 tool_choice: toolChoice.mode === "required" ? "required" : "auto",
332 };
333 }
334 return {
335 tools: tools.map(prepareTool),
336 tool_choice: toolChoice === "none" || toolChoice === "required" ? toolChoice : "auto",
337 };
338 };
339
340 // ---------------------------------------------------------------------------
341 // Request body
342 // ---------------------------------------------------------------------------
343
344 const toRequestBody = ({
345 options,
346 parameters,
347 stream,
348 }: {
349 readonly options: AiLanguageModel.ProviderOptions;
350 readonly parameters: LanguageModelOptions["parameters"];
351 readonly stream: boolean;
352 }): WorkersAiInputs => {
353 const messages = convertPromptToMessages(options.prompt);
354 const { tools, tool_choice } = prepareTools(options.tools, options.toolChoice);
355 return {
356 messages,
357 ...(tools !== undefined ? { tools } : {}),
358 ...(tool_choice !== undefined ? { tool_choice } : {}),
359 // `stream_options.include_usage` is the OpenAI-compatible opt-in for
360 // usage tokens to appear in the final streamed chunk. Without it most
361 // Workers AI models omit `usage` from the stream entirely, leaving the
362 // `finish` part with zeroed counts.
363 ...(stream ? { stream: true, stream_options: { include_usage: true } } : {}),
364 ...(parameters?.maxTokens !== undefined ? { max_tokens: parameters.maxTokens } : {}),
365 ...(parameters?.temperature !== undefined ? { temperature: parameters.temperature } : {}),
366 ...(parameters?.topP !== undefined ? { top_p: parameters.topP } : {}),
367 ...(parameters?.topK !== undefined ? { top_k: parameters.topK } : {}),
368 ...(parameters?.seed !== undefined ? { random_seed: parameters.seed } : {}),
369 ...(parameters?.frequencyPenalty !== undefined
370 ? { frequency_penalty: parameters.frequencyPenalty }
371 : {}),
372 ...(parameters?.presencePenalty !== undefined
373 ? { presence_penalty: parameters.presencePenalty }
374 : {}),
375 };
376 };
377
378 // ---------------------------------------------------------------------------
379 // Finish reason / usage mapping
380 // ---------------------------------------------------------------------------
381
382 const mapFinishReason = (raw: unknown): Response.FinishReason => {
383 switch (raw) {
384 case "stop":
385 return "stop";
386 case "length":
387 case "model_length":
388 return "length";
389 case "tool_calls":
390 return "tool-calls";
391 case "content_filter":
392 case "content-filter":
393 return "content-filter";
394 case "error":
395 return "error";
396 case undefined:
397 case null:
398 return "unknown";
399 default:
400 return "other";
401 }
402 };
403
404 const finishMetadata = (
405 raw: Record<string, unknown> | undefined,
406 ): { readonly metadata?: Response.FinishPartMetadata } => {
407 const neurons = (raw?.usage as Record<string, unknown> | undefined)?.neurons;
408 return typeof neurons === "number" && Number.isFinite(neurons)
409 ? { metadata: { cloudflare: { neurons } } }
410 : {};
411 };
412
413 /**
414 * The neurons Workers AI billed for a response, read from its finish part.
415 * Workers AI reports this as `usage.neurons`; the adapter carries it on the
416 * finish part as `metadata.cloudflare.neurons`. `undefined` when the model
417 * did not report it.
418 *
419 * @example
420 * ```typescript
421 * const response = yield* LanguageModel.generateText({ prompt });
422 * const finish = response.content.find((part) => part.type === "finish");
423 * const neurons = finish ? Cloudflare.AI.finishNeurons(finish) : undefined;
424 * ```
425 */
426 export const finishNeurons = (part: {
427 readonly metadata: Response.FinishPartMetadata;
428 }): number | undefined =>
429 Option.getOrUndefined(Option.map(decodeNeurons(part.metadata), (m) => m.cloudflare.neurons));
430
431 const decodeNeurons = Schema.decodeUnknownOption(
432 Schema.Struct({ cloudflare: Schema.Struct({ neurons: Schema.Number }) }),
433 );
434
435 const mapUsage = (raw: Record<string, unknown> | undefined): Response.Usage => {
436 const usage = (raw?.usage as Record<string, unknown> | undefined) ?? {};
437 const promptTokens = (usage.prompt_tokens as number | undefined) ?? 0;
438 const completionTokens = (usage.completion_tokens as number | undefined) ?? 0;
439 const cached = (usage.prompt_tokens_details as { cached_tokens?: number } | undefined)
440 ?.cached_tokens;
441 // Construct an actual `Response.Usage` instance — `Schema.Class<Usage>`
442 // encodes by going through the class constructor / `isInstance` check, so a
443 // plain struct that "matches" the encoded shape isn't enough.
444 return Response.Usage.make({
445 inputTokens: {
446 uncached: cached !== undefined ? Math.max(0, promptTokens - cached) : promptTokens,
447 total: promptTokens,
448 cacheRead: cached ?? 0,
449 cacheWrite: 0,
450 },
451 outputTokens: {
452 total: completionTokens,
453 text: 0,
454 reasoning: 0,
455 },
456 });
457 };
458
459 // ---------------------------------------------------------------------------
460 // Inline reasoning
461 //
462 // Some models (e.g. `@cf/deepseek-ai/deepseek-r1-distill-qwen-32b`) write
463 // their chain of thought into the text itself, as `<think>…</think>`
464 // followed by the answer. Others (e.g. `@cf/qwen/qwq-32b`) have a chat
465 // template that pre-opens `<think>`, so only the closing tag is generated.
466 // `splitThink` incrementally routes text before `</think>` to reasoning and
467 // the rest to text, holding back any suffix that could be a split tag.
468 // ---------------------------------------------------------------------------
469
470 /**
471 * Models whose chat template opens the `<think>` block in the prompt, so
472 * their output starts inside reasoning and only `</think>` is generated.
473 */
474 const THINK_PREFILLED_MODELS: ReadonlySet<string> = new Set(["@cf/qwen/qwq-32b"]);
475
476 const THINK_OPEN = "<think>";
477 const THINK_CLOSE = "</think>";
478
479 interface ThinkState {
480 /**
481 * - `pending`: no non-whitespace text yet; a leading `<think>` opens reasoning
482 * - `inside`: inside the `<think>` block
483 * - `after`: just closed the block; drop the whitespace separating the answer
484 * - `outside`: plain text
485 */
486 readonly mode: "pending" | "inside" | "after" | "outside";
487 /** Held-back text that may be the start of a tag split across chunks. */
488 readonly carry: string;
489 }
490
491 const initialThinkState = (startsInThink: boolean): ThinkState => ({
492 mode: startsInThink ? "inside" : "pending",
493 carry: "",
494 });
495
496 /** Length of the longest suffix of `s` that is a proper prefix of `tag`. */
497 const partialTagSuffix = (s: string, tag: string): number => {
498 for (let n = Math.min(tag.length - 1, s.length); n > 0; n--) {
499 if (s.endsWith(tag.slice(0, n))) return n;
500 }
501 return 0;
502 };
503
504 const splitThink = (
505 state: ThinkState,
506 input: string,
507 ): { readonly state: ThinkState; readonly reasoning: string; readonly text: string } => {
508 let buf = state.carry + input;
509 let mode = state.mode;
510 let reasoning = "";
511 while (true) {
512 switch (mode) {
513 case "pending": {
514 const trimmed = buf.trimStart();
515 if (trimmed.startsWith(THINK_OPEN)) {
516 buf = trimmed.slice(THINK_OPEN.length);
517 mode = "inside";
518 continue;
519 }
520 if (THINK_OPEN.startsWith(trimmed)) {
521 return { state: { mode, carry: buf }, reasoning, text: "" };
522 }
523 mode = "outside";
524 continue;
525 }
526 case "inside": {
527 const end = buf.indexOf(THINK_CLOSE);
528 if (end >= 0) {
529 reasoning += buf.slice(0, end);
530 buf = buf.slice(end + THINK_CLOSE.length);
531 mode = "after";
532 continue;
533 }
534 const held = partialTagSuffix(buf, THINK_CLOSE);
535 reasoning += buf.slice(0, buf.length - held);
536 return { state: { mode, carry: buf.slice(buf.length - held) }, reasoning, text: "" };
537 }
538 case "after": {
539 buf = buf.trimStart();
540 if (buf.length === 0) return { state: { mode, carry: "" }, reasoning, text: "" };
541 mode = "outside";
542 continue;
543 }
544 case "outside":
545 return { state: { mode, carry: "" }, reasoning, text: buf };
546 }
547 }
548 };
549
550 /**
551 * Split a complete response text. A `</think>` with no opening tag means the
552 * template pre-opened the block, so everything before it is reasoning.
553 */
554 const splitThinkText = (
555 text: string,
556 startsInThink: boolean,
557 ): { readonly reasoning: string; readonly text: string } => {
558 const prefilled =
559 startsInThink || (text.includes(THINK_CLOSE) && !text.trimStart().startsWith(THINK_OPEN));
560 const split = splitThink(initialThinkState(prefilled), text);
561 return split.state.mode === "inside"
562 ? { reasoning: split.reasoning + split.state.carry, text: split.text }
563 : { reasoning: split.reasoning, text: split.text + split.state.carry };
564 };
565
566 // ---------------------------------------------------------------------------
567 // generateText: JSON → Response.PartEncoded[]
568 //
569 // Normalize the dual-shape (native + OpenAI) response into a single
570 // `DecodedResponse` once, then build the part list with pure spreads.
571 // ---------------------------------------------------------------------------
572
573 interface DecodedToolCall {
574 readonly rawId: string;
575 readonly name: string;
576 readonly arguments: unknown;
577 }
578
579 interface DecodedResponse {
580 readonly text: string | undefined;
581 readonly reasoning: string | undefined;
582 readonly toolCalls: ReadonlyArray<DecodedToolCall>;
583 readonly finishReason: string | undefined;
584 }
585
586 const decodeResponse = (raw: Record<string, unknown>): DecodedResponse => {
587 const choice = (
588 raw.choices as
589 | Array<{
590 message?: {
591 content?: string | null;
592 reasoning_content?: string;
593 reasoning?: string;
594 tool_calls?: ReadonlyArray<Record<string, unknown>>;
595 };
596 finish_reason?: string;
597 }>
598 | undefined
599 )?.[0];
600 const message = choice?.message;
601
602 const openAiText = message?.content;
603 const text =
604 typeof openAiText === "string" && openAiText.length > 0
605 ? openAiText
606 : nativeTextOf(raw.response);
607 const reasoning = message?.reasoning_content ?? message?.reasoning;
608 const rawToolCalls =
609 message?.tool_calls ??
610 (Array.isArray(raw.tool_calls)
611 ? (raw.tool_calls as ReadonlyArray<Record<string, unknown>>)
612 : []);
613
614 return {
615 text,
616 reasoning: reasoning && reasoning.length > 0 ? reasoning : undefined,
617 toolCalls: rawToolCalls.flatMap(decodeToolCall),
618 finishReason: choice?.finish_reason ?? (raw.finish_reason as string | undefined),
619 };
620 };
621
622 const nativeTextOf = (raw: unknown): string | undefined => {
623 if (raw == null) return undefined;
624 // Workers AI responses are JSON-decoded, so JSON.stringify matches
625 // String() for every scalar and serializes objects/arrays.
626 const text = typeof raw === "string" ? raw : JSON.stringify(raw);
627 return text.length > 0 ? text : undefined;
628 };
629
630 const decodeToolCall = (tc: Record<string, unknown>): ReadonlyArray<DecodedToolCall> => {
631 const fn = tc.function as { name?: string; arguments?: unknown } | undefined;
632 const rawId = (tc.id as string | undefined) ?? "";
633 if (fn?.name) {
634 return [{ rawId, name: fn.name, arguments: fn.arguments ?? "" }];
635 }
636 const flatName = tc.name as string | undefined;
637 if (flatName) {
638 return [{ rawId, name: flatName, arguments: tc.arguments ?? "" }];
639 }
640 return [];
641 };
642
643 const tryParseJsonArgs = (raw: unknown): unknown => {
644 if (typeof raw !== "string") return raw;
645 try {
646 return JSON.parse(raw);
647 } catch {
648 // Leave as raw string; the framework's tool-result decoder will fail loudly.
649 return raw;
650 }
651 };
652
653 const parseGenerateText = Effect.fn(function* (
654 raw: Record<string, unknown>,
655 startsInThink: boolean,
656 ) {
657 const idGen = yield* IdGenerator.IdGenerator;
658 const decoded = decodeResponse(raw);
659 const inline =
660 decoded.text === undefined ? undefined : splitThinkText(decoded.text, startsInThink);
661 const reasoning = [decoded.reasoning, inline?.reasoning]
662 .filter((r): r is string => r !== undefined && r.length > 0)
663 .join("");
664 const text = inline?.text ?? decoded.text;
665
666 const toolCallParts = yield* Effect.forEach(decoded.toolCalls, (tc) =>
667 Effect.gen(function* () {
668 const id = tc.rawId || (yield* idGen.generateId());
669 return {
670 type: "tool-call" as const,
671 id,
672 name: tc.name,
673 params: tryParseJsonArgs(tc.arguments),
674 };
675 }),
676 );
677
678 const finish = mapFinishReason(
679 decoded.finishReason ?? (decoded.toolCalls.length > 0 ? "tool_calls" : "stop"),
680 );
681
682 return [
683 ...(reasoning.length > 0 ? [{ type: "reasoning" as const, text: reasoning }] : []),
684 ...(text !== undefined && text.length > 0 ? [{ type: "text" as const, text }] : []),
685 ...toolCallParts,
686 {
687 type: "finish" as const,
688 reason: finish,
689 usage: mapUsage(raw),
690 response: undefined,
691 ...finishMetadata(raw),
692 },
693 ] satisfies ReadonlyArray<Response.PartEncoded>;
694 });
695
696 // ---------------------------------------------------------------------------
697 // streamText: SSE byte stream → Stream<Response.StreamPartEncoded>
698 //
699 // Immutable `StreamState` is threaded through `Stream.mapAccumEffect`. The
700 // per-chunk output buffer (`parts: Array<StreamPartEncoded>`) is mutable for
701 // performance — it's scoped to one chunk, never escapes the handler, and lets
702 // us avoid the O(n²) array-spread that pure threading would force in the hot
703 // path. This matches the pattern Effect's own `@effect/ai-*` adapters use.
704 // ---------------------------------------------------------------------------
705
706 interface StreamState {
707 readonly textId: string | undefined;
708 readonly reasoningId: string | undefined;
709 readonly toolCalls: ReadonlyMap<
710 number,
711 { readonly id: string; readonly name: string; readonly arguments: string }
712 >;
713 readonly lastToolIndex: number | undefined;
714 readonly closedToolIndices: ReadonlySet<number>;
715 readonly usage: Record<string, unknown> | undefined;
716 readonly finishReason: string | undefined;
717 readonly receivedAnyData: boolean;
718 readonly receivedDone: boolean;
719 // Workers AI's *native* streaming for function-calling models (e.g.
720 // `@cf/meta/llama-3.3-70b`) does not emit structured `tool_calls` deltas
721 // — it streams the call as a JSON document inside the `response` text
722 // field (`{"name":"get_weather","parameters":{...}}`). When tools were
723 // requested we buffer that text here instead of emitting it as
724 // text-delta, then parse it into tool-params parts on finalize.
725 readonly nativeToolBuffer: string;
726 readonly nativeToolId: string | undefined;
727 readonly think: ThinkState;
728 }
729
730 const initialStreamState = (startsInThink: boolean): StreamState => ({
731 textId: undefined,
732 reasoningId: undefined,
733 toolCalls: new Map(),
734 lastToolIndex: undefined,
735 closedToolIndices: new Set(),
736 usage: undefined,
737 finishReason: undefined,
738 receivedAnyData: false,
739 receivedDone: false,
740 nativeToolBuffer: "",
741 nativeToolId: undefined,
742 think: initialThinkState(startsInThink),
743 });
744
745 type StreamParts = Array<Response.StreamPartEncoded>;
746
747 const tryParseJson = (data: string): Record<string, unknown> | undefined => {
748 try {
749 const v = JSON.parse(data);
750 return v && typeof v === "object" ? (v as Record<string, unknown>) : undefined;
751 } catch {
752 return undefined;
753 }
754 };
755
756 const isNullFinalizationToolCall = (tc: Record<string, unknown>): boolean => {
757 const fn = tc.function as Record<string, unknown> | undefined;
758 const name = fn?.name ?? tc.name ?? null;
759 const args = fn?.arguments ?? tc.arguments ?? null;
760 const id = tc.id ?? null;
761 return !id && !name && (!args || args === "");
762 };
763
764 const closeReasoning = (state: StreamState, parts: StreamParts): StreamState => {
765 if (state.reasoningId === undefined) return state;
766 parts.push({ type: "reasoning-end", id: state.reasoningId });
767 return { ...state, reasoningId: undefined };
768 };
769
770 const closeToolCall = (state: StreamState, index: number, parts: StreamParts): StreamState => {
771 if (state.closedToolIndices.has(index)) return state;
772 const tc = state.toolCalls.get(index);
773 if (!tc) return state;
774 parts.push({ type: "tool-params-end", id: tc.id });
775 parts.push({
776 type: "tool-call",
777 id: tc.id,
778 name: tc.name,
779 params: tryParseJsonArgs(tc.arguments),
780 });
781 const closed = new Set(state.closedToolIndices);
782 closed.add(index);
783 return { ...state, closedToolIndices: closed };
784 };
785
786 const emitTextDelta = (
787 state: StreamState,
788 delta: string,
789 parts: StreamParts,
790 idGen: IdGenerator.Service,
791 ): Effect.Effect<StreamState> =>
792 Effect.gen(function* () {
793 let s = closeReasoning(state, parts);
794 if (s.textId === undefined) {
795 const id = yield* idGen.generateId();
796 parts.push({ type: "text-start", id });
797 s = { ...s, textId: id };
798 }
799 parts.push({ type: "text-delta", id: s.textId!, delta });
800 return s;
801 });
802
803 const emitReasoningDelta = (
804 state: StreamState,
805 delta: string,
806 parts: StreamParts,
807 idGen: IdGenerator.Service,
808 ): Effect.Effect<StreamState> =>
809 Effect.gen(function* () {
810 let s = state;
811 if (s.reasoningId === undefined) {
812 const id = yield* idGen.generateId();
813 parts.push({ type: "reasoning-start", id });
814 s = { ...s, reasoningId: id };
815 }
816 parts.push({ type: "reasoning-delta", id: s.reasoningId!, delta });
817 return s;
818 });
819
820 const handleToolDeltas = (
821 state: StreamState,
822 deltas: ReadonlyArray<Record<string, unknown>>,
823 parts: StreamParts,
824 idGen: IdGenerator.Service,
825 ): Effect.Effect<StreamState> =>
826 Effect.gen(function* () {
827 let s = state;
828 for (const d of deltas) {
829 if (isNullFinalizationToolCall(d)) {
830 if (s.lastToolIndex !== undefined) {
831 s = closeToolCall(s, s.lastToolIndex, parts);
832 }
833 continue;
834 }
835 const idx = (d.index as number | undefined) ?? 0;
836 const fn = d.function as { name?: string; arguments?: string } | undefined;
837 const name = fn?.name ?? (d.name as string | undefined) ?? "";
838 const args = fn?.arguments ?? (d.arguments as string | undefined) ?? "";
839 const rawId = (d.id as string | undefined) ?? "";
840
841 const existing = s.toolCalls.get(idx);
842 if (existing === undefined) {
843 if (s.lastToolIndex !== undefined && s.lastToolIndex !== idx) {
844 s = closeToolCall(s, s.lastToolIndex, parts);
845 }
846 const id = rawId || (yield* idGen.generateId());
847 const entry = { id, name, arguments: args };
848 const next = new Map(s.toolCalls);
849 next.set(idx, entry);
850 s = { ...s, toolCalls: next, lastToolIndex: idx };
851 parts.push({ type: "tool-params-start", id, name });
852 if (args.length > 0) {
853 parts.push({ type: "tool-params-delta", id, delta: args });
854 }
855 } else {
856 // OpenAI-compatible providers stream argument fragments, while
857 // Workers AI can resend the complete accumulated JSON on every
858 // chunk. Normalize both shapes to incremental deltas.
859 const delta = args.startsWith(existing.arguments)
860 ? args.slice(existing.arguments.length)
861 : args;
862 const accumulated = args.startsWith(existing.arguments) ? args : existing.arguments + args;
863 const next = new Map(s.toolCalls);
864 next.set(idx, { ...existing, arguments: accumulated });
865 s = { ...s, toolCalls: next, lastToolIndex: idx };
866 if (delta.length > 0) {
867 parts.push({
868 type: "tool-params-delta",
869 id: existing.id,
870 delta,
871 });
872 }
873 }
874 }
875 return s;
876 });
877
878 const hasNonZeroUsage = (raw: unknown): boolean => {
879 if (raw == null || typeof raw !== "object") return false;
880 const u = raw as Record<string, unknown>;
881 const prompt = (u.prompt_tokens as number | undefined) ?? 0;
882 const completion = (u.completion_tokens as number | undefined) ?? 0;
883 const total = (u.total_tokens as number | undefined) ?? 0;
884 return prompt > 0 || completion > 0 || total > 0;
885 };
886
887 const updateChunkMeta = (state: StreamState, chunk: Record<string, unknown>): StreamState => {
888 let s = state;
889 // Workers AI's native stream emits the real usage chunk, then a
890 // "zero-valued terminator" chunk where every count is 0 (it also re-emits
891 // `usage` with all zeros). Treat the zero chunk as a no-op so we keep the
892 // meaningful counts.
893 if (chunk.usage !== undefined && hasNonZeroUsage(chunk.usage)) {
894 s = { ...s, usage: chunk };
895 }
896 const choices = chunk.choices as Array<{ finish_reason?: string }> | undefined;
897 const finish = choices?.[0]?.finish_reason ?? (chunk.finish_reason as string | undefined);
898 if (finish != null) s = { ...s, finishReason: finish };
899 return s;
900 };
901
902 /**
903 * Route a text fragment through the inline-reasoning splitter, emitting any
904 * reasoning it contains and returning the remaining answer text.
905 */
906 const splitInlineReasoning = (
907 state: StreamState,
908 input: string,
909 parts: StreamParts,
910 idGen: IdGenerator.Service,
911 ): Effect.Effect<readonly [StreamState, string]> =>
912 Effect.gen(function* () {
913 const split = splitThink(state.think, input);
914 let s: StreamState = { ...state, think: split.state };
915 if (split.reasoning.length > 0) {
916 s = yield* emitReasoningDelta(s, split.reasoning, parts, idGen);
917 }
918 return [s, split.text] as const;
919 });
920
921 const handleNativeText = (
922 state: StreamState,
923 chunk: Record<string, unknown>,
924 parts: StreamParts,
925 idGen: IdGenerator.Service,
926 hasTools: boolean,
927 ): Effect.Effect<StreamState> => {
928 const native = chunk.response;
929 if (native == null || native === "") return Effect.succeed(state);
930 const text = typeof native === "string" ? native : JSON.stringify(native);
931 if (text.length === 0) return Effect.succeed(state);
932 // Some models (e.g. `@cf/meta/llama-3.3-70b`) mirror every text fragment
933 // in both the native `response` field and the OpenAI-compatible delta of
934 // the same chunk. Prefer the latter so the text is neither emitted nor
935 // buffered twice.
936 const openAiContent = (chunk.choices as Array<{ delta?: { content?: unknown } }> | undefined)?.[0]
937 ?.delta?.content;
938 if (openAiContent === text) return Effect.succeed(state);
939 return Effect.gen(function* () {
940 const [s, answer] = yield* splitInlineReasoning(state, text, parts, idGen);
941 if (answer.length === 0) return s;
942 // When tools were requested, the native `response` stream is the
943 // tool-call JSON, not prose — buffer it and decide on finalize. We
944 // pre-allocate the tool id here (we're in an Effect, finalize is sync).
945 if (hasTools) {
946 const id = s.nativeToolId ?? (yield* idGen.generateId());
947 return {
948 ...s,
949 nativeToolId: id,
950 nativeToolBuffer: s.nativeToolBuffer + answer,
951 };
952 }
953 return yield* emitTextDelta(s, answer, parts, idGen);
954 });
955 };
956
957 const handleNativeToolCalls = (
958 state: StreamState,
959 chunk: Record<string, unknown>,
960 parts: StreamParts,
961 idGen: IdGenerator.Service,
962 ): Effect.Effect<StreamState> => {
963 const openAiToolCalls = (
964 chunk.choices as Array<{ delta?: { tool_calls?: ReadonlyArray<unknown> } }> | undefined
965 )?.[0]?.delta?.tool_calls;
966 // Workers AI can mirror the same fragments in both its top-level native
967 // field and the OpenAI-compatible delta. Prefer the latter so every
968 // fragment is emitted exactly once.
969 if (Array.isArray(openAiToolCalls)) return Effect.succeed(state);
970 // Workers AI attaches an empty `tool_calls: []` to ordinary chunks;
971 // treating it as a tool delta would close the open reasoning block on
972 // every chunk.
973 if (!Array.isArray(chunk.tool_calls) || chunk.tool_calls.length === 0) {
974 return Effect.succeed(state);
975 }
976 return Effect.gen(function* () {
977 const s = closeReasoning(state, parts);
978 return yield* handleToolDeltas(
979 s,
980 chunk.tool_calls as ReadonlyArray<Record<string, unknown>>,
981 parts,
982 idGen,
983 );
984 });
985 };
986
987 const handleOpenAiDelta = (
988 state: StreamState,
989 chunk: Record<string, unknown>,
990 parts: StreamParts,
991 idGen: IdGenerator.Service,
992 ): Effect.Effect<StreamState> => {
993 const delta = (chunk.choices as Array<{ delta?: Record<string, unknown> }> | undefined)?.[0]
994 ?.delta;
995 if (!delta) return Effect.succeed(state);
996 return Effect.gen(function* () {
997 let s = state;
998 const reasoning = (delta.reasoning_content ?? delta.reasoning) as string | undefined;
999 if (reasoning && reasoning.length > 0) {
1000 s = yield* emitReasoningDelta(s, reasoning, parts, idGen);
1001 }
1002 const text = delta.content as string | undefined;
1003 if (text && text.length > 0) {
1004 const [next, answer] = yield* splitInlineReasoning(s, text, parts, idGen);
1005 s = answer.length > 0 ? yield* emitTextDelta(next, answer, parts, idGen) : next;
1006 }
1007 const toolDeltas = delta.tool_calls as ReadonlyArray<Record<string, unknown>> | undefined;
1008 if (Array.isArray(toolDeltas) && toolDeltas.length > 0) {
1009 s = closeReasoning(s, parts);
1010 s = yield* handleToolDeltas(s, toolDeltas, parts, idGen);
1011 }
1012 return s;
1013 });
1014 };
1015
1016 const handleStreamChunk = (
1017 state: StreamState,
1018 data: string,
1019 idGen: IdGenerator.Service,
1020 hasTools: boolean,
1021 ): Effect.Effect<readonly [StreamState, ReadonlyArray<Response.StreamPartEncoded>]> =>
1022 Effect.gen(function* () {
1023 if (data === "") return [state, []] as const;
1024 if (data === "[DONE]") {
1025 return [{ ...state, receivedDone: true }, []] as const;
1026 }
1027 const chunk = tryParseJson(data);
1028 if (chunk === undefined) return [state, []] as const;
1029
1030 const parts: StreamParts = [];
1031 let s: StreamState = { ...state, receivedAnyData: true };
1032 s = updateChunkMeta(s, chunk);
1033 s = yield* handleNativeText(s, chunk, parts, idGen, hasTools);
1034 s = yield* handleNativeToolCalls(s, chunk, parts, idGen);
1035 s = yield* handleOpenAiDelta(s, chunk, parts, idGen);
1036 return [s, parts] as const;
1037 });
1038
1039 /**
1040 * Decode a buffered native-streaming tool call. Workers AI's native shape
1041 * for a function call is a JSON document with `name` plus `parameters` (or
1042 * `arguments`), optionally wrapped in an array. Returns one entry per call,
1043 * or `undefined` if the buffer isn't a tool-call document (so the caller
1044 * can fall back to treating it as plain text).
1045 */
1046 const decodeNativeToolBuffer = (
1047 buffer: string,
1048 ): ReadonlyArray<{ name: string; args: string }> | undefined => {
1049 const trimmed = buffer.trim();
1050 if (trimmed.length === 0 || !(trimmed[0] === "{" || trimmed[0] === "[")) {
1051 return undefined;
1052 }
1053 let parsed: unknown;
1054 try {
1055 parsed = JSON.parse(trimmed);
1056 } catch {
1057 return undefined;
1058 }
1059 const items = Array.isArray(parsed) ? parsed : [parsed];
1060 const calls = items.flatMap(
1061 (
1062 item,
1063 ): ReadonlyArray<{
1064 name: string;
1065 args: string;
1066 }> => {
1067 if (item == null || typeof item !== "object") return [];
1068 const rec = item as Record<string, unknown>;
1069 const fn = rec.function as Record<string, unknown> | undefined;
1070 const name = (fn?.name ?? rec.name) as string | undefined;
1071 if (typeof name !== "string" || name.length === 0) return [];
1072 const rawArgs = fn?.arguments ?? rec.arguments ?? rec.parameters ?? {};
1073 const args = typeof rawArgs === "string" ? rawArgs : JSON.stringify(rawArgs);
1074 return [{ name, args }];
1075 },
1076 );
1077 return calls.length > 0 ? calls : undefined;
1078 };
1079
1080 const flushNativeToolBuffer = (state: StreamState, parts: StreamParts): StreamState => {
1081 if (state.nativeToolBuffer.length === 0) return state;
1082 const calls = decodeNativeToolBuffer(state.nativeToolBuffer);
1083 // Not a tool-call document — the model returned prose despite tools being
1084 // available; surface it as text so nothing is dropped.
1085 if (calls === undefined) {
1086 const id = state.nativeToolId ?? "text-0";
1087 parts.push({ type: "text-start", id });
1088 parts.push({ type: "text-delta", id, delta: state.nativeToolBuffer });
1089 parts.push({ type: "text-end", id });
1090 return { ...state, nativeToolBuffer: "", nativeToolId: undefined };
1091 }
1092 const baseId = state.nativeToolId ?? "tool-0";
1093 calls.forEach((call, i) => {
1094 const id = i === 0 ? baseId : `${baseId}-${i}`;
1095 parts.push({ type: "tool-params-start", id, name: call.name });
1096 if (call.args.length > 0) {
1097 parts.push({ type: "tool-params-delta", id, delta: call.args });
1098 }
1099 parts.push({ type: "tool-params-end", id });
1100 parts.push({ type: "tool-call", id, name: call.name, params: tryParseJsonArgs(call.args) });
1101 });
1102 return { ...state, nativeToolBuffer: "", nativeToolId: undefined };
1103 };
1104
1105 /**
1106 * Emit text the inline-reasoning splitter held back as a possible split tag
1107 * when the stream ended before it resolved.
1108 */
1109 const flushThinkCarry = (state: StreamState, parts: StreamParts): StreamState => {
1110 const { mode, carry } = state.think;
1111 let s: StreamState = { ...state, think: { mode, carry: "" } };
1112 if (carry.length === 0) return s;
1113 if (mode === "inside") {
1114 if (s.reasoningId === undefined) {
1115 parts.push({ type: "reasoning-start", id: "reasoning-0" });
1116 s = { ...s, reasoningId: "reasoning-0" };
1117 }
1118 parts.push({ type: "reasoning-delta", id: s.reasoningId!, delta: carry });
1119 return s;
1120 }
1121 if (carry.trim().length === 0) return s;
1122 s = closeReasoning(s, parts);
1123 if (s.textId === undefined) {
1124 parts.push({ type: "text-start", id: "text-0" });
1125 s = { ...s, textId: "text-0" };
1126 }
1127 parts.push({ type: "text-delta", id: s.textId!, delta: carry });
1128 return s;
1129 };
1130
1131 const finalizeStream = (state: StreamState): ReadonlyArray<Response.StreamPartEncoded> => {
1132 const parts: StreamParts = [];
1133 let s = flushThinkCarry(state, parts);
1134 for (const [idx] of s.toolCalls) s = closeToolCall(s, idx, parts);
1135 s = closeReasoning(s, parts);
1136 if (s.textId !== undefined) parts.push({ type: "text-end", id: s.textId });
1137 // Emit any buffered native-streaming tool call (llama-style) before the
1138 // terminal finish part.
1139 s = flushNativeToolBuffer(s, parts);
1140
1141 // Three cases for the final reason:
1142 // 1. The model emitted an explicit `finish_reason` → map it.
1143 // 2. The stream ended cleanly (`[DONE]` seen) but no reason → "stop".
1144 // Workers AI's native shape never includes `finish_reason`,
1145 // so without this rule every native-mode stream would report
1146 // `unknown` despite completing successfully.
1147 // 3. The stream ended abnormally (no `[DONE]`, no reason) → "error".
1148 const reason: Response.FinishReason =
1149 s.finishReason !== undefined
1150 ? mapFinishReason(s.finishReason)
1151 : s.receivedDone
1152 ? "stop"
1153 : s.receivedAnyData
1154 ? "error"
1155 : "unknown";
1156
1157 parts.push({
1158 type: "finish",
1159 reason,
1160 usage: mapUsage(s.usage),
1161 response: undefined,
1162 ...finishMetadata(s.usage),
1163 });
1164 return parts;
1165 };
1166
1167 const parseStreamText = (
1168 resp: Response,
1169 idGen: IdGenerator.Service,
1170 hasTools: boolean,
1171 startsInThink: boolean,
1172 ): Stream.Stream<Response.StreamPartEncoded, AiError.AiError> => {
1173 const body = resp.body;
1174 if (body === null) {
1175 return Stream.fromIterable<Response.StreamPartEncoded>(
1176 finalizeStream(initialStreamState(startsInThink)),
1177 );
1178 }
1179 return Stream.fromReadableStream<Uint8Array, AiError.AiError>({
1180 evaluate: () => body,
1181 onError: (cause) => toAiError(cause, "streamText"),
1182 }).pipe(
1183 Stream.decodeText(),
1184 Stream.pipeThroughChannel(Sse.decode<AiError.AiError, unknown>()),
1185 Stream.catchTags({
1186 Retry: (retry) => Stream.die(retry),
1187 SseError: (error) => Stream.fail(toAiError(error, "streamText")),
1188 }),
1189 Stream.mapAccumEffect(
1190 () => initialStreamState(startsInThink),
1191 (state, event) => handleStreamChunk(state, event.data, idGen, hasTools),
1192 { onHalt: (state) => finalizeStream(state) },
1193 ),
1194 );
1195 };
1196
1197 // ---------------------------------------------------------------------------
1198 // Error mapping
1199 // ---------------------------------------------------------------------------
1200
1201 const toAiError = (cause: unknown, method: "generateText" | "streamText"): AiError.AiError =>
1202 AiError.AiError.make({
1203 module: "Cloudflare.AI.LanguageModel",
1204 method,
1205 reason: AiError.UnknownError.make({
1206 description: cause instanceof Error ? cause.message : "AI Gateway request failed",
1207 }),
1208 });