@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 449 lines 14.6 KB
Raw
1 import { makeLanguageModel } from "@alchemy.run/cloudflare/AI/LanguageModel";
2 /// <reference types="@cloudflare/workers-types" />
3 import { describe, expect, test } from "@alchemy.run/test";
4 import { RuntimeContext } from "alchemy/RuntimeContext";
5 import { LanguageModel, Tool, Toolkit } from "effect/ai";
6 import * as Effect from "effect/Effect";
7 import * as Layer from "effect/Layer";
8 import * as Schema from "effect/Schema";
9 import * as Stream from "effect/Stream";
10
11 // Deterministic coverage for how the Workers AI adapter maps raw model output
12 // to Effect AI parts. A fake `Ai` binding replays chunk shapes recorded from
13 // live Workers AI models; the live matrix in `LanguageModel.test.ts` covers the
14 // real models themselves.
15
16 const tags = ["unit", "provider:cloudflare", "provider:cloudflare:ai", "local"];
17
18 const R1 = "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b";
19 const QWQ = "@cf/qwen/qwq-32b";
20
21 const GetWeather = Tool.make("get_weather", {
22 parameters: Schema.Struct({ city: Schema.String }),
23 success: Schema.String,
24 });
25 const Tools = Toolkit.make(GetWeather);
26 const ToolHandlers = Tools.toLayer({
27 get_weather: ({ city }) => Effect.succeed(`Sunny in ${city}`),
28 });
29
30 const sse = (chunks: ReadonlyArray<unknown>) =>
31 [...chunks, "[DONE]"]
32 .map((c) => `data: ${typeof c === "string" ? c : JSON.stringify(c)}\n\n`)
33 .join("");
34
35 /** Native Workers AI chunk: text in the top-level `response` field. */
36 const native = (response: string) => ({ response, tool_calls: [] });
37 /** OpenAI-compatible chunk. */
38 const openai = (delta: Record<string, unknown>) => ({
39 choices: [{ index: 0, delta, finish_reason: null }],
40 tool_calls: [],
41 });
42
43 const layerFor = (model: string, body: () => globalThis.Response) => {
44 const ai = { run: async () => body() } as unknown as Ai;
45 return Layer.effect(
46 LanguageModel.LanguageModel,
47 makeLanguageModel({ model, client: { raw: Effect.succeed(ai) } }),
48 ).pipe(Layer.provide(RuntimeContext.phantom));
49 };
50
51 type Summary = {
52 readonly reasoning: string;
53 readonly text: string;
54 /** Part types with consecutive deltas collapsed, e.g. `reasoning-start, reasoning-delta, …`. */
55 readonly shape: ReadonlyArray<string>;
56 };
57
58 const stream = (
59 chunks: ReadonlyArray<unknown>,
60 options: { readonly model?: string; readonly tools?: boolean } = {},
61 ) =>
62 Effect.gen(function* () {
63 const layer = layerFor(options.model ?? R1, () => new Response(sse(chunks)));
64 const parts = yield* (
65 options.tools
66 ? LanguageModel.streamText({ prompt: "hi", toolkit: Tools }).pipe(
67 Stream.provide(ToolHandlers),
68 )
69 : LanguageModel.streamText({ prompt: "hi" })
70 ).pipe(Stream.provide(layer), Stream.runCollect);
71 const all = Array.from(parts);
72 const shape = all
73 .map((p) => p.type as string)
74 .filter((t, i, ts) => !(t.endsWith("-delta") && ts[i - 1] === t));
75 return {
76 parts: all,
77 reasoning: all.flatMap((p) => (p.type === "reasoning-delta" ? [p.delta] : [])).join(""),
78 text: all.flatMap((p) => (p.type === "text-delta" ? [p.delta] : [])).join(""),
79 shape,
80 };
81 });
82
83 const generate = (message: Record<string, unknown>, model = R1) =>
84 Effect.gen(function* () {
85 const layer = layerFor(model, () => Response.json(message));
86 const response = yield* LanguageModel.generateText({ prompt: "hi" }).pipe(
87 Effect.provide(layer),
88 );
89 return { reasoning: response.reasoningText, text: response.text };
90 });
91
92 const REASONED: ReadonlyArray<string> = [
93 "reasoning-start",
94 "reasoning-delta",
95 "reasoning-end",
96 "text-start",
97 "text-delta",
98 "text-end",
99 "finish",
100 ];
101
102 const expectSummary = (actual: Summary, expected: Partial<Summary>) =>
103 expect({
104 reasoning: actual.reasoning,
105 text: actual.text,
106 ...(expected.shape ? { shape: actual.shape } : {}),
107 }).toEqual({ reasoning: "", text: "", ...expected });
108
109 /** Every way to cut `text` into two chunks, plus one chunk per character. */
110 const chunkings = (text: string): ReadonlyArray<ReadonlyArray<string>> => [
111 ...Array.from({ length: text.length - 1 }, (_, i) => [text.slice(0, i + 1), text.slice(i + 1)]),
112 Array.from(text),
113 ];
114
115 describe("streamText <think> splitting", () => {
116 test.effect(
117 "text and reasoning stream incrementally, one delta per chunk",
118 () =>
119 Effect.gen(function* () {
120 // The splitter only holds back a possible `<think>` prefix; anything
121 // else must be released immediately, not buffered until finalize.
122 const chunks = ["Hello", " <", "b>", " world"];
123 const shapes: ReadonlyArray<(t: string) => unknown> = [
124 native,
125 (content) => openai({ content }),
126 ];
127 for (const shape of shapes) {
128 const out = yield* stream(chunks.map(shape));
129 const deltas = out.parts.flatMap((p) => (p.type === "text-delta" ? [p.delta] : []));
130 expect(deltas).toEqual(chunks);
131 }
132 // Reasoning streams too; only a possible `</think>` prefix waits.
133 const reasoning = yield* stream(
134 ["<think>", "a ", "b </", "i> c", "</think>", "x"].map(native),
135 );
136 expect(
137 reasoning.parts.flatMap((p) => (p.type === "reasoning-delta" ? [p.delta] : [])),
138 ).toEqual(["a ", "b ", "</i> c"]);
139 // A leading `<` waits for exactly one more chunk to rule out `<think>`.
140 const lead = yield* stream(["<", "b>", "bold"].map(native));
141 expect(lead.parts.flatMap((p) => (p.type === "text-delta" ? [p.delta] : []))).toEqual([
142 "<b>",
143 "bold",
144 ]);
145 }),
146 { tags },
147 );
148
149 test.effect(
150 "tags streamed as their own chunks (deepseek-r1-distill shape)",
151 () =>
152 Effect.gen(function* () {
153 const out = yield* stream(
154 [
155 "<think>",
156 "\n",
157 "Pong",
158 " is",
159 " wanted",
160 ".",
161 "\n",
162 "</think>",
163 "\n\n",
164 "Pong",
165 "!",
166 ].map(native),
167 );
168 expectSummary(out, { reasoning: "\nPong is wanted.\n", text: "Pong!", shape: REASONED });
169 }),
170 { tags },
171 );
172
173 test.effect(
174 "block and answer in a single chunk",
175 () =>
176 Effect.gen(function* () {
177 const out = yield* stream([native("<think>plan</think>answer")]);
178 expectSummary(out, { reasoning: "plan", text: "answer", shape: REASONED });
179 }),
180 { tags },
181 );
182
183 test.effect(
184 "whitespace before the opening tag still opens reasoning",
185 () =>
186 Effect.gen(function* () {
187 const out = yield* stream(["\n", " <think>", "plan", "</think>", "answer"].map(native));
188 expectSummary(out, { reasoning: "plan", text: "answer" });
189 }),
190 { tags },
191 );
192
193 test.effect(
194 "an empty block emits no reasoning parts",
195 () =>
196 Effect.gen(function* () {
197 const out = yield* stream(["<think>", "</think>", "\n\n", "answer"].map(native));
198 expectSummary(out, {
199 text: "answer",
200 shape: ["text-start", "text-delta", "text-end", "finish"],
201 });
202 }),
203 { tags },
204 );
205
206 test.effect(
207 "tags split at every chunk boundary and per character",
208 () =>
209 Effect.gen(function* () {
210 // Includes look-alikes inside reasoning (`</b>`, `</thin`) that must be
211 // held back as possible tag prefixes and then released verbatim.
212 const full = "\n<think>a </b> and </thin k</think>\n\nanswer";
213 for (const chunks of chunkings(full)) {
214 const out = yield* stream(chunks.map(native));
215 expectSummary(out, { reasoning: "a </b> and </thin k", text: "answer" });
216 }
217 }),
218 { tags },
219 );
220
221 test.effect(
222 "split tags in the OpenAI `delta.content` shape",
223 () =>
224 Effect.gen(function* () {
225 for (const chunks of chunkings("<think>plan</think>answer")) {
226 const out = yield* stream(chunks.map((content) => openai({ content })));
227 expectSummary(out, { reasoning: "plan", text: "answer" });
228 }
229 }),
230 { tags },
231 );
232
233 test.effect(
234 "text that only resembles an opening tag is emitted verbatim",
235 () =>
236 Effect.gen(function* () {
237 for (const full of ["<b>bold</b>", "<thinking>no</thinking>", "<", "<thi"]) {
238 for (const chunks of full.length > 1 ? chunkings(full) : [[full]]) {
239 const out = yield* stream(chunks.map(native));
240 expectSummary(out, { text: full });
241 }
242 }
243 }),
244 { tags },
245 );
246
247 test.effect(
248 "tags after the answer has started stay text",
249 () =>
250 Effect.gen(function* () {
251 const midProse = yield* stream(["Use ", "<think>", " tags", "</think>", "."].map(native));
252 expectSummary(midProse, { text: "Use <think> tags</think>." });
253 const afterBlock = yield* stream(
254 ["<think>", "plan", "</think>", "x ", "</think>", " y"].map(native),
255 );
256 expectSummary(afterBlock, { reasoning: "plan", text: "x </think> y" });
257 }),
258 { tags },
259 );
260
261 test.effect(
262 "a stream truncated inside reasoning keeps everything as reasoning",
263 () =>
264 Effect.gen(function* () {
265 // `</thi` is held back as a possible tag and flushed on finalize.
266 const out = yield* stream(["<think>", "still thinking", " </thi"].map(native));
267 expectSummary(out, {
268 reasoning: "still thinking </thi",
269 shape: ["reasoning-start", "reasoning-delta", "reasoning-end", "finish"],
270 });
271 }),
272 { tags },
273 );
274
275 test.effect(
276 "whitespace-only output never opens a text block",
277 () =>
278 Effect.gen(function* () {
279 const out = yield* stream(["\n", " "].map(native));
280 expectSummary(out, { shape: ["finish"] });
281 }),
282 { tags },
283 );
284
285 test.effect(
286 "a prefilled-`<think>` model (qwq) starts inside reasoning",
287 () =>
288 Effect.gen(function* () {
289 const chunks = ["Okay", ", pong.", "\n", "</think>", "\n\n", "Pong"].map(native);
290 const qwq = yield* stream(chunks, { model: QWQ });
291 expectSummary(qwq, { reasoning: "Okay, pong.\n", text: "Pong", shape: REASONED });
292 // The same bytes from a model that opens its own tag are plain text:
293 // a stream can't take back text it already emitted.
294 const other = yield* stream(chunks);
295 expectSummary(other, { text: "Okay, pong.\n</think>\n\nPong" });
296 }),
297 { tags },
298 );
299
300 test.effect(
301 "a prefilled model truncated before `</think>` is all reasoning",
302 () =>
303 Effect.gen(function* () {
304 const out = yield* stream(["Okay", ", so"].map(native), { model: QWQ });
305 expectSummary(out, { reasoning: "Okay, so" });
306 }),
307 { tags },
308 );
309
310 test.effect(
311 "with tools: a tool call after the block becomes a tool-call part",
312 () =>
313 Effect.gen(function* () {
314 const out = yield* stream(
315 [
316 "<think>",
317 "need weather",
318 "</think>",
319 "\n",
320 '{"name":"get_weather",',
321 '"parameters":{"city":"Paris"}}',
322 ].map(native),
323 { tools: true },
324 );
325 expect(out.reasoning).toBe("need weather");
326 expect(out.text).toBe("");
327 const call = out.parts.find((p) => p.type === "tool-call");
328 expect(call).toMatchObject({ name: "get_weather", params: { city: "Paris" } });
329 expect(out.parts.some((p) => p.type === "tool-result")).toBe(true);
330 }),
331 { tags },
332 );
333
334 test.effect(
335 "with tools: prose after the block is emitted once",
336 () =>
337 Effect.gen(function* () {
338 const out = yield* stream(["<think>", "no tool", "</think>", "\n\n", "Pong"].map(native), {
339 tools: true,
340 });
341 expectSummary(out, { reasoning: "no tool", text: "Pong" });
342 expect(out.parts.some((p) => p.type === "tool-call")).toBe(false);
343 }),
344 { tags },
345 );
346 });
347
348 describe("streamText chunk-shape regressions", () => {
349 test.effect(
350 "text mirrored in `response` and `delta.content` is emitted once (#1907)",
351 () =>
352 Effect.gen(function* () {
353 const mirrored = (t: string) => ({ ...openai({ content: t }), response: t });
354 const out = yield* stream(["Hi", "!"].map(mirrored));
355 expectSummary(out, { text: "Hi!" });
356 const withTools = yield* stream(["Hi", "!"].map(mirrored), { tools: true });
357 expectSummary(withTools, { text: "Hi!" });
358 }),
359 { tags },
360 );
361
362 test.effect(
363 "mirrored reasoning fields with empty `tool_calls` form one reasoning block",
364 () =>
365 Effect.gen(function* () {
366 const reasoning = (t: string) => openai({ reasoning: t, reasoning_content: t });
367 const out = yield* stream([
368 reasoning("a"),
369 reasoning("b"),
370 reasoning("c"),
371 openai({ content: "answer" }),
372 ]);
373 expectSummary(out, { reasoning: "abc", text: "answer", shape: REASONED });
374 }),
375 { tags },
376 );
377 });
378
379 describe("generateText <think> splitting", () => {
380 test.effect(
381 "a leading block is split from the answer",
382 () =>
383 Effect.gen(function* () {
384 const out = yield* generate({ response: "\n<think>\nplan\n</think>\n\nanswer" });
385 expect(out).toEqual({ reasoning: "\nplan\n", text: "answer" });
386 }),
387 { tags },
388 );
389
390 test.effect(
391 "a bare `</think>` marks reasoning on any model",
392 () =>
393 Effect.gen(function* () {
394 const out = yield* generate({ response: "plan\n</think>\n\nanswer" });
395 expect(out).toEqual({ reasoning: "plan\n", text: "answer" });
396 }),
397 { tags },
398 );
399
400 test.effect(
401 "an unclosed block (truncated) is all reasoning",
402 () =>
403 Effect.gen(function* () {
404 expect(yield* generate({ response: "<think>still going" })).toEqual({
405 reasoning: "still going",
406 text: "",
407 });
408 expect(yield* generate({ response: "still going" }, QWQ)).toEqual({
409 reasoning: "still going",
410 text: "",
411 });
412 }),
413 { tags },
414 );
415
416 test.effect(
417 "text without a leading block is left untouched",
418 () =>
419 Effect.gen(function* () {
420 // `<`/`<thi` are held back as possible tag prefixes and must come back.
421 for (const response of [
422 "answer",
423 "<b>bold</b>",
424 "Use <think> tags.",
425 "<thinking>x",
426 "<",
427 "<thi",
428 " <thi",
429 ]) {
430 expect(yield* generate({ response })).toEqual({ reasoning: undefined, text: response });
431 }
432 }),
433 { tags },
434 );
435
436 test.effect(
437 "a separate reasoning field passes through with the answer",
438 () =>
439 Effect.gen(function* () {
440 const out = yield* generate({
441 choices: [
442 { message: { content: "answer", reasoning_content: "plan" }, finish_reason: "stop" },
443 ],
444 });
445 expect(out).toEqual({ reasoning: "plan", text: "answer" });
446 }),
447 { tags },
448 );
449 });