The best way to control Claude Code, Codex and more from any computer or phone

main
ts 229 lines 8.83 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import type { RuntimeContext } from "alchemy/RuntimeContext";
3 import { LanguageModel as AiLanguageModel, Tool, Toolkit } from "effect/ai";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import { HttpServerRequest } from "effect/http/HttpServerRequest";
7 import * as HttpServerResponse from "effect/http/HttpServerResponse";
8 import * as Schema from "effect/Schema";
9 import * as Stream from "effect/Stream";
10 import { Gateway } from "./Gateway.ts";
11
12 // `@cf/meta/llama-3.1-8b-instruct` was deprecated by Cloudflare on
13 // 2026-05-30 (the API answers 410), so use the supported fast 3.3 model.
14 const MODEL = "@cf/meta/llama-3.3-70b-instruct-fp8-fast";
15 // Tool tests need a model that reliably honors `tool_choice: "required"`.
16 // The previous choice (`@cf/moonshotai/kimi-k2.6`) is a reasoning model
17 // that intermittently emitted reasoning/text and *no* tool call, flaking
18 // the tool assertions. Llama 3.3 70B is non-reasoning and reliably returns
19 // an OpenAI-shaped `tool_calls` array under `tool_choice: "required"`.
20 const TOOL_MODEL = "@cf/meta/llama-3.3-70b-instruct-fp8-fast";
21
22 const GetWeather = Tool.make("get_weather", {
23 description:
24 "Get the current weather for a city. Always call this tool when the user asks about the weather.",
25 parameters: Schema.Struct({ city: Schema.String }),
26 success: Schema.Struct({
27 city: Schema.String,
28 temperatureF: Schema.Number,
29 condition: Schema.String,
30 }),
31 });
32
33 const WeatherToolkit = Toolkit.make(GetWeather);
34
35 const WeatherToolkitLayer = WeatherToolkit.toLayer({
36 get_weather: ({ city }) => Effect.succeed({ city, temperatureF: 72, condition: "sunny" }),
37 });
38
39 export default class LanguageModelTestWorker extends Cloudflare.Worker<LanguageModelTestWorker>()(
40 "LanguageModelTestWorker",
41 { main: import.meta.url },
42 Effect.gen(function* () {
43 const aiGateway = yield* Cloudflare.AI.QueryGateway(Gateway);
44
45 const languageModel = aiGateway.model({
46 model: MODEL,
47 parameters: { temperature: 0.7, maxTokens: 1024 },
48 });
49 const toolLanguageModel = aiGateway.model({
50 model: TOOL_MODEL,
51 parameters: { temperature: 0.2, maxTokens: 1024 },
52 });
53 const dumpRawStream = (
54 model: string,
55 messages: ReadonlyArray<{ role: string; content: string }>,
56 streamOptions: Record<string, unknown> | undefined,
57 ) =>
58 Effect.gen(function* () {
59 const ai = yield* aiGateway.raw;
60 const gatewayId = yield* aiGateway.id;
61 const response = yield* Effect.tryPromise({
62 try: () =>
63 ai.run(
64 model as keyof AiModels,
65 {
66 messages,
67 stream: true,
68 ...streamOptions,
69 } as unknown as AiModels[keyof AiModels]["inputs"],
70 { gateway: { id: gatewayId }, returnRawResponse: true },
71 ),
72 catch: (e) => e,
73 });
74 if (response.body == null) return HttpServerResponse.text("no body", { status: 500 });
75 const body = Stream.fromReadableStream<Uint8Array, never>({
76 evaluate: () => response.body!,
77 onError: () => undefined as never,
78 });
79 return HttpServerResponse.stream(body, {
80 headers: { "content-type": "text/event-stream" },
81 });
82 }).pipe(Effect.orDie);
83
84 return {
85 fetch: Effect.gen(function* () {
86 const request = yield* HttpServerRequest;
87 const ctx = yield* Effect.context<RuntimeContext>();
88 const url = new URL(request.url, "http://worker");
89 const prompt =
90 url.searchParams.get("prompt") ?? "Say the single word 'pong' and nothing else.";
91
92 // `?model=` overrides the default model so tests can cover the
93 // different Workers AI response shapes.
94 const modelParam = url.searchParams.get("model");
95 const modelFor = (fallback: typeof languageModel, temperature: number) =>
96 modelParam
97 ? aiGateway.model({ model: modelParam, parameters: { temperature, maxTokens: 1024 } })
98 : fallback;
99
100 if (url.pathname === "/generate") {
101 const response = yield* AiLanguageModel.generateText({ prompt }).pipe(
102 Effect.provide(modelFor(languageModel, 0.7)),
103 Effect.orDie,
104 );
105 return yield* HttpServerResponse.json({
106 text: response.text,
107 reasoningText: response.reasoningText,
108 finishReason: response.finishReason,
109 usage: {
110 inputTokens: response.usage.inputTokens.total,
111 outputTokens: response.usage.outputTokens.total,
112 },
113 neurons: response.content
114 .filter((part) => part.type === "finish")
115 .map(Cloudflare.AI.finishNeurons)[0],
116 });
117 }
118
119 if (url.pathname === "/raw-stream") {
120 const model = modelParam ?? MODEL;
121 const includeUsage = url.searchParams.get("include_usage") === "1";
122 return yield* dumpRawStream(
123 model,
124 [{ role: "user", content: prompt }],
125 includeUsage ? { stream_options: { include_usage: true } } : undefined,
126 );
127 }
128
129 if (url.pathname === "/raw-tool-stream") {
130 // Dump the raw Workers AI SSE for a streamed tool call so we can
131 // see exactly how the model encodes tool_calls in the stream.
132 return yield* dumpRawStream(TOOL_MODEL, [{ role: "user", content: prompt }], {
133 tools: [
134 {
135 type: "function",
136 function: {
137 name: "get_weather",
138 description: "Get the current weather for a city.",
139 parameters: {
140 type: "object",
141 properties: { city: { type: "string" } },
142 required: ["city"],
143 },
144 },
145 },
146 ],
147 tool_choice: "required",
148 });
149 }
150
151 if (url.pathname === "/test-stream") {
152 // Synthetic stream: 5 chunks with 200ms gaps. If the client sees
153 // them at staggered timestamps, worker→edge streaming works.
154 // If they all land at once, output is buffered downstream.
155 const encoder = new TextEncoder();
156 const body = Stream.range(0, 5).pipe(
157 Stream.mapEffect((i) =>
158 Effect.sleep(Duration.millis(200)).pipe(
159 Effect.as(encoder.encode(`data: chunk-${i}\n\n`)),
160 ),
161 ),
162 );
163 return HttpServerResponse.stream(body, {
164 headers: { "content-type": "text/event-stream" },
165 });
166 }
167
168 if (url.pathname === "/tool") {
169 const response = yield* AiLanguageModel.generateText({
170 prompt,
171 toolkit: WeatherToolkit,
172 toolChoice: "required",
173 }).pipe(
174 Effect.provide(WeatherToolkitLayer),
175 Effect.provide(toolLanguageModel),
176 Effect.orDie,
177 );
178 return yield* HttpServerResponse.json({
179 text: response.text,
180 finishReason: response.finishReason,
181 toolCalls: response.toolCalls.map((call) => ({
182 id: call.id,
183 name: call.name,
184 params: call.params,
185 })),
186 toolResults: response.toolResults.map((result) => ({
187 id: result.id,
188 name: result.name,
189 result: result.result,
190 isFailure: result.isFailure,
191 })),
192 });
193 }
194
195 if (url.pathname === "/tool-stream") {
196 const encoder = new TextEncoder();
197 const toolChoice = url.searchParams.get("toolChoice") === "auto" ? "auto" : "required";
198 const body = AiLanguageModel.streamText({
199 prompt,
200 toolkit: WeatherToolkit,
201 toolChoice,
202 }).pipe(
203 Stream.map((part) => encoder.encode(`data: ${JSON.stringify(part)}\n\n`)),
204 Stream.provide(WeatherToolkitLayer),
205 Stream.provide(modelFor(toolLanguageModel, 0.2)),
206 Stream.provideContext(ctx),
207 );
208 return HttpServerResponse.stream(body, {
209 headers: { "content-type": "text/event-stream" },
210 });
211 }
212
213 if (url.pathname === "/stream") {
214 const encoder = new TextEncoder();
215 const body = AiLanguageModel.streamText({ prompt }).pipe(
216 Stream.map((part) => encoder.encode(`data: ${JSON.stringify(part)}\n\n`)),
217 Stream.provide(modelFor(languageModel, 0.7)),
218 Stream.provideContext(ctx),
219 );
220 return HttpServerResponse.stream(body, {
221 headers: { "content-type": "text/event-stream" },
222 });
223 }
224
225 return HttpServerResponse.text("ok");
226 }).pipe(Effect.provide(languageModel)),
227 };
228 }).pipe(Effect.provide(Cloudflare.AI.QueryGatewayBinding)),
229 ) {}