@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 163 lines 6.02 KB
Raw
1 import * as Bedrock from "@alchemy.run/aws/Bedrock";
2 import * as Lambda from "@alchemy.run/aws/Lambda";
3 import { LanguageModel as AiLanguageModel, Tool, Toolkit } from "effect/ai";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import { HttpServerRequest } from "effect/http/HttpServerRequest";
7 import * as HttpServerResponse from "effect/http/HttpServerResponse";
8 import * as Schema from "effect/Schema";
9 import * as Stream from "effect/Stream";
10 import path from "pathe";
11
12 const main = path.resolve(import.meta.dirname, "language-model-handler.ts");
13
14 // Nova Micro via the us cross-region inference profile: the cheapest
15 // on-demand conversational model in the testing account, with full
16 // Converse tool-use + streaming support.
17 const MODEL = "us.amazon.nova-micro-v1:0";
18 // A second bound model to exercise the per-call `modelId` override.
19 const LITE_MODEL = "us.amazon.nova-lite-v1:0";
20
21 const GetWeather = Tool.make("get_weather", {
22 description:
23 "Get the current weather for a city. Always call this tool when the user asks about the weather.",
24 parameters: Schema.Struct({ city: Schema.String }),
25 success: Schema.Struct({
26 city: Schema.String,
27 temperatureF: Schema.Number,
28 condition: Schema.String,
29 }),
30 });
31
32 const WeatherToolkit = Toolkit.make(GetWeather);
33
34 const WeatherToolkitLayer = WeatherToolkit.toLayer({
35 get_weather: ({ city }) => Effect.succeed({ city, temperatureF: 72, condition: "sunny" }),
36 });
37
38 const toSse = (parts: Iterable<unknown>): string =>
39 [...parts].map((part) => `data: ${JSON.stringify(part)}\n\n`).join("");
40
41 export class BedrockLanguageModelFunction extends Lambda.Function<Lambda.Function>()(
42 "BedrockLanguageModelFunction",
43 ) {}
44
45 export default BedrockLanguageModelFunction.make(
46 {
47 main,
48 functionUrl: true,
49 // Model inference regularly exceeds Lambda's 3s default timeout.
50 timeout: Duration.seconds(120),
51 },
52 Effect.gen(function* () {
53 const model = yield* Bedrock.LanguageModel([MODEL, LITE_MODEL], {
54 parameters: { maxTokens: 1024, temperature: 0.2 },
55 });
56
57 return {
58 fetch: Effect.gen(function* () {
59 const request = yield* HttpServerRequest;
60 const url = new URL(request.originalUrl);
61 const pathname = url.pathname;
62 const prompt =
63 url.searchParams.get("prompt") ?? "Say the single word 'pong' and nothing else.";
64
65 // Cheap readiness route — no Bedrock call.
66 if (pathname === "/ping") {
67 return yield* HttpServerResponse.json({ ok: true });
68 }
69
70 if (pathname === "/generate") {
71 const response = yield* AiLanguageModel.generateText({ prompt });
72 return yield* HttpServerResponse.json({
73 text: response.text,
74 finishReason: response.finishReason,
75 usage: {
76 inputTokens: response.usage.inputTokens.total,
77 outputTokens: response.usage.outputTokens.total,
78 },
79 });
80 }
81
82 if (pathname === "/generate-short") {
83 // Runtime override: clamp the same bound model to a tiny budget.
84 const response = yield* AiLanguageModel.generateText({ prompt }).pipe(
85 Bedrock.withModelParameters({ maxTokens: 8, temperature: 0 }),
86 );
87 return yield* HttpServerResponse.json({
88 text: response.text,
89 finishReason: response.finishReason,
90 outputTokens: response.usage.outputTokens.total,
91 });
92 }
93
94 if (pathname === "/generate-lite") {
95 // Runtime override: route this call to the second bound model.
96 const response = yield* AiLanguageModel.generateText({ prompt }).pipe(
97 Bedrock.withModelParameters({ modelId: LITE_MODEL }),
98 );
99 return yield* HttpServerResponse.json({
100 text: response.text,
101 finishReason: response.finishReason,
102 });
103 }
104
105 if (pathname === "/stream") {
106 // Collected server-side: Lambda function URLs buffer responses by
107 // default, and the tests assert on the part sequence, not
108 // incremental delivery.
109 const parts = yield* Stream.runCollect(AiLanguageModel.streamText({ prompt }));
110 return HttpServerResponse.text(toSse(parts), {
111 headers: { "content-type": "text/event-stream" },
112 });
113 }
114
115 if (pathname === "/tool") {
116 const response = yield* AiLanguageModel.generateText({
117 prompt,
118 toolkit: WeatherToolkit,
119 toolChoice: "required",
120 }).pipe(Effect.provide(WeatherToolkitLayer));
121 return yield* HttpServerResponse.json({
122 text: response.text,
123 finishReason: response.finishReason,
124 toolCalls: response.toolCalls.map((call) => ({
125 id: call.id,
126 name: call.name,
127 params: call.params,
128 })),
129 toolResults: response.toolResults.map((result) => ({
130 id: result.id,
131 name: result.name,
132 result: result.result,
133 isFailure: result.isFailure,
134 })),
135 });
136 }
137
138 if (pathname === "/tool-stream") {
139 const parts = yield* Stream.runCollect(
140 AiLanguageModel.streamText({
141 prompt,
142 toolkit: WeatherToolkit,
143 toolChoice: "required",
144 }).pipe(Stream.provide(WeatherToolkitLayer)),
145 );
146 return HttpServerResponse.text(toSse(parts), {
147 headers: { "content-type": "text/event-stream" },
148 });
149 }
150
151 return yield* HttpServerResponse.json({ error: "Not found", pathname }, { status: 404 });
152 }).pipe(
153 // Surface adapter/model failures as a JSON 500 so live-test runs can
154 // read the failure without digging through CloudWatch.
155 Effect.catchTag("AiError", (error) =>
156 HttpServerResponse.json({ error: String(error.message) }, { status: 500 }),
157 ),
158 Effect.provide(model),
159 Effect.orDie,
160 ),
161 };
162 }).pipe(Effect.provide(Bedrock.LanguageModelHttp)),
163 );