@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 287 lines 11.4 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Core from "@alchemy.run/test/Core";
5 import * as Data from "effect/Data";
6 import * as Effect from "effect/Effect";
7 import * as HttpClient from "effect/http/HttpClient";
8 import * as HttpClientRequest from "effect/http/HttpClientRequest";
9 import * as Schedule from "effect/Schedule";
10 import BedrockLanguageModelFunctionLive, {
11 BedrockLanguageModelFunction,
12 } from "./language-model-handler";
13
14 const testOptions = { providers: AWS.providers() };
15 const { test, beforeAll, afterAll } = Test.make(testOptions);
16 const sharedStack = Core.scratchStack(testOptions, "BedrockLanguageModel");
17
18 // Lambda function URL cold-start (DNS, IAM propagation, init) can take well
19 // over 60s on a fresh deploy under parallel-suite load. Budget ~150s of
20 // readiness polling.
21 const readinessPolicy = Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(75)]);
22
23 let baseUrl: string;
24
25 class TransientUpstream extends Data.TaggedError("TransientUpstream")<{
26 readonly status: number;
27 readonly body: string;
28 }> {}
29
30 // The shared Lambda fixture occasionally answers a transient 5xx under
31 // parallel load (cold re-init, IAM propagation, Bedrock throttling). Retry
32 // 5xx only; a genuine 4xx/assertion failure surfaces immediately.
33 const send = (request: HttpClientRequest.HttpClientRequest) =>
34 HttpClient.execute(request).pipe(
35 Effect.flatMap((response) =>
36 response.status >= 500
37 ? response.text.pipe(
38 Effect.flatMap((body) =>
39 Effect.fail(new TransientUpstream({ status: response.status, body })),
40 ),
41 )
42 : Effect.succeed(response),
43 ),
44 Effect.retry({
45 while: (e) => e._tag === "TransientUpstream",
46 schedule: Schedule.max([Schedule.exponential("1 second"), Schedule.recurs(5)]),
47 }),
48 );
49
50 interface StreamPart {
51 type: string;
52 id?: string;
53 name?: string;
54 delta?: string;
55 reason?: string;
56 usage?: { inputTokens?: { total?: number }; outputTokens?: { total?: number } };
57 }
58
59 const parseSse = (sse: string): ReadonlyArray<StreamPart> =>
60 sse
61 .split("\n\n")
62 .map((frame) => frame.replace(/^data:\s*/, "").trim())
63 .filter((line) => line.length > 0)
64 .map((line) => JSON.parse(line) as StreamPart);
65
66 describe(
67 "Bedrock LanguageModel",
68 { tags: ["provider:aws", "provider:aws:bedrock", "provider:aws:lambda", "live"] },
69 () => {
70 beforeAll(
71 Effect.gen(function* () {
72 yield* Effect.logInfo("Bedrock LanguageModel setup: destroying previous resources");
73 yield* sharedStack.destroy();
74
75 yield* Effect.logInfo("Bedrock LanguageModel setup: deploying fixture");
76 const { functionUrl } = yield* sharedStack.deploy(
77 Effect.gen(function* () {
78 return yield* BedrockLanguageModelFunction;
79 }).pipe(Effect.provide(BedrockLanguageModelFunctionLive)),
80 );
81
82 expect(functionUrl).toBeTruthy();
83 baseUrl = functionUrl!.replace(/\/+$/, "");
84 const readinessUrl = `${baseUrl}/ping`;
85
86 yield* Effect.logInfo(`Bedrock LanguageModel setup: probing readiness at ${readinessUrl}`);
87 yield* HttpClient.get(readinessUrl).pipe(
88 Effect.flatMap((response) =>
89 response.status === 200
90 ? Effect.succeed(response)
91 : Effect.fail(new Error(`Function not ready: ${response.status}`)),
92 ),
93 Effect.tapError((error) =>
94 Effect.logWarning(
95 `Bedrock LanguageModel setup: fixture not ready yet (${String(error)})`,
96 ),
97 ),
98 Effect.retry({ schedule: readinessPolicy }),
99 );
100 }),
101 { timeout: 240_000 },
102 );
103
104 afterAll.skipIf(!!process.env.NO_DESTROY)(sharedStack.destroy(), { timeout: 120_000 });
105
106 test.provider(
107 "generateText answers with text, usage, and a terminal finish reason",
108 (_stack) =>
109 Effect.gen(function* () {
110 const response = (yield* send(
111 HttpClientRequest.get(`${baseUrl}/generate?prompt=${encodeURIComponent("Say pong.")}`),
112 ).pipe(Effect.flatMap((r) => r.json))) as {
113 text: string;
114 finishReason: string;
115 usage: { inputTokens: number; outputTokens: number };
116 };
117
118 expect(typeof response.text).toBe("string");
119 expect(response.text.length).toBeGreaterThan(0);
120 expect(response.usage.inputTokens).toBeGreaterThan(0);
121 expect(response.usage.outputTokens).toBeGreaterThan(0);
122 // A normal Converse completion (`end_turn`) maps to `stop`.
123 expect(["stop", "length"]).toContain(response.finishReason);
124 }),
125 { timeout: 120_000 },
126 );
127
128 test.provider(
129 "withModelParameters overrides maxTokens at runtime",
130 (_stack) =>
131 Effect.gen(function* () {
132 // The binding's default is maxTokens: 1024; the route clamps to 8
133 // via withModelParameters. Truncation proves the override reached
134 // Bedrock: the model runs out of budget mid-answer.
135 const response = (yield* send(
136 HttpClientRequest.get(
137 `${baseUrl}/generate-short?prompt=${encodeURIComponent(
138 "Write a detailed multi-paragraph essay about the history of infrastructure as code.",
139 )}`,
140 ),
141 ).pipe(Effect.flatMap((r) => r.json))) as { finishReason: string; outputTokens: number };
142
143 expect(response.finishReason).toBe("length");
144 expect(response.outputTokens).toBeLessThanOrEqual(8);
145 }),
146 { timeout: 120_000 },
147 );
148
149 test.provider(
150 "withModelParameters routes a call to another bound model",
151 (_stack) =>
152 Effect.gen(function* () {
153 // The layer binds [nova-micro, nova-lite]; this route overrides
154 // modelId to nova-lite. A 200 with text proves the per-call model
155 // selection and the multi-model IAM grant both work.
156 const response = (yield* send(
157 HttpClientRequest.get(
158 `${baseUrl}/generate-lite?prompt=${encodeURIComponent("Say pong.")}`,
159 ),
160 ).pipe(Effect.flatMap((r) => r.json))) as { text: string; finishReason: string };
161
162 expect(response.text.length).toBeGreaterThan(0);
163 expect(["stop", "length"]).toContain(response.finishReason);
164 }),
165 { timeout: 120_000 },
166 );
167
168 test.provider(
169 "streamText emits ordered parts: text-start → text-delta+ → text-end → finish",
170 (_stack) =>
171 Effect.gen(function* () {
172 const sse = yield* send(
173 HttpClientRequest.get(
174 `${baseUrl}/stream?prompt=${encodeURIComponent(
175 "Write a short paragraph (around 80 words) about why TypeScript developers might enjoy Effect TS.",
176 )}`,
177 ),
178 ).pipe(Effect.flatMap((r) => r.text));
179 const parts = parseSse(sse);
180
181 const startIdx = parts.findIndex((p) => p.type === "text-start");
182 const firstDeltaIdx = parts.findIndex((p) => p.type === "text-delta");
183 const deltas = parts.filter((p) => p.type === "text-delta");
184 const lastDeltaIdx = parts.lastIndexOf(deltas[deltas.length - 1]!);
185 const endIdx = parts.findIndex((p) => p.type === "text-end");
186 const finishIdx = parts.findIndex((p) => p.type === "finish");
187
188 // Adapter invariant: a text block opens before its first delta and
189 // closes before the terminal finish part.
190 expect(startIdx).toBeGreaterThanOrEqual(0);
191 expect(firstDeltaIdx).toBeGreaterThan(startIdx);
192 expect(endIdx).toBeGreaterThan(lastDeltaIdx);
193 expect(finishIdx).toBe(parts.length - 1);
194
195 // Bedrock streams one text-delta per contentBlockDelta event — a
196 // long response must produce many (a single fused delta would mean
197 // the adapter buffered the stream).
198 expect(deltas.length).toBeGreaterThan(3);
199 const text = deltas.map((p) => p.delta ?? "").join("");
200 expect(text.length).toBeGreaterThan(20);
201
202 // The metadata event's usage must survive into the finish part.
203 const finish = parts[finishIdx]!;
204 expect(finish.usage?.inputTokens?.total).toBeGreaterThan(0);
205 expect(finish.usage?.outputTokens?.total).toBeGreaterThan(0);
206 expect(["stop", "length"]).toContain(finish.reason);
207 }),
208 { timeout: 120_000 },
209 );
210
211 test.provider(
212 "generateText invokes a tool and returns its result",
213 (_stack) =>
214 Effect.gen(function* () {
215 const response = (yield* send(
216 HttpClientRequest.get(
217 `${baseUrl}/tool?prompt=${encodeURIComponent(
218 "What's the weather in San Francisco?",
219 )}`,
220 ),
221 ).pipe(Effect.flatMap((r) => r.json))) as {
222 finishReason: string;
223 toolCalls: Array<{ id: string; name: string; params: { city: string } }>;
224 toolResults: Array<{
225 id: string;
226 name: string;
227 result: { city: string; temperatureF: number; condition: string };
228 isFailure: boolean;
229 }>;
230 };
231
232 expect(response.toolCalls.length).toBeGreaterThan(0);
233 const call = response.toolCalls[0]!;
234 expect(call.name).toBe("get_weather");
235 expect(typeof call.params.city).toBe("string");
236 expect(call.params.city.toLowerCase()).toContain("san francisco");
237
238 expect(response.toolResults.length).toBeGreaterThan(0);
239 const result = response.toolResults[0]!;
240 expect(result.name).toBe("get_weather");
241 expect(result.isFailure).toBe(false);
242 expect(result.result.temperatureF).toBe(72);
243 expect(result.result.condition).toBe("sunny");
244 }),
245 { timeout: 120_000 },
246 );
247
248 test.provider(
249 "streamText emits tool-params parts whose deltas concatenate to the arguments",
250 (_stack) =>
251 Effect.gen(function* () {
252 const sse = yield* send(
253 HttpClientRequest.get(
254 `${baseUrl}/tool-stream?prompt=${encodeURIComponent(
255 "What's the weather in Portland?",
256 )}`,
257 ),
258 ).pipe(Effect.flatMap((r) => r.text));
259 const parts = parseSse(sse);
260
261 const starts = parts.filter((p) => p.type === "tool-params-start");
262 const ends = parts.filter((p) => p.type === "tool-params-end");
263 expect(starts.length).toBeGreaterThan(0);
264 expect(starts[0]?.name).toBe("get_weather");
265 // Adapter invariant: every contentBlockStart-opened tool block is
266 // closed by contentBlockStop (or finalize), matched by id.
267 expect(new Set(ends.map((p) => p.id))).toEqual(new Set(starts.map((p) => p.id)));
268
269 const firstId = starts[0]!.id!;
270 const joined = parts
271 .filter((p) => p.type === "tool-params-delta" && p.id === firstId)
272 .map((p) => p.delta ?? "")
273 .join("");
274 const args = yield* Effect.try({
275 try: () => JSON.parse(joined) as { city?: string },
276 catch: (cause) =>
277 new Error(`Invalid concatenated tool arguments ${JSON.stringify(joined)}: ${cause}`),
278 });
279 expect(typeof args.city).toBe("string");
280 expect(args.city!.toLowerCase()).toContain("portland");
281
282 expect(parts[parts.length - 1]?.type).toBe("finish");
283 }),
284 { timeout: 120_000 },
285 );
286 },
287 );