The best way to control Claude Code, Codex and more from any computer or phone

main
ts 242 lines 9.25 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Core from "@alchemy.run/test/Core";
5 import * as Data from "effect/Data";
6 import * as Effect from "effect/Effect";
7 import * as HttpClient from "effect/http/HttpClient";
8 import * as HttpClientRequest from "effect/http/HttpClientRequest";
9 import * as Schedule from "effect/Schedule";
10 import BedrockTestFunctionLive, { BedrockTestFunction } from "./handler";
11
12 const testOptions = { providers: AWS.providers() };
13 const { test, beforeAll, afterAll } = Test.make(testOptions);
14 const sharedStack = Core.scratchStack(testOptions, "BedrockBindings");
15
16 // Lambda function URL cold-start (DNS, IAM propagation, init) can take well
17 // over 60s on a fresh deploy under parallel-suite load. Budget ~150s of
18 // readiness polling.
19 const readinessPolicy = Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(75)]);
20
21 let baseUrl: string;
22
23 class TransientUpstream extends Data.TaggedError("TransientUpstream")<{
24 readonly status: number;
25 readonly body: string;
26 }> {}
27
28 // The shared Lambda fixture occasionally answers a transient 5xx under
29 // parallel load (cold re-init, IAM propagation, Bedrock throttling surfaced
30 // as a defect). Retry 5xx only; a genuine 4xx/assertion failure surfaces
31 // immediately.
32 const send = (request: HttpClientRequest.HttpClientRequest) =>
33 HttpClient.execute(request).pipe(
34 Effect.flatMap((response) =>
35 response.status >= 500
36 ? response.text.pipe(
37 Effect.flatMap((body) =>
38 Effect.fail(new TransientUpstream({ status: response.status, body })),
39 ),
40 )
41 : Effect.succeed(response),
42 ),
43 Effect.retry({
44 while: (e) => e._tag === "TransientUpstream",
45 schedule: Schedule.max([Schedule.exponential("1 second"), Schedule.recurs(5)]),
46 }),
47 );
48
49 describe(
50 "Bedrock Bindings",
51 { tags: ["provider:aws", "provider:aws:bedrock", "provider:aws:lambda", "live"] },
52 () => {
53 beforeAll(
54 Effect.gen(function* () {
55 yield* Effect.logInfo("Bedrock test setup: destroying previous resources");
56 yield* sharedStack.destroy();
57
58 yield* Effect.logInfo("Bedrock test setup: deploying fixture");
59 const { functionUrl } = yield* sharedStack.deploy(
60 Effect.gen(function* () {
61 return yield* BedrockTestFunction;
62 }).pipe(Effect.provide(BedrockTestFunctionLive)),
63 );
64
65 expect(functionUrl).toBeTruthy();
66 baseUrl = functionUrl!.replace(/\/+$/, "");
67 const readinessUrl = `${baseUrl}/ping`;
68
69 yield* Effect.logInfo(`Bedrock test setup: probing readiness at ${readinessUrl}`);
70 yield* HttpClient.get(readinessUrl).pipe(
71 Effect.flatMap((response) =>
72 response.status === 200
73 ? Effect.succeed(response)
74 : Effect.fail(new Error(`Function not ready: ${response.status}`)),
75 ),
76 Effect.tapError((error) =>
77 Effect.logWarning(`Bedrock test setup: fixture not ready yet (${String(error)})`),
78 ),
79 Effect.retry({ schedule: readinessPolicy }),
80 );
81 }),
82 { timeout: 240_000 },
83 );
84
85 afterAll.skipIf(!!process.env.NO_DESTROY)(sharedStack.destroy(), { timeout: 120_000 });
86
87 describe("CountTokens", () => {
88 test.provider(
89 "counts input tokens without invoking the model",
90 (_stack) =>
91 Effect.gen(function* () {
92 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/count-tokens`)).pipe(
93 Effect.flatMap((r) => r.json),
94 )) as { inputTokens: number };
95
96 expect(response.inputTokens).toBeGreaterThan(0);
97 }),
98 { timeout: 120_000 },
99 );
100 });
101
102 describe("Rerank", () => {
103 test.provider(
104 "reranks inline documents by relevance to the query",
105 (_stack) =>
106 Effect.gen(function* () {
107 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/rerank`)).pipe(
108 Effect.flatMap((r) => r.json),
109 )) as { results: Array<{ index: number; relevanceScore: number }> };
110
111 expect(response.results).toHaveLength(2);
112 // The alchemy document must outrank the banana document.
113 expect(response.results[0]!.index).toBe(0);
114 expect(response.results[0]!.relevanceScore).toBeGreaterThan(
115 response.results[1]!.relevanceScore,
116 );
117 }),
118 { timeout: 120_000 },
119 );
120 });
121
122 describe("InvokeAgent", () => {
123 test.provider(
124 "invokes the bound agent alias and streams a completion",
125 (_stack) =>
126 Effect.gen(function* () {
127 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/invoke-agent`)).pipe(
128 Effect.flatMap((r) => r.json),
129 )) as { sessionId: string; chunkEvents: number; text: string };
130
131 expect(response.sessionId.length).toBeGreaterThan(0);
132 expect(response.chunkEvents).toBeGreaterThan(0);
133 // Assert a non-empty completion — never exact model output.
134 expect(response.text.trim().length).toBeGreaterThan(0);
135 }),
136 { timeout: 120_000 },
137 );
138 });
139
140 describe("GetAgentMemory + DeleteAgentMemory", () => {
141 test.provider(
142 "reads and clears the agent's long-term memory for a memory id",
143 (_stack) =>
144 Effect.gen(function* () {
145 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/agent-memory`)).pipe(
146 Effect.flatMap((r) => r.json),
147 )) as { memoryContents: number; deleted: boolean };
148
149 // Session summaries generate asynchronously long after a session
150 // ends, so a fresh agent has none — the route proves the IAM
151 // grants and the read+delete round-trip, not summary generation.
152 expect(response.memoryContents).toBeGreaterThanOrEqual(0);
153 expect(response.deleted).toBe(true);
154 }),
155 { timeout: 120_000 },
156 );
157 });
158
159 describe("Converse", () => {
160 test.provider(
161 "converses with the bound model and returns a completion",
162 (_stack) =>
163 Effect.gen(function* () {
164 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/converse`)).pipe(
165 Effect.flatMap((r) => r.json),
166 )) as { text: string; stopReason: string; outputTokens: number };
167
168 // Assert a non-empty completion — never exact model output (LLMs
169 // are nondeterministic even at temperature 0).
170 expect(response.text.trim().length).toBeGreaterThan(0);
171 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
172 expect(response.outputTokens).toBeGreaterThan(0);
173 }),
174 { timeout: 120_000 },
175 );
176 });
177
178 describe("ConverseStream", () => {
179 test.provider(
180 "streams a conversation as typed events",
181 (_stack) =>
182 Effect.gen(function* () {
183 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/converse-stream`)).pipe(
184 Effect.flatMap((r) => r.json),
185 )) as {
186 text: string;
187 deltaEvents: number;
188 totalEvents: number;
189 stopReason: string | undefined;
190 };
191
192 // Assert a non-empty streamed completion — never exact model output.
193 expect(response.text.trim().length).toBeGreaterThan(0);
194 expect(response.deltaEvents).toBeGreaterThan(0);
195 expect(response.totalEvents).toBeGreaterThan(response.deltaEvents);
196 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
197 }),
198 { timeout: 120_000 },
199 );
200 });
201
202 describe("InvokeModelWithResponseStream", () => {
203 test.provider(
204 "streams raw payload chunks from the bound model",
205 (_stack) =>
206 Effect.gen(function* () {
207 const response = (yield* send(
208 HttpClientRequest.get(`${baseUrl}/invoke-model-stream`),
209 ).pipe(Effect.flatMap((r) => r.json))) as {
210 contentType: string;
211 chunkEvents: number;
212 text: string;
213 };
214
215 expect(response.contentType).toBe("application/json");
216 expect(response.chunkEvents).toBeGreaterThan(0);
217 // Assert a non-empty streamed completion — never exact model output.
218 expect(response.text.trim().length).toBeGreaterThan(0);
219 }),
220 { timeout: 120_000 },
221 );
222 });
223
224 describe("InvokeModel", () => {
225 test.provider(
226 "invokes the bound model with a raw payload and streams the response",
227 (_stack) =>
228 Effect.gen(function* () {
229 const response = (yield* send(HttpClientRequest.get(`${baseUrl}/invoke-model`)).pipe(
230 Effect.flatMap((r) => r.json),
231 )) as { contentType: string; text: string; stopReason: string };
232
233 expect(response.contentType).toBe("application/json");
234 // Assert a non-empty completion — never exact model output.
235 expect(response.text.trim().length).toBeGreaterThan(0);
236 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
237 }),
238 { timeout: 120_000 },
239 );
240 });
241 },
242 );