The best way to control Claude Code, Codex and more from any computer or phone

main
ts 356 lines 11.7 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { makeDynamoDBState, makeS3State } from "@alchemy.run/aws";
3 import { describe, expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as Alchemy from "alchemy";
6 import type { StackServices } from "alchemy/Stack";
7 import type { ResourceState, StateService, StateStoreError } from "alchemy/State";
8 import type { State } from "alchemy/State/State";
9 import * as Duration from "effect/Duration";
10 import * as Effect from "effect/Effect";
11 import * as FileSystem from "effect/FileSystem";
12 import * as Layer from "effect/Layer";
13 import * as Path from "effect/Path";
14
15 /**
16 * Benchmarks the AWS state stores against each other.
17 *
18 * The store cases drive each store with the calls a plan makes (`list`, a
19 * `get` per record, `getReplacedResources`, all unbounded, as `Plan.ts`
20 * issues them) and the writes an apply makes. The stack cases deploy the
21 * same Stack of SQS queues once per store.
22 *
23 * Timings go to the run log and to `.alchemy/benchmarks/state-store.md`.
24 * Only correctness is asserted: wall-clock numbers vary too much between
25 * runs to gate on.
26 */
27
28 const { test, afterAll } = Test.make({ providers: AWS.providers() });
29
30 const tags = [
31 "provider:aws",
32 "provider:aws:dynamodb",
33 "provider:aws:s3",
34 "provider:aws:statestore",
35 "benchmark",
36 "live",
37 ];
38
39 interface Row {
40 readonly suite: string;
41 readonly backend: string;
42 readonly step: string;
43 readonly ms: number;
44 }
45
46 const rows: Row[] = [];
47
48 /** Run `effect`, record how long it took, and return its result. */
49 const time = <A, E, R>(
50 suite: string,
51 backend: string,
52 step: string,
53 effect: Effect.Effect<A, E, R>,
54 ) =>
55 Effect.timed(effect).pipe(
56 Effect.tap(([duration]) =>
57 Effect.gen(function* () {
58 const ms = Math.round(Duration.toMillis(duration));
59 rows.push({ suite, backend, step, ms });
60 yield* Effect.logInfo(`[bench] ${suite} | ${backend} | ${step}: ${ms}ms`);
61 }),
62 ),
63 Effect.map(([, result]) => result),
64 );
65
66 /** A resource state shaped like a real one, about `bytes` long encoded. */
67 const resource = (fqn: string, bytes: number, status = "created"): ResourceState =>
68 ({
69 resourceType: "AWS.Lambda.Function",
70 namespace: undefined,
71 fqn,
72 logicalId: fqn,
73 instanceId: `instance-${fqn}`,
74 providerVersion: 1,
75 status,
76 downstream: [],
77 bindings: [],
78 props: { functionName: `${fqn}-function`, memorySize: 1024, timeout: 30 },
79 attr: {
80 functionArn: `arn:aws:lambda:us-west-2:123456789012:function:${fqn}`,
81 padding: "x".repeat(Math.max(0, bytes - 400)),
82 },
83 }) as ResourceState;
84
85 const STACK = "StateStoreBenchmark";
86
87 interface Workload {
88 readonly name: string;
89 readonly records: number;
90 readonly bytes: number;
91 }
92
93 const workloads: ReadonlyArray<Workload> = [
94 { name: "50 records x 1 KiB", records: 50, bytes: 1024 },
95 { name: "250 records x 1 KiB", records: 250, bytes: 1024 },
96 // Above the DynamoDB store's 300 KiB threshold: every record overflows.
97 { name: "10 records x 350 KiB", records: 10, bytes: 350 * 1024 },
98 ];
99
100 /**
101 * What a deploy does to the store: write every record, then a fresh
102 * process plans (list + get all + replaced), plans again warm, and the
103 * stage is destroyed.
104 */
105 const runWorkload = Effect.fn(function* <R>(
106 workload: Workload,
107 backend: string,
108 make: Effect.Effect<StateService, StateStoreError, R>,
109 ) {
110 const stage = `bench-${workload.records}-${workload.bytes}`;
111 const values = Array.from({ length: workload.records }, (_, i) =>
112 resource(`Resource${i}`, workload.bytes, i % 10 === 0 ? "replaced" : "created"),
113 );
114 const bench = <A, E, R>(step: string, effect: Effect.Effect<A, E, R>) =>
115 time(workload.name, backend, step, effect);
116
117 const plan = (state: StateService) =>
118 Effect.gen(function* () {
119 const fqns = yield* state.list({ stack: STACK, stage });
120 const found = yield* Effect.forEach(fqns, (fqn) => state.get({ stack: STACK, stage, fqn }), {
121 concurrency: "unbounded",
122 });
123 const replaced = yield* state.getReplacedResources({ stack: STACK, stage });
124 return { found, replaced };
125 });
126
127 // Start from an empty stage.
128 yield* Effect.scoped(Effect.flatMap(make, (state) => state.deleteStack({ stack: STACK, stage })));
129
130 yield* Effect.scoped(
131 Effect.gen(function* () {
132 const state = yield* make;
133 yield* bench(
134 "write all (unbounded)",
135 Effect.forEach(
136 values,
137 (value) => state.set({ stack: STACK, stage, fqn: value.fqn, value }),
138 {
139 concurrency: "unbounded",
140 },
141 ),
142 );
143 yield* bench(
144 "update all (sequential)",
145 Effect.forEach(
146 values,
147 (value) => state.set({ stack: STACK, stage, fqn: value.fqn, value }),
148 {
149 discard: true,
150 },
151 ),
152 );
153 }),
154 );
155
156 yield* Effect.scoped(
157 Effect.gen(function* () {
158 const state = yield* make;
159 const cold = yield* bench("plan, cold (new store)", plan(state));
160 expect(cold.found.filter((value) => value !== undefined)).toHaveLength(workload.records);
161 expect(cold.replaced).toHaveLength(Math.ceil(workload.records / 10));
162 const warm = yield* bench("plan, warm (same store)", plan(state));
163 expect(warm.found).toHaveLength(workload.records);
164 yield* bench("delete stage", state.deleteStack({ stack: STACK, stage }));
165 }),
166 );
167 });
168
169 const StoreStorage = Effect.gen(function* () {
170 const table = yield* AWS.DynamoDB.Table("BenchTable", {
171 partitionKey: "PK",
172 sortKey: "SK",
173 attributes: { PK: "S", SK: "S" },
174 });
175 const overflow = yield* AWS.S3.Bucket("BenchOverflow", { forceDestroy: true });
176 const bucket = yield* AWS.S3.Bucket("BenchStateBucket", { forceDestroy: true });
177 return {
178 tableName: table.tableName,
179 overflowBucket: overflow.bucketName,
180 stateBucket: bucket.bucketName,
181 };
182 });
183
184 describe.sequential("state store operations", { tags }, () => {
185 test.provider.skipIf(!!process.env.FAST)(
186 "S3 and DynamoDB stores under plan and apply workloads",
187 (stack) =>
188 Effect.gen(function* () {
189 yield* stack.destroy();
190 const storage = yield* stack.deploy(StoreStorage);
191 for (const workload of workloads) {
192 yield* runWorkload(
193 workload,
194 "s3",
195 makeS3State({ bucketName: storage.stateBucket, prefix: "bench" }),
196 );
197 yield* runWorkload(
198 workload,
199 "dynamodb",
200 makeDynamoDBState({
201 tableName: storage.tableName,
202 overflowBucketName: storage.overflowBucket,
203 migrateFrom: false,
204 }),
205 );
206 }
207 yield* stack.destroy();
208 }),
209 { timeout: 900_000 },
210 );
211 });
212
213 const QUEUES = 20;
214
215 /** The same Stack of SQS queues, named per store so the runs don't share state. */
216 const queueStack = <S extends Layer.Layer<State, never, StackServices>>(name: string, state: S) => {
217 const options = { providers: AWS.providers(), state };
218 return {
219 harness: Test.make(options),
220 stack: Alchemy.Stack(
221 name,
222 options,
223 Effect.gen(function* () {
224 const queues = yield* Effect.forEach(
225 Array.from({ length: QUEUES }, (_, i) => i),
226 (i) => AWS.SQS.Queue(`Queue${i}`),
227 { concurrency: "unbounded" },
228 );
229 return { count: queues.length };
230 }),
231 ),
232 };
233 };
234
235 const stacks = [
236 { backend: "s3", ...queueStack("StateBenchS3", AWS.state({ backend: "s3" })) },
237 { backend: "dynamodb", ...queueStack("StateBenchDynamoDB", AWS.state()) },
238 { backend: "local", ...queueStack("StateBenchLocal", Alchemy.localState()) },
239 ];
240
241 describe.sequential(
242 `a Stack of ${QUEUES} SQS queues`,
243 { tags: [...tags, "provider:aws:sqs"] },
244 () => {
245 for (const { backend, harness, stack } of stacks) {
246 harness.test.skipIf(!!process.env.FAST)(
247 `deploys, redeploys and destroys on the ${backend} store`,
248 Effect.gen(function* () {
249 const suite = `Stack of ${QUEUES} queues`;
250 yield* harness.destroy(stack);
251 const fresh = yield* time(suite, backend, "fresh deploy", harness.deploy(stack));
252 expect(fresh.count).toBe(QUEUES);
253 yield* time(suite, backend, "unchanged redeploy #1", harness.deploy(stack));
254 yield* time(suite, backend, "unchanged redeploy #2", harness.deploy(stack));
255 yield* time(suite, backend, "destroy", harness.destroy(stack));
256 }),
257 { timeout: 600_000 },
258 );
259 }
260 },
261 );
262
263 /**
264 * Stacks of `Alchemy.Random` resources: they live only in state, so these
265 * cases measure the engine and the store at sizes a Stack of cloud
266 * resources would take far longer (and cost more) to reach.
267 */
268 const SIZES = [1_000, 5_000, 10_000];
269
270 const randomStack = <S extends Layer.Layer<State, never, StackServices>>(
271 name: string,
272 state: S,
273 size: number,
274 ) => {
275 const options = { providers: AWS.providers(), state };
276 return {
277 harness: Test.make(options),
278 stack: Alchemy.Stack(
279 name,
280 options,
281 Effect.gen(function* () {
282 const randoms = yield* Effect.forEach(
283 Array.from({ length: size }, (_, i) => i),
284 (i) => Alchemy.Random(`Random${i}`),
285 { concurrency: "unbounded" },
286 );
287 return { count: randoms.length };
288 }),
289 ),
290 };
291 };
292
293 describe.sequential("large Stacks", { tags }, () => {
294 for (const size of SIZES) {
295 const large = [
296 { backend: "s3", ...randomStack(`StateBenchS3x${size}`, AWS.state({ backend: "s3" }), size) },
297 {
298 backend: "dynamodb",
299 ...randomStack(`StateBenchDynamoDBx${size}`, AWS.state(), size),
300 },
301 { backend: "local", ...randomStack(`StateBenchLocalx${size}`, Alchemy.localState(), size) },
302 ];
303 for (const { backend, harness, stack } of large) {
304 harness.test.skipIf(!!process.env.FAST)(
305 `${size} resources deploy, redeploy and destroy on the ${backend} store`,
306 Effect.gen(function* () {
307 const suite = `Stack of ${size} resources`;
308 yield* harness.destroy(stack);
309 const fresh = yield* time(suite, backend, "fresh deploy", harness.deploy(stack));
310 expect(fresh.count).toBe(size);
311 yield* time(suite, backend, "unchanged redeploy", harness.deploy(stack));
312 yield* time(suite, backend, "destroy", harness.destroy(stack));
313 }),
314 // S3 takes about half a second per resource per step.
315 { timeout: size * 2_000 },
316 );
317 }
318 }
319 });
320
321 /** One table per suite: a step per row, a store per column. */
322 const report = () => {
323 const lines: string[] = ["# State store benchmark", ""];
324 for (const suite of Array.from(new Set(rows.map((row) => row.suite)))) {
325 const inSuite = rows.filter((row) => row.suite === suite);
326 const backends = Array.from(new Set(inSuite.map((row) => row.backend)));
327 lines.push(
328 `## ${suite}`,
329 "",
330 `| step | ${backends.join(" | ")} |`,
331 `|---|${backends.map(() => "---:").join("|")}|`,
332 );
333 for (const step of Array.from(new Set(inSuite.map((row) => row.step)))) {
334 const cells = backends.map((backend) => {
335 const row = inSuite.find((r) => r.backend === backend && r.step === step);
336 return row === undefined ? "" : `${row.ms} ms`;
337 });
338 lines.push(`| ${step} | ${cells.join(" | ")} |`);
339 }
340 lines.push("");
341 }
342 return lines.join("\n");
343 };
344
345 afterAll(
346 Effect.gen(function* () {
347 if (rows.length === 0) return;
348 const fs = yield* FileSystem.FileSystem;
349 const path = yield* Path.Path;
350 const markdown = report();
351 yield* Effect.logInfo(`\n${markdown}`);
352 const dir = path.join(yield* Effect.sync(() => process.cwd()), ".alchemy", "benchmarks");
353 yield* fs.makeDirectory(dir, { recursive: true });
354 yield* fs.writeFileString(path.join(dir, "state-store.md"), markdown);
355 }).pipe(Effect.orDie),
356 );