The best way to control Claude Code, Codex and more from any computer or phone

main
ts 351 lines 13.4 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
3 import { describe, expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import { Credentials, formatHeaders } from "@distilled.cloud/cloudflare/Credentials";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as HttpClient from "effect/http/HttpClient";
9 import * as HttpClientRequest from "effect/http/HttpClientRequest";
10 import * as Schedule from "effect/Schedule";
11 import { requestWorker } from "../Utils/WorkerRequest.ts";
12 import BenchV1 from "./fixtures/bench/v1.ts";
13 import BenchV2Summary from "./fixtures/bench/v2-summary.ts";
14 import BenchV2Total from "./fixtures/bench/v2-total.ts";
15 import BenchV2 from "./fixtures/bench/v2.ts";
16
17 /**
18 * What a backfill costs and how long a deploy waits for it on Cloudflare. Each
19 * run deploys accounts, seeds history, then deploys a view and reads the
20 * Durable Object requests and SQLite rows of its backfill from Cloudflare's
21 * analytics, printing a `BENCH` line of JSON. Three shapes:
22 *
23 * - width (`DOMAIN_BENCH_INSTANCES`, default 1000): N accounts with 10 deposits
24 * each, then a view keyed by account;
25 * - depth (`DOMAIN_BENCH_DEPTH`, default 1000): one account with N deposits,
26 * then a summary per account;
27 * - bank-wide (`DOMAIN_BENCH_TOTAL`, `<accounts>x<events>`, default 100x100):
28 * accounts with deposits, then one total that every account feeds.
29 *
30 * Sizes are comma-separated; an empty variable skips that shape.
31 */
32
33 const EVENTS = 10;
34 // A Worker request runs at most 6 outgoing calls at once: seed in small batches, many at a time.
35 const SEED_BATCH = 10;
36 // Deposits to one account per request: one object takes them in order.
37 const DEPOSIT_BATCH = 1000;
38 const list = (name: string, fallback: string) =>
39 (process.env[name] ?? fallback).split(",").filter((size) => size !== "");
40 const widths = list("DOMAIN_BENCH_INSTANCES", "1000").map(Number);
41 const depths = list("DOMAIN_BENCH_DEPTH", "1000").map(Number);
42 const totals = list("DOMAIN_BENCH_TOTAL", "100x100").map(
43 (size) => size.split("x").map(Number) as [number, number],
44 );
45
46 class NotYet extends Data.TaggedError("NotYet")<{
47 readonly status: number;
48 readonly body: string;
49 }> {}
50
51 const call = (url: string, method: "GET" | "POST", path: string) =>
52 requestWorker(
53 (method === "GET" ? HttpClientRequest.get : HttpClientRequest.post)(`${url}${path}`).pipe(
54 HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }),
55 ),
56 ).pipe(
57 Effect.retry({
58 while: (error) => error._tag === "HttpClientError",
59 schedule: Schedule.spaced("1 second"),
60 times: 5,
61 }),
62 Effect.flatMap((res) =>
63 res.text.pipe(
64 Effect.flatMap((body) =>
65 res.status === 200
66 ? Effect.succeed(body)
67 : Effect.fail(new NotYet({ status: res.status, body })),
68 ),
69 ),
70 ),
71 );
72
73 const serving = (url: string, version: string) =>
74 call(url, "GET", "/version").pipe(
75 Effect.filterOrFail(
76 (body) => body === version,
77 (body) => new NotYet({ status: 200, body }),
78 ),
79 Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 90 }),
80 );
81
82 interface Usage {
83 readonly requests: number;
84 readonly byType: Record<string, number>;
85 readonly rowsRead: number;
86 readonly rowsWritten: number;
87 /** Billed duration, in GB-seconds. */
88 readonly duration: number;
89 }
90
91 /** The Worker's Durable Object usage between two instants, from Cloudflare's GraphQL analytics. */
92 const usage = (scriptName: string, from: Date, to: Date) =>
93 Effect.gen(function* () {
94 const { accountId } = yield* yield* CloudflareEnvironment;
95 const headers = formatHeaders(yield* yield* Credentials);
96 const client = yield* HttpClient.HttpClient;
97 const query = <T>(query: string) =>
98 client
99 .execute(
100 HttpClientRequest.post("https://api.cloudflare.com/client/v4/graphql").pipe(
101 HttpClientRequest.setHeaders(headers),
102 HttpClientRequest.bodyJsonUnsafe({ query }),
103 ),
104 )
105 .pipe(
106 Effect.flatMap((res) => res.json),
107 Effect.map(
108 (body) => (body as { data: { viewer: { accounts: [T] } } }).data.viewer.accounts[0],
109 ),
110 );
111 const window = `datetime_geq: "${from.toISOString()}", datetime_leq: "${to.toISOString()}"`;
112 const { invocations } = yield* query<{
113 invocations: ReadonlyArray<{
114 sum: { requests: number };
115 dimensions: { namespaceId: string; type: string };
116 }>;
117 }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) {
118 invocations: durableObjectsInvocationsAdaptiveGroups(limit: 1000, filter: { scriptName: "${scriptName}", ${window} }) {
119 sum { requests } dimensions { namespaceId type }
120 } } } }`);
121 const namespaces = [...new Set(invocations.map((row) => row.dimensions.namespaceId))];
122 const { periodic } =
123 namespaces.length === 0
124 ? { periodic: [] }
125 : yield* query<{
126 periodic: ReadonlyArray<{
127 sum: { rowsRead: number; rowsWritten: number; duration: number };
128 }>;
129 }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) {
130 periodic: durableObjectsPeriodicGroups(limit: 1000, filter: { namespaceId_in: ${JSON.stringify(namespaces)}, ${window} }) {
131 sum { rowsRead rowsWritten duration } dimensions { namespaceId }
132 } } } }`);
133 const byType: Record<string, number> = {};
134 for (const row of invocations) {
135 byType[row.dimensions.type] = (byType[row.dimensions.type] ?? 0) + row.sum.requests;
136 }
137 return {
138 requests: invocations.reduce((total, row) => total + row.sum.requests, 0),
139 byType,
140 rowsRead: periodic.reduce((total, row) => total + row.sum.rowsRead, 0),
141 rowsWritten: periodic.reduce((total, row) => total + row.sum.rowsWritten, 0),
142 duration: periodic.reduce((total, row) => total + row.sum.duration, 0),
143 } satisfies Usage;
144 });
145
146 /**
147 * Analytics arrive minutes late, and later the more there are: wait five minutes,
148 * then read until three reads a minute apart agree. Read before the stack is
149 * destroyed: a deleted Worker's rows lose its script name.
150 */
151 const settledUsage = (scriptName: string, from: Date, to: Date) =>
152 Effect.gen(function* () {
153 yield* Effect.sleep("5 minutes");
154 let last = yield* usage(scriptName, from, to);
155 let agreed = 0;
156 for (let attempt = 0; attempt < 15 && agreed < 2; attempt++) {
157 yield* Effect.sleep("1 minute");
158 const next = yield* usage(scriptName, from, to);
159 agreed =
160 next.requests === last.requests && next.rowsWritten === last.rowsWritten ? agreed + 1 : 0;
161 last = next;
162 }
163 return last;
164 });
165
166 const now = Effect.sync(() => new Date());
167
168 const { test } = Test.make({ providers: Cloudflare.providers() });
169
170 /** Retry a seed request, logging why: a slow seed is otherwise silent. */
171 const seedRetry = <A, E extends { readonly _tag: string }, R>(
172 path: string,
173 effect: Effect.Effect<A, E, R>,
174 ) =>
175 effect.pipe(
176 // A seed request can hang for many minutes: fail it and send it again (seeds are idempotent).
177 Effect.timeout("2 minutes"),
178 Effect.tapError((error) =>
179 Effect.sync(() =>
180 console.log(
181 `SEED-RETRY ${path} ${error._tag}${"status" in error ? ` ${String(error.status)}` : ""}${"body" in error ? ` ${String(error.body).slice(0, 120)}` : ""}`,
182 ),
183 ),
184 ),
185 Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 10 }),
186 );
187
188 const seedAccounts = (url: string, accounts: number, events: number) =>
189 Effect.forEach(
190 Array.from({ length: Math.ceil(accounts / SEED_BATCH) }, (_, i) => i * SEED_BATCH),
191 (from) => {
192 const path = `/seed/${from}/${Math.min(SEED_BATCH, accounts - from)}/${events}`;
193 return seedRetry(path, call(url, "POST", path));
194 },
195 { concurrency: 64, discard: true },
196 );
197
198 const seedOne = (url: string, account: string, events: number) =>
199 Effect.forEach(
200 Array.from({ length: Math.ceil(events / DEPOSIT_BATCH) }, (_, i) => i * DEPOSIT_BATCH + 1),
201 (from) => {
202 const path = `/deposit/${account}/${from}/${Math.min(DEPOSIT_BATCH, events - from + 1)}`;
203 return seedRetry(path, call(url, "POST", path));
204 },
205 { concurrency: 4, discard: true },
206 );
207
208 type Next = typeof BenchV2 | typeof BenchV2Summary | typeof BenchV2Total;
209
210 /**
211 * Deploy accounts, seed them, deploy `next` and wait for its backfill; check
212 * the view, then read the backfill's usage from analytics.
213 */
214 const run = (
215 stack: Test.ScratchStack,
216 shape: Record<string, unknown> & { readonly events: number; readonly instances: number },
217 seed: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>,
218 next: Next,
219 check: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>,
220 ) =>
221 Effect.gen(function* () {
222 yield* stack.destroy();
223 const { worker } = yield* stack.deploy(
224 Effect.gen(function* () {
225 return { worker: yield* BenchV1 };
226 }),
227 );
228 const url = worker.url!.replace(/\/+$/, "");
229 const scriptName = worker.workerName;
230 yield* serving(url, "v1");
231
232 // History before the view exists.
233 const seedStart = yield* now;
234 yield* seed(url);
235 const seedEnd = yield* now;
236 // Keep the seed out of the backfill's analytics window.
237 yield* Effect.sleep("2 minutes");
238
239 const deployStart = yield* now;
240 yield* stack.deploy(
241 Effect.gen(function* () {
242 return { worker: yield* next };
243 }),
244 );
245 const deployEnd = yield* now;
246 yield* serving(url, "v2");
247
248 // The deploy returned once the backfill was done.
249 const history = JSON.parse(yield* call(url, "GET", "/history")) as { complete: boolean };
250 expect(history.complete).toBe(true);
251 yield* check(url);
252
253 const timing = {
254 ...shape,
255 deploySeconds: +((deployEnd.getTime() - deployStart.getTime()) / 1000).toFixed(1),
256 seedSeconds: +((seedEnd.getTime() - seedStart.getTime()) / 1000).toFixed(1),
257 };
258 // DOMAIN_BENCH_TIME_ONLY: report the deploy's time without waiting minutes for analytics.
259 if (process.env.DOMAIN_BENCH_TIME_ONLY) {
260 yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(timing)}`));
261 return yield* stack.destroy();
262 }
263 const settleEnd = new Date(deployEnd.getTime() + 2 * 60_000);
264 const seedUsage = yield* settledUsage(scriptName, seedStart, seedEnd);
265 const backfill = yield* settledUsage(scriptName, deployStart, settleEnd);
266 const result = {
267 ...timing,
268 backfill,
269 seed: seedUsage,
270 requestsPerInstance: +(backfill.requests / shape.instances).toFixed(2),
271 rowsWrittenPerEvent: +(backfill.rowsWritten / shape.events).toFixed(2),
272 rowsReadPerEvent: +(backfill.rowsRead / shape.events).toFixed(2),
273 };
274 yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(result)}`));
275 yield* stack.destroy();
276 });
277
278 const viewOf = (url: string, key: string) =>
279 call(url, "GET", `/view/${key}`).pipe(Effect.map((body) => JSON.parse(body)));
280
281 // Each run has its own Worker and stack: run them at once, or a full sweep takes hours.
282 describe.concurrent("live backfill benchmarks", () => {
283 for (const instances of widths) {
284 test.provider.skipIf(!!process.env.FAST)(
285 `live: backfill of ${instances} instances with ${EVENTS} events each`,
286 (stack) =>
287 run(
288 stack,
289 { shape: "width", instances, events: instances * EVENTS },
290 (url) => seedAccounts(url, instances, EVENTS),
291 BenchV2,
292 (url) =>
293 Effect.gen(function* () {
294 const expected = Array.from({ length: EVENTS }, (_, n) => n + 1);
295 for (const i of [0, Math.floor(instances / 2), instances - 1]) {
296 const account = JSON.parse(yield* call(url, "GET", `/account/b${i}`)) as {
297 balance: number;
298 };
299 expect(account.balance).toBe((EVENTS * (EVENTS + 1)) / 2);
300 expect(
301 ((yield* viewOf(url, `b${i}`)) as { amounts: Array<number> }).amounts,
302 ).toEqual(expected);
303 }
304 }),
305 ),
306 { timeout: 7_200_000 },
307 );
308 }
309
310 for (const events of depths) {
311 test.provider.skipIf(!!process.env.FAST)(
312 `live: backfill of one account with ${events} events`,
313 (stack) =>
314 run(
315 stack,
316 { shape: "depth", instances: 1, events },
317 (url) => seedOne(url, "deep", events),
318 BenchV2Summary,
319 (url) =>
320 Effect.gen(function* () {
321 expect(yield* viewOf(url, "deep")).toEqual({
322 count: events,
323 total: (events * (events + 1)) / 2,
324 });
325 }),
326 ),
327 { timeout: 7_200_000 },
328 );
329 }
330
331 for (const [accounts, events] of totals) {
332 test.provider.skipIf(!!process.env.FAST)(
333 `live: backfill of a bank-wide total over ${accounts} accounts with ${events} events each`,
334 (stack) =>
335 run(
336 stack,
337 { shape: "total", instances: accounts, events: accounts * events },
338 (url) => seedAccounts(url, accounts, events),
339 BenchV2Total,
340 (url) =>
341 Effect.gen(function* () {
342 expect(yield* viewOf(url, "bank")).toEqual({
343 count: accounts * events,
344 total: (accounts * events * (events + 1)) / 2,
345 });
346 }),
347 ),
348 { timeout: 7_200_000 },
349 );
350 }
351 });