The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as Cloudflare from "@alchemy.run/cloudflare"; |
| 2 | import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment"; |
| 3 | import { describe, expect } from "@alchemy.run/test"; |
| 4 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 5 | import { Credentials, formatHeaders } from "@distilled.cloud/cloudflare/Credentials"; |
| 6 | import * as Data from "effect/Data"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as HttpClient from "effect/http/HttpClient"; |
| 9 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import { requestWorker } from "../Utils/WorkerRequest.ts"; |
| 12 | import BenchV1 from "./fixtures/bench/v1.ts"; |
| 13 | import BenchV2Summary from "./fixtures/bench/v2-summary.ts"; |
| 14 | import BenchV2Total from "./fixtures/bench/v2-total.ts"; |
| 15 | import BenchV2 from "./fixtures/bench/v2.ts"; |
| 16 | |
| 17 | /** |
| 18 | * What a backfill costs and how long a deploy waits for it on Cloudflare. Each |
| 19 | * run deploys accounts, seeds history, then deploys a view and reads the |
| 20 | * Durable Object requests and SQLite rows of its backfill from Cloudflare's |
| 21 | * analytics, printing a `BENCH` line of JSON. Three shapes: |
| 22 | * |
| 23 | * - width (`DOMAIN_BENCH_INSTANCES`, default 1000): N accounts with 10 deposits |
| 24 | * each, then a view keyed by account; |
| 25 | * - depth (`DOMAIN_BENCH_DEPTH`, default 1000): one account with N deposits, |
| 26 | * then a summary per account; |
| 27 | * - bank-wide (`DOMAIN_BENCH_TOTAL`, `<accounts>x<events>`, default 100x100): |
| 28 | * accounts with deposits, then one total that every account feeds. |
| 29 | * |
| 30 | * Sizes are comma-separated; an empty variable skips that shape. |
| 31 | */ |
| 32 | |
| 33 | const EVENTS = 10; |
| 34 | // A Worker request runs at most 6 outgoing calls at once: seed in small batches, many at a time. |
| 35 | const SEED_BATCH = 10; |
| 36 | // Deposits to one account per request: one object takes them in order. |
| 37 | const DEPOSIT_BATCH = 1000; |
| 38 | const list = (name: string, fallback: string) => |
| 39 | (process.env[name] ?? fallback).split(",").filter((size) => size !== ""); |
| 40 | const widths = list("DOMAIN_BENCH_INSTANCES", "1000").map(Number); |
| 41 | const depths = list("DOMAIN_BENCH_DEPTH", "1000").map(Number); |
| 42 | const totals = list("DOMAIN_BENCH_TOTAL", "100x100").map( |
| 43 | (size) => size.split("x").map(Number) as [number, number], |
| 44 | ); |
| 45 | |
| 46 | class NotYet extends Data.TaggedError("NotYet")<{ |
| 47 | readonly status: number; |
| 48 | readonly body: string; |
| 49 | }> {} |
| 50 | |
| 51 | const call = (url: string, method: "GET" | "POST", path: string) => |
| 52 | requestWorker( |
| 53 | (method === "GET" ? HttpClientRequest.get : HttpClientRequest.post)(`${url}${path}`).pipe( |
| 54 | HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }), |
| 55 | ), |
| 56 | ).pipe( |
| 57 | Effect.retry({ |
| 58 | while: (error) => error._tag === "HttpClientError", |
| 59 | schedule: Schedule.spaced("1 second"), |
| 60 | times: 5, |
| 61 | }), |
| 62 | Effect.flatMap((res) => |
| 63 | res.text.pipe( |
| 64 | Effect.flatMap((body) => |
| 65 | res.status === 200 |
| 66 | ? Effect.succeed(body) |
| 67 | : Effect.fail(new NotYet({ status: res.status, body })), |
| 68 | ), |
| 69 | ), |
| 70 | ), |
| 71 | ); |
| 72 | |
| 73 | const serving = (url: string, version: string) => |
| 74 | call(url, "GET", "/version").pipe( |
| 75 | Effect.filterOrFail( |
| 76 | (body) => body === version, |
| 77 | (body) => new NotYet({ status: 200, body }), |
| 78 | ), |
| 79 | Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 90 }), |
| 80 | ); |
| 81 | |
| 82 | interface Usage { |
| 83 | readonly requests: number; |
| 84 | readonly byType: Record<string, number>; |
| 85 | readonly rowsRead: number; |
| 86 | readonly rowsWritten: number; |
| 87 | /** Billed duration, in GB-seconds. */ |
| 88 | readonly duration: number; |
| 89 | } |
| 90 | |
| 91 | /** The Worker's Durable Object usage between two instants, from Cloudflare's GraphQL analytics. */ |
| 92 | const usage = (scriptName: string, from: Date, to: Date) => |
| 93 | Effect.gen(function* () { |
| 94 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 95 | const headers = formatHeaders(yield* yield* Credentials); |
| 96 | const client = yield* HttpClient.HttpClient; |
| 97 | const query = <T>(query: string) => |
| 98 | client |
| 99 | .execute( |
| 100 | HttpClientRequest.post("https://api.cloudflare.com/client/v4/graphql").pipe( |
| 101 | HttpClientRequest.setHeaders(headers), |
| 102 | HttpClientRequest.bodyJsonUnsafe({ query }), |
| 103 | ), |
| 104 | ) |
| 105 | .pipe( |
| 106 | Effect.flatMap((res) => res.json), |
| 107 | Effect.map( |
| 108 | (body) => (body as { data: { viewer: { accounts: [T] } } }).data.viewer.accounts[0], |
| 109 | ), |
| 110 | ); |
| 111 | const window = `datetime_geq: "${from.toISOString()}", datetime_leq: "${to.toISOString()}"`; |
| 112 | const { invocations } = yield* query<{ |
| 113 | invocations: ReadonlyArray<{ |
| 114 | sum: { requests: number }; |
| 115 | dimensions: { namespaceId: string; type: string }; |
| 116 | }>; |
| 117 | }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) { |
| 118 | invocations: durableObjectsInvocationsAdaptiveGroups(limit: 1000, filter: { scriptName: "${scriptName}", ${window} }) { |
| 119 | sum { requests } dimensions { namespaceId type } |
| 120 | } } } }`); |
| 121 | const namespaces = [...new Set(invocations.map((row) => row.dimensions.namespaceId))]; |
| 122 | const { periodic } = |
| 123 | namespaces.length === 0 |
| 124 | ? { periodic: [] } |
| 125 | : yield* query<{ |
| 126 | periodic: ReadonlyArray<{ |
| 127 | sum: { rowsRead: number; rowsWritten: number; duration: number }; |
| 128 | }>; |
| 129 | }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) { |
| 130 | periodic: durableObjectsPeriodicGroups(limit: 1000, filter: { namespaceId_in: ${JSON.stringify(namespaces)}, ${window} }) { |
| 131 | sum { rowsRead rowsWritten duration } dimensions { namespaceId } |
| 132 | } } } }`); |
| 133 | const byType: Record<string, number> = {}; |
| 134 | for (const row of invocations) { |
| 135 | byType[row.dimensions.type] = (byType[row.dimensions.type] ?? 0) + row.sum.requests; |
| 136 | } |
| 137 | return { |
| 138 | requests: invocations.reduce((total, row) => total + row.sum.requests, 0), |
| 139 | byType, |
| 140 | rowsRead: periodic.reduce((total, row) => total + row.sum.rowsRead, 0), |
| 141 | rowsWritten: periodic.reduce((total, row) => total + row.sum.rowsWritten, 0), |
| 142 | duration: periodic.reduce((total, row) => total + row.sum.duration, 0), |
| 143 | } satisfies Usage; |
| 144 | }); |
| 145 | |
| 146 | /** |
| 147 | * Analytics arrive minutes late, and later the more there are: wait five minutes, |
| 148 | * then read until three reads a minute apart agree. Read before the stack is |
| 149 | * destroyed: a deleted Worker's rows lose its script name. |
| 150 | */ |
| 151 | const settledUsage = (scriptName: string, from: Date, to: Date) => |
| 152 | Effect.gen(function* () { |
| 153 | yield* Effect.sleep("5 minutes"); |
| 154 | let last = yield* usage(scriptName, from, to); |
| 155 | let agreed = 0; |
| 156 | for (let attempt = 0; attempt < 15 && agreed < 2; attempt++) { |
| 157 | yield* Effect.sleep("1 minute"); |
| 158 | const next = yield* usage(scriptName, from, to); |
| 159 | agreed = |
| 160 | next.requests === last.requests && next.rowsWritten === last.rowsWritten ? agreed + 1 : 0; |
| 161 | last = next; |
| 162 | } |
| 163 | return last; |
| 164 | }); |
| 165 | |
| 166 | const now = Effect.sync(() => new Date()); |
| 167 | |
| 168 | const { test } = Test.make({ providers: Cloudflare.providers() }); |
| 169 | |
| 170 | /** Retry a seed request, logging why: a slow seed is otherwise silent. */ |
| 171 | const seedRetry = <A, E extends { readonly _tag: string }, R>( |
| 172 | path: string, |
| 173 | effect: Effect.Effect<A, E, R>, |
| 174 | ) => |
| 175 | effect.pipe( |
| 176 | // A seed request can hang for many minutes: fail it and send it again (seeds are idempotent). |
| 177 | Effect.timeout("2 minutes"), |
| 178 | Effect.tapError((error) => |
| 179 | Effect.sync(() => |
| 180 | console.log( |
| 181 | `SEED-RETRY ${path} ${error._tag}${"status" in error ? ` ${String(error.status)}` : ""}${"body" in error ? ` ${String(error.body).slice(0, 120)}` : ""}`, |
| 182 | ), |
| 183 | ), |
| 184 | ), |
| 185 | Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 10 }), |
| 186 | ); |
| 187 | |
| 188 | const seedAccounts = (url: string, accounts: number, events: number) => |
| 189 | Effect.forEach( |
| 190 | Array.from({ length: Math.ceil(accounts / SEED_BATCH) }, (_, i) => i * SEED_BATCH), |
| 191 | (from) => { |
| 192 | const path = `/seed/${from}/${Math.min(SEED_BATCH, accounts - from)}/${events}`; |
| 193 | return seedRetry(path, call(url, "POST", path)); |
| 194 | }, |
| 195 | { concurrency: 64, discard: true }, |
| 196 | ); |
| 197 | |
| 198 | const seedOne = (url: string, account: string, events: number) => |
| 199 | Effect.forEach( |
| 200 | Array.from({ length: Math.ceil(events / DEPOSIT_BATCH) }, (_, i) => i * DEPOSIT_BATCH + 1), |
| 201 | (from) => { |
| 202 | const path = `/deposit/${account}/${from}/${Math.min(DEPOSIT_BATCH, events - from + 1)}`; |
| 203 | return seedRetry(path, call(url, "POST", path)); |
| 204 | }, |
| 205 | { concurrency: 4, discard: true }, |
| 206 | ); |
| 207 | |
| 208 | type Next = typeof BenchV2 | typeof BenchV2Summary | typeof BenchV2Total; |
| 209 | |
| 210 | /** |
| 211 | * Deploy accounts, seed them, deploy `next` and wait for its backfill; check |
| 212 | * the view, then read the backfill's usage from analytics. |
| 213 | */ |
| 214 | const run = ( |
| 215 | stack: Test.ScratchStack, |
| 216 | shape: Record<string, unknown> & { readonly events: number; readonly instances: number }, |
| 217 | seed: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>, |
| 218 | next: Next, |
| 219 | check: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>, |
| 220 | ) => |
| 221 | Effect.gen(function* () { |
| 222 | yield* stack.destroy(); |
| 223 | const { worker } = yield* stack.deploy( |
| 224 | Effect.gen(function* () { |
| 225 | return { worker: yield* BenchV1 }; |
| 226 | }), |
| 227 | ); |
| 228 | const url = worker.url!.replace(/\/+$/, ""); |
| 229 | const scriptName = worker.workerName; |
| 230 | yield* serving(url, "v1"); |
| 231 | |
| 232 | // History before the view exists. |
| 233 | const seedStart = yield* now; |
| 234 | yield* seed(url); |
| 235 | const seedEnd = yield* now; |
| 236 | // Keep the seed out of the backfill's analytics window. |
| 237 | yield* Effect.sleep("2 minutes"); |
| 238 | |
| 239 | const deployStart = yield* now; |
| 240 | yield* stack.deploy( |
| 241 | Effect.gen(function* () { |
| 242 | return { worker: yield* next }; |
| 243 | }), |
| 244 | ); |
| 245 | const deployEnd = yield* now; |
| 246 | yield* serving(url, "v2"); |
| 247 | |
| 248 | // The deploy returned once the backfill was done. |
| 249 | const history = JSON.parse(yield* call(url, "GET", "/history")) as { complete: boolean }; |
| 250 | expect(history.complete).toBe(true); |
| 251 | yield* check(url); |
| 252 | |
| 253 | const timing = { |
| 254 | ...shape, |
| 255 | deploySeconds: +((deployEnd.getTime() - deployStart.getTime()) / 1000).toFixed(1), |
| 256 | seedSeconds: +((seedEnd.getTime() - seedStart.getTime()) / 1000).toFixed(1), |
| 257 | }; |
| 258 | // DOMAIN_BENCH_TIME_ONLY: report the deploy's time without waiting minutes for analytics. |
| 259 | if (process.env.DOMAIN_BENCH_TIME_ONLY) { |
| 260 | yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(timing)}`)); |
| 261 | return yield* stack.destroy(); |
| 262 | } |
| 263 | const settleEnd = new Date(deployEnd.getTime() + 2 * 60_000); |
| 264 | const seedUsage = yield* settledUsage(scriptName, seedStart, seedEnd); |
| 265 | const backfill = yield* settledUsage(scriptName, deployStart, settleEnd); |
| 266 | const result = { |
| 267 | ...timing, |
| 268 | backfill, |
| 269 | seed: seedUsage, |
| 270 | requestsPerInstance: +(backfill.requests / shape.instances).toFixed(2), |
| 271 | rowsWrittenPerEvent: +(backfill.rowsWritten / shape.events).toFixed(2), |
| 272 | rowsReadPerEvent: +(backfill.rowsRead / shape.events).toFixed(2), |
| 273 | }; |
| 274 | yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(result)}`)); |
| 275 | yield* stack.destroy(); |
| 276 | }); |
| 277 | |
| 278 | const viewOf = (url: string, key: string) => |
| 279 | call(url, "GET", `/view/${key}`).pipe(Effect.map((body) => JSON.parse(body))); |
| 280 | |
| 281 | // Each run has its own Worker and stack: run them at once, or a full sweep takes hours. |
| 282 | describe.concurrent("live backfill benchmarks", () => { |
| 283 | for (const instances of widths) { |
| 284 | test.provider.skipIf(!!process.env.FAST)( |
| 285 | `live: backfill of ${instances} instances with ${EVENTS} events each`, |
| 286 | (stack) => |
| 287 | run( |
| 288 | stack, |
| 289 | { shape: "width", instances, events: instances * EVENTS }, |
| 290 | (url) => seedAccounts(url, instances, EVENTS), |
| 291 | BenchV2, |
| 292 | (url) => |
| 293 | Effect.gen(function* () { |
| 294 | const expected = Array.from({ length: EVENTS }, (_, n) => n + 1); |
| 295 | for (const i of [0, Math.floor(instances / 2), instances - 1]) { |
| 296 | const account = JSON.parse(yield* call(url, "GET", `/account/b${i}`)) as { |
| 297 | balance: number; |
| 298 | }; |
| 299 | expect(account.balance).toBe((EVENTS * (EVENTS + 1)) / 2); |
| 300 | expect( |
| 301 | ((yield* viewOf(url, `b${i}`)) as { amounts: Array<number> }).amounts, |
| 302 | ).toEqual(expected); |
| 303 | } |
| 304 | }), |
| 305 | ), |
| 306 | { timeout: 7_200_000 }, |
| 307 | ); |
| 308 | } |
| 309 | |
| 310 | for (const events of depths) { |
| 311 | test.provider.skipIf(!!process.env.FAST)( |
| 312 | `live: backfill of one account with ${events} events`, |
| 313 | (stack) => |
| 314 | run( |
| 315 | stack, |
| 316 | { shape: "depth", instances: 1, events }, |
| 317 | (url) => seedOne(url, "deep", events), |
| 318 | BenchV2Summary, |
| 319 | (url) => |
| 320 | Effect.gen(function* () { |
| 321 | expect(yield* viewOf(url, "deep")).toEqual({ |
| 322 | count: events, |
| 323 | total: (events * (events + 1)) / 2, |
| 324 | }); |
| 325 | }), |
| 326 | ), |
| 327 | { timeout: 7_200_000 }, |
| 328 | ); |
| 329 | } |
| 330 | |
| 331 | for (const [accounts, events] of totals) { |
| 332 | test.provider.skipIf(!!process.env.FAST)( |
| 333 | `live: backfill of a bank-wide total over ${accounts} accounts with ${events} events each`, |
| 334 | (stack) => |
| 335 | run( |
| 336 | stack, |
| 337 | { shape: "total", instances: accounts, events: accounts * events }, |
| 338 | (url) => seedAccounts(url, accounts, events), |
| 339 | BenchV2Total, |
| 340 | (url) => |
| 341 | Effect.gen(function* () { |
| 342 | expect(yield* viewOf(url, "bank")).toEqual({ |
| 343 | count: accounts * events, |
| 344 | total: (accounts * events * (events + 1)) / 2, |
| 345 | }); |
| 346 | }), |
| 347 | ), |
| 348 | { timeout: 7_200_000 }, |
| 349 | ); |
| 350 | } |
| 351 | }); |