@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 612 lines 22.2 KB
Raw
1 import * as Retry from "@distilled.cloud/cloudflare/Retry";
2 import * as workers from "@distilled.cloud/cloudflare/workers";
3 import * as wfp from "@distilled.cloud/cloudflare/workers-for-platforms";
4 import type { ScopedPlanStatusSession } from "alchemy/Report";
5 import { sha256, sha256Object } from "alchemy/Util";
6 import { listFileSystemDirectory, parseIgnoreRules, walkIgnoring } from "alchemy/Util/Ignore";
7 import { initialCwd } from "alchemy/Util/Node";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as FileSystem from "effect/FileSystem";
11 import * as Path from "effect/Path";
12 import type { PlatformError } from "effect/PlatformError";
13 import * as Predicate from "effect/Predicate";
14 import * as Schedule from "effect/Schedule";
15 import * as Semaphore from "effect/Semaphore";
16
17 const MAX_ASSET_SIZE = 1024 * 1024 * 25; // 25MB
18 const MAX_ASSET_COUNT = 20_000;
19
20 export interface Assets {
21 kind: "Cloudflare.Workers.Assets";
22 }
23
24 export const isAssets = (value: any): value is Assets =>
25 value?.kind === "Cloudflare.Workers.Assets";
26
27 /**
28 * Routing configuration for a Worker's static assets — sent to Cloudflare
29 * as `metadata.assets.config` on script upload. Declared explicitly (not
30 * derived from the distilled API schema) so alchemy owns and documents its
31 * public surface.
32 */
33 export interface AssetsConfig {
34 /**
35 * Determines the redirects and rewrites of requests for HTML content:
36 * whether `/page` serves `page.html`, and whether trailing slashes are
37 * added or dropped.
38 *
39 * @default "auto-trailing-slash"
40 */
41 htmlHandling?: "auto-trailing-slash" | "force-trailing-slash" | "drop-trailing-slash" | "none";
42 /**
43 * Determines the response when a request does not match a static asset:
44 * `"404-page"` serves the nearest `404.html`, and
45 * `"single-page-application"` serves `index.html` for client-side
46 * routing. When the Worker has a script, an unmatched request falls
47 * through to the Worker instead.
48 *
49 * @default "none"
50 */
51 notFoundHandling?: "none" | "404-page" | "single-page-application";
52 /**
53 * Routes requests through the Worker *before* static-asset matching.
54 *
55 * Assets-first by default: a request matching a file is served directly
56 * and never invokes the Worker. `true` routes every request through the
57 * Worker ahead of the asset layer — serve files yourself via the
58 * `ASSETS` binding. A path-rule array routes only matching paths
59 * worker-first (e.g. `["/api/*"]`): glob (`*`) and negative (`!`) rules
60 * are supported, rules must start with `/` or `!/`, and negative rules
61 * take precedence. The same routing applies under `alchemy dev`.
62 *
63 * @default false
64 */
65 runWorkerFirst?: boolean | string[];
66 /**
67 * Legacy routing flag predating `runWorkerFirst`.
68 */
69 serveDirectly?: boolean;
70 /**
71 * Raw contents of a `_headers` file — header rules applied by the asset
72 * layer. Overrides a `_headers` file read from the assets directory.
73 */
74 headers?: string;
75 /**
76 * Raw contents of a `_redirects` file — redirect rules applied by the
77 * asset layer. Overrides a `_redirects` file read from the assets
78 * directory.
79 */
80 redirects?: string;
81 }
82
83 export interface AssetReadResult {
84 directory: string;
85 /**
86 * The normalized `base` this manifest was keyed with (`""` when the
87 * assets are served from the origin root). Manifest keys are request
88 * paths, so `uploadAssets` strips this back off to find each file on
89 * disk.
90 */
91 pathPrefix: string;
92 config: AssetsConfig | undefined;
93 manifest: Record<string, { hash: string; size: number }>;
94 _headers: string | undefined;
95 _redirects: string | undefined;
96 hash: string;
97 }
98
99 export interface AssetsProps extends AssetsConfig {
100 directory: string;
101 /**
102 * The path this site is served from, when it is not the origin root —
103 * e.g. `"/docs"` for a Worker on the route `example.com/docs*`. Matches
104 * Vite's `base`, and `Website.Vite` fills it in from the resolved Vite
105 * config automatically; set it by hand when you bring your own build.
106 *
107 * Cloudflare's asset router matches request paths against the manifest
108 * literally and never strips a prefix, so its model is that the assets
109 * directory mirrors the served path. This does that at manifest time:
110 * `dist/app.js` is uploaded as `/docs/app.js`, leaving the build output
111 * on disk untouched.
112 *
113 * Bases that name no path — `"/"`, `"./"`, `"https://cdn.example.com/"` —
114 * are ignored, as an absolute base means the assets are served by a CDN
115 * rather than by this Worker.
116 *
117 * `_headers` and `_redirects` are NOT rewritten: their rules match the
118 * incoming request path, so author them with the full served path
119 * (`/docs/old /docs/new 301`).
120 *
121 * @default undefined (assets are served from the origin root)
122 * @see https://developers.cloudflare.com/workers/static-assets/routing/advanced/serving-a-subdirectory/
123 */
124 base?: string;
125 }
126
127 /**
128 * `base` → manifest path prefix. Only a root-relative base names a path
129 * this Worker serves; `"/"`, `"./"`, protocol-relative and absolute URLs
130 * all mean "no prefix".
131 */
132 export const getAssetsPathPrefix = (base: string | undefined) =>
133 base?.startsWith("/") && !base.startsWith("//") ? base.replace(/\/+$/, "") : "";
134
135 export type ValidationError =
136 | AssetTooLargeError
137 | TooManyAssetsError
138 | AssetNotFoundError
139 | FailedToReadAssetError;
140
141 export class AssetTooLargeError extends Data.TaggedError("AssetTooLargeError")<{
142 message: string;
143 assetPath: string;
144 size: number;
145 }> {}
146
147 export class TooManyAssetsError extends Data.TaggedError("TooManyAssetsError")<{
148 message: string;
149 directory: string;
150 count: number;
151 }> {}
152
153 export class AssetNotFoundError extends Data.TaggedError("AssetNotFoundError")<{
154 message: string;
155 hash: string;
156 }> {}
157
158 export class FailedToReadAssetError extends Data.TaggedError("FailedToReadAssetError")<{
159 message: string;
160 assetPath: string;
161 cause: PlatformError;
162 }> {}
163
164 export class AssetUploadSessionError extends Data.TaggedError("AssetUploadSessionError")<{
165 message: string;
166 workerName: string;
167 }> {}
168
169 const contentTypesByExtension: Record<string, string> = {
170 html: "text/html",
171 htm: "text/html",
172 txt: "text/plain",
173 md: "text/markdown",
174 sql: "text/sql",
175 json: "application/json",
176 // Source maps are JSON; serving them as such lets devtools consume them.
177 map: "application/json",
178 jsonld: "application/ld+json",
179 xml: "application/xml",
180 csv: "text/csv",
181 // Browsers only accept JavaScript module scripts when the MIME type is a
182 // "JavaScript MIME type" (e.g. text/javascript). application/javascript+module
183 // is not valid and causes strict module loading to fail.
184 js: "text/javascript; charset=utf-8",
185 mjs: "text/javascript; charset=utf-8",
186 css: "text/css",
187 wasm: "application/wasm",
188 pdf: "application/pdf",
189 // images
190 png: "image/png",
191 jpg: "image/jpeg",
192 jpeg: "image/jpeg",
193 gif: "image/gif",
194 webp: "image/webp",
195 avif: "image/avif",
196 svg: "image/svg+xml",
197 ico: "image/x-icon",
198 bmp: "image/bmp",
199 // fonts
200 woff: "font/woff",
201 woff2: "font/woff2",
202 ttf: "font/ttf",
203 otf: "font/otf",
204 eot: "application/vnd.ms-fontobject",
205 // media
206 mp3: "audio/mpeg",
207 wav: "audio/wav",
208 ogg: "audio/ogg",
209 mp4: "video/mp4",
210 webm: "video/webm",
211 // app manifests
212 webmanifest: "application/manifest+json",
213 };
214
215 const getContentType = (name: string) => {
216 const dot = name.lastIndexOf(".");
217 const ext = dot === -1 ? "" : name.slice(dot + 1).toLowerCase();
218 return contentTypesByExtension[ext] ?? "application/octet-stream";
219 };
220
221 const maybeReadString = Effect.fn(function* (file: string) {
222 const fs = yield* FileSystem.FileSystem;
223 return yield* fs.readFileString(file).pipe(
224 Effect.catchIf(
225 (error) => error._tag === "PlatformError" && error.reason._tag === "NotFound",
226 () => Effect.undefined,
227 ),
228 );
229 });
230
231 /**
232 * Read the special `_headers` / `_redirects` files from an assets
233 * directory. They are excluded from the upload manifest, but their raw
234 * contents must be sent to Cloudflare in the script metadata's asset
235 * config (`config._headers` / `config._redirects`) for the rules to
236 * apply.
237 *
238 * The directory is optional: a config-only assets object (e.g. Vite's
239 * `assets: { runWorkerFirst: true }`, whose directory is supplied by the
240 * build) has no directory to read, and in `dev` there is no build output at
241 * all. That yields no rules rather than an error.
242 */
243 export const readAssetsConfigFiles = Effect.fn(function* (directory: string | undefined) {
244 if (directory === undefined) {
245 return { _headers: undefined, _redirects: undefined };
246 }
247 const path = yield* Path.Path;
248 // Anchored: see `readAssets` — the directory may be initial-cwd-relative
249 // and live cwd reads race concurrent tools' transient chdir.
250 const resolvedDirectory = path.resolve(initialCwd, directory);
251 const [_headers, _redirects] = yield* Effect.all([
252 maybeReadString(path.join(resolvedDirectory, "_headers")),
253 maybeReadString(path.join(resolvedDirectory, "_redirects")),
254 ]);
255 return { _headers, _redirects };
256 });
257
258 /**
259 * Merge `_headers` / `_redirects` file contents into an asset config,
260 * producing the config to send in the script-upload metadata. Explicit
261 * `headers` / `redirects` props win over the files.
262 */
263 export const mergeAssetsConfigFiles = (
264 config: AssetsConfig | undefined,
265 files: { _headers: string | undefined; _redirects: string | undefined },
266 ): AssetsConfig | undefined => {
267 const headers = config?.headers ?? files._headers;
268 const redirects = config?.redirects ?? files._redirects;
269 if (headers === undefined && redirects === undefined) {
270 return config;
271 }
272 return {
273 ...config,
274 ...(headers !== undefined ? { headers } : undefined),
275 ...(redirects !== undefined ? { redirects } : undefined),
276 };
277 };
278
279 export const readAssets = Effect.fn(function* ({ directory, base, ...config }: AssetsProps) {
280 const fs = yield* FileSystem.FileSystem;
281 const path = yield* Path.Path;
282 // `base` nests the *manifest* paths (what Cloudflare matches request
283 // pathnames against) under a prefix; files stay where they are on disk.
284 // It is deliberately excluded from the `config` sent to Cloudflare — it
285 // is not part of the API's asset config shape.
286 const pathPrefix = getAssetsPathPrefix(base);
287 // Anchored: `directory` may be a relative path persisted by
288 // `Command.Build` (relative to the initial cwd), and a live
289 // `process.cwd()` read can race a concurrent tool's transient chdir.
290 const resolvedDirectory = path.resolve(initialCwd, directory);
291 const [ignore, _headers, _redirects] = yield* Effect.all([
292 maybeReadString(path.join(resolvedDirectory, ".assetsignore")),
293 maybeReadString(path.join(resolvedDirectory, "_headers")),
294 maybeReadString(path.join(resolvedDirectory, "_redirects")),
295 ]);
296 // `.assetsignore` uses gitignore semantics (as in wrangler). Excluded
297 // directories are skipped without being read.
298 const files = yield* walkIgnoring({
299 list: yield* listFileSystemDirectory(resolvedDirectory),
300 rules: parseIgnoreRules(
301 [
302 ".assetsignore",
303 "_headers",
304 "_redirects",
305 ...(ignore
306 ?.split("\n")
307 .map((line) => line.trim())
308 .filter((line) => line.length > 0 && !line.startsWith("#")) ?? []),
309 ],
310 "gitignore",
311 ),
312 }).pipe(
313 // Symlinked files are uploaded by their target's content (stat follows
314 // links below); directories only contribute their files.
315 Effect.map((entries) =>
316 entries.flatMap((entry) => (entry.type === "Directory" ? [] : [entry.path])),
317 ),
318 );
319 const manifest = new Map<string, { hash: string; size: number }>();
320 let count = 0;
321 yield* Effect.forEach(
322 files,
323 Effect.fn(function* (name) {
324 const file = path.join(resolvedDirectory, name);
325 const stat = yield* fs.stat(file);
326 if (stat.type !== "File") {
327 return;
328 }
329 const size = Number(stat.size);
330 if (size > MAX_ASSET_SIZE) {
331 return yield* new AssetTooLargeError({
332 message: `Asset ${name} is too large (the maximum size is ${MAX_ASSET_SIZE / 1024 / 1024} MB; this asset is ${size / 1024 / 1024} MB)`,
333 assetPath: name,
334 size,
335 });
336 }
337 // Hash content + extension (matching wrangler): the upload API stores
338 // one blob + content type per hash, so two identical bodies under
339 // different extensions must not collapse into a single entry — the
340 // second file would serve with the first file's content type.
341 const extension = name.slice(name.lastIndexOf(".") + 1).toLowerCase();
342 const hash = yield* fs.readFile(file).pipe(
343 Effect.flatMap((content) =>
344 Effect.sync(() => {
345 const extBytes = new TextEncoder().encode(extension);
346 const hashed = new Uint8Array(content.length + extBytes.length);
347 hashed.set(content);
348 hashed.set(extBytes, content.length);
349 return hashed;
350 }),
351 ),
352 Effect.flatMap(sha256),
353 Effect.map((hash) => hash.slice(0, 32)),
354 );
355 count++;
356 if (count > MAX_ASSET_COUNT) {
357 return yield* new TooManyAssetsError({
358 message: `Too many assets (the maximum count is ${MAX_ASSET_COUNT}; this directory has ${count} assets)`,
359 directory,
360 count,
361 });
362 }
363 manifest.set(
364 `${pathPrefix}${(name.startsWith("/") ? name : `/${name}`).replaceAll("\\", "/")}`,
365 {
366 hash,
367 size,
368 },
369 );
370 }),
371 );
372 // Cloudflare's SPA fallback is hard-coded to `/index.html` at the
373 // manifest root, so prefixing every key would 404 every client-side
374 // route under the base. Alias the shell back to the root — one extra
375 // manifest line, zero extra uploads, since entries are content-addressed.
376 const indexHtml = manifest.get(`${pathPrefix}/index.html`);
377 if (
378 indexHtml &&
379 config.notFoundHandling === "single-page-application" &&
380 !manifest.has("/index.html")
381 ) {
382 manifest.set("/index.html", indexHtml);
383 }
384 const sortedManifest = Object.fromEntries(
385 Array.from(manifest.entries()).sort((a, b) => a[0].localeCompare(b[0])),
386 );
387 // Hash only inputs that affect what gets uploaded — the file
388 // manifest, asset config, and the special `_headers` / `_redirects`
389 // files. `directory` is deliberately excluded: identical bytes at a
390 // different absolute path must produce the same hash, otherwise
391 // diffing across machines (CI runner → local laptop, monorepo
392 // root → workspace root, etc.) spuriously reports "changed" and
393 // causes both unnecessary re-uploads and `NotFound` failures when
394 // the previously-recorded path is gone.
395 const hash = yield* sha256Object({
396 config,
397 manifest: sortedManifest,
398 _headers,
399 _redirects,
400 });
401 return {
402 directory,
403 pathPrefix,
404 // Fold the `_headers` / `_redirects` file contents into the config
405 // that gets sent to Cloudflare (`metadata.assets.config`). Merged
406 // *after* hashing so the hash input shape stays stable for
407 // already-deployed workers. Deliberately NOT `base`-prefixed: their
408 // rules match the incoming request path, which already carries the
409 // base, so they are authored with the full served path.
410 config: mergeAssetsConfigFiles(config, { _headers, _redirects }),
411 manifest: sortedManifest,
412 _headers,
413 _redirects,
414 hash,
415 };
416 });
417
418 /**
419 * Bulk buckets (up to ~50 MiB of base64 each) go up this many at a time —
420 * the same number wrangler uses.
421 */
422 const BULK_UPLOAD_CONCURRENCY = 3;
423 /**
424 * Per-bucket retries after a gateway error (502/503/504), with concurrency
425 * already dropped to one. Mirrors wrangler's `MAX_UPLOAD_GATEWAY_ERRORS`.
426 */
427 const MAX_UPLOAD_GATEWAY_RETRIES = 5;
428
429 const isGatewayError = (error: unknown) =>
430 Predicate.isTagged(error, "BadGateway") ||
431 Predicate.isTagged(error, "ServiceUnavailable") ||
432 Predicate.isTagged(error, "GatewayTimeout");
433
434 /**
435 * The SDK's default policy retries gateway errors internally (~20s of
436 * backoff) before the caller ever sees one, so the concurrency fallback
437 * below could never kick in. Let those surface to the bucket loop; every
438 * other transient error keeps the default handling.
439 */
440 const surfaceGatewayErrors: Retry.Factory = (lastError) => {
441 const base = Retry.makeDefault(lastError);
442 return {
443 ...base,
444 while: (error) => !isGatewayError(error) && (base.while?.(error) ?? false),
445 };
446 };
447
448 export const uploadAssets = Effect.fn(function* (
449 accountId: string,
450 workerName: string,
451 assets: AssetReadResult,
452 { note }: ScopedPlanStatusSession,
453 dispatchNamespace?: string,
454 ) {
455 const fs = yield* FileSystem.FileSystem;
456 const path = yield* Path.Path;
457 const createAssetUpload = yield* workers.createAssetUpload;
458
459 // Manifest keys are the paths Cloudflare *serves*, so they carry the
460 // `base` prefix. The files themselves are on disk at the un-prefixed
461 // path relative to the assets directory, so drop the prefix to get
462 // back to something `readFile` can open.
463 const toDiskPath = (name: string) =>
464 assets.pathPrefix && name.startsWith(`${assets.pathPrefix}/`)
465 ? name.slice(assets.pathPrefix.length)
466 : name;
467
468 const assetsByHash = new Map<string, string>();
469 for (const [name, { hash }] of Object.entries(assets.manifest)) {
470 assetsByHash.set(hash, toDiskPath(name));
471 }
472 // Anchored: `assets.directory` may be relative to the initial cwd (a
473 // `Command.Build` outdir), and a live `process.cwd()` read can race a
474 // concurrent tool's transient chdir (framework source builds).
475 const directory = path.resolve(initialCwd, assets.directory);
476
477 // Bucket concurrency, wrangler-style: three in flight, and the first
478 // gateway error resizes it to one for the rest of the deploy. Hoisted out
479 // of the session so a JWT-expiry retry stays degraded instead of bursting
480 // back to three against a gateway that just told us to back off.
481 const semaphore = yield* Semaphore.make(BULK_UPLOAD_CONCURRENCY);
482
483 const uploadBucket = Effect.fn(function* (bucket: readonly string[], uploadJwt: string) {
484 const body: Record<string, File> = {};
485 yield* Effect.forEach(
486 bucket,
487 Effect.fn(function* (hash) {
488 const name = assetsByHash.get(hash);
489 if (!name) {
490 return yield* new AssetNotFoundError({
491 message: `Asset ${hash} not found in manifest`,
492 hash,
493 });
494 }
495 const file = yield* fs.readFile(path.join(directory, name)).pipe(
496 Effect.mapError(
497 (error) =>
498 new FailedToReadAssetError({
499 message: `Failed to read asset ${name}: ${error.message}`,
500 assetPath: name,
501 cause: error,
502 }),
503 ),
504 );
505 body[hash] = new File([Buffer.from(file).toString("base64")], hash, {
506 type: getContentType(name),
507 });
508 }),
509 );
510 return yield* createAssetUpload({
511 accountId,
512 base64: true,
513 body,
514 jwtToken: uploadJwt,
515 }).pipe(
516 Retry.policy(surfaceGatewayErrors),
517 Effect.tapError((error) =>
518 isGatewayError(error)
519 ? semaphore.resize(1).pipe(
520 Effect.andThen(
521 note("Asset upload hit a gateway error, retrying one bucket at a time...", {
522 kind: "status",
523 }),
524 ),
525 )
526 : Effect.void,
527 ),
528 Effect.retry({
529 while: isGatewayError,
530 schedule: Schedule.exponential("2 seconds"),
531 times: MAX_UPLOAD_GATEWAY_RETRIES,
532 }),
533 );
534 });
535
536 // One full upload session: ask Cloudflare which assets are missing,
537 // upload each bucket, and return the completion JWT that putWorker
538 // must redeem. The session JWT can expire mid-upload on very large
539 // asset sets (Unauthorized), and the final bucket response has been
540 // observed in the wild to omit the completion JWT — both cases are
541 // retried below with a fresh session. Already-uploaded assets are
542 // not re-bucketed, so a retry resumes where the last session
543 // stopped, and a fresh session with nothing left to upload returns
544 // the completion JWT directly.
545 const runSession = Effect.fn(function* () {
546 yield* note("Checking assets...", { kind: "status" });
547 const session = dispatchNamespace
548 ? yield* wfp.createDispatchNamespaceScriptAssetUpload({
549 accountId,
550 dispatchNamespace,
551 scriptName: workerName,
552 manifest: assets.manifest,
553 })
554 : yield* workers.createScriptAssetUpload({
555 accountId,
556 scriptName: workerName,
557 manifest: assets.manifest,
558 });
559 if (!session.buckets?.length) {
560 if (!session.jwt) {
561 return yield* new AssetUploadSessionError({
562 message: `Asset upload session for worker ${workerName} returned no completion token`,
563 workerName,
564 });
565 }
566 return session.jwt;
567 }
568 if (!session.jwt) {
569 return yield* new AssetUploadSessionError({
570 message: `Asset upload session for worker ${workerName} returned ${session.buckets.length} buckets to upload but no upload token`,
571 workerName,
572 });
573 }
574 const uploadJwt = session.jwt;
575 let uploaded = 0;
576 const total = session.buckets.flat().length;
577 yield* note(`Uploaded ${uploaded} of ${total} assets...`);
578 // Cloudflare only returns the completion JWT on the last bucket it
579 // receives, which under concurrency is not necessarily the last one
580 // in the list — so every response is checked.
581 let jwt: string | undefined | null;
582 yield* Effect.forEach(
583 session.buckets,
584 Effect.fn(function* (bucket) {
585 const result = yield* uploadBucket(bucket, uploadJwt).pipe(semaphore.withPermits(1));
586 uploaded += bucket.length;
587 yield* note(`Uploaded ${uploaded} of ${total} assets...`);
588 if (result.jwt) {
589 jwt = result.jwt;
590 }
591 }),
592 { concurrency: BULK_UPLOAD_CONCURRENCY },
593 );
594 if (!jwt) {
595 return yield* new AssetUploadSessionError({
596 message: `Uploaded ${total} assets for worker ${workerName} but Cloudflare did not return a completion token`,
597 workerName,
598 });
599 }
600 return jwt;
601 });
602
603 const jwt = yield* runSession().pipe(
604 Effect.retry({
605 while: (error): boolean =>
606 error._tag === "Unauthorized" || error._tag === "AssetUploadSessionError",
607 schedule: Schedule.exponential("1 second"),
608 times: 3,
609 }),
610 );
611 return { jwt };
612 });