The best way to control Claude Code, Codex and more from any computer or phone

main
ts 460 lines 20.6 KB
Raw
1 import type { WorkerEnv } from "../alchemy.run.ts";
2 import { referenceDestination } from "./reference-links.ts";
3 import { SOCIAL_REDIRECTS } from "./social-redirects.ts";
4
5 // Minimal `HTMLRewriter` shape — the workers runtime exposes it as a
6 // global, but we don't pull in `@cloudflare/workers-types`, so declare
7 // just what this file uses.
8 declare class HTMLRewriter {
9 on(selector: string, handler: { element(el: HTMLRewriterElement): void }): HTMLRewriter;
10 transform(response: Response): Response;
11 }
12 interface HTMLRewriterElement {
13 getAttribute(name: string): string | null;
14 setAttribute(name: string, value: string): HTMLRewriterElement;
15 }
16
17 /**
18 * `https://alchemy.run` is the one indexable origin. Every other host this
19 * worker answers on — `main.alchemy.run`, PR previews on `*.workers.dev`,
20 * future aliases — is a mirror of the same build:
21 *
22 * - `<link rel="canonical">` is baked at build time against `site` and is left
23 * alone, so mirrors point search engines back at production.
24 * - Every mirror response carries `X-Robots-Tag: noindex` and its `robots.txt`
25 * advertises no sitemap, so a mirror URL that leaks (a PR comment, a shared
26 * link) never becomes a duplicate search result.
27 * - `og:*` / `twitter:*` card URLs ARE rewritten to the request host, so a
28 * Slack/X unfurl of a preview URL shows that preview's card, not
29 * production's.
30 */
31 const CANONICAL_HOST = "alchemy.run";
32 const CANONICAL_ORIGIN = `https://${CANONICAL_HOST}`;
33
34 /**
35 * The v1 docs moved to their own host. v1-era URLs (`/docs/...`, and the
36 * pre-hub `/guides/...` / `/providers/...` / `/concepts/...` trees) are still
37 * in search indexes; instead of 404ing them we probe v1 and 301 when it has
38 * the page.
39 */
40 const V1_ORIGIN = "https://v1.alchemy.run";
41 const V1_PREFIXES = ["/docs/", "/guides/", "/providers/", "/concepts/"];
42
43 /**
44 * 301s for the docs restructure (guides/tutorials moved into per-cloud hubs).
45 * Keys and targets are extensionless; `.md` requests (agents fetching raw
46 * markdown) are redirected to the target's `.md` form with any fragment
47 * dropped.
48 */
49 const REDIRECTS: Record<string, string> = {
50 "/tutorial/part-1": "/cloudflare/tutorial/part-1",
51 "/tutorial/part-2": "/cloudflare/tutorial/part-2",
52 "/tutorial/part-3": "/cloudflare/tutorial/part-3",
53 "/tutorial/part-4": "/cloudflare/tutorial/part-4",
54 "/tutorial/part-5": "/cloudflare/tutorial/part-5",
55 "/tutorial/cloudflare/compute/durable-objects": "/cloudflare/compute/durable-objects",
56 "/tutorial/cloudflare/data/hyperdrive": "/cloudflare/data/hyperdrive",
57 "/tutorial/cloudflare/queue-consumer": "/cloudflare/messaging/queues",
58 "/tutorial/cloudflare/rpc-durable-object": "/cloudflare/compute/durable-objects#schemaless-rpc",
59 "/tutorial/cloudflare/rpc-worker": "/cloudflare/compute/workers#call-another-worker",
60 "/tutorial/cloudflare/ai-gateway": "/cloudflare/ai/ai-gateway",
61 "/tutorial/cloudflare/ai-search": "/cloudflare/ai/ai-search",
62 "/tutorial/cloudflare/artifacts": "/cloudflare/data/artifacts",
63 "/tutorial/cloudflare/branch-from-shared-database":
64 "/cloudflare/data/branch-from-shared-database",
65 "/tutorial/cloudflare/compute/containers": "/cloudflare/compute/run-a-container",
66 "/tutorial/cloudflare/cross-worker-durable-object":
67 "/cloudflare/compute/cross-worker-durable-object",
68 "/tutorial/cloudflare/drizzle": "/cloudflare/data/drizzle",
69 "/tutorial/cloudflare/hibernatable-websockets": "/cloudflare/compute/hibernatable-websockets",
70 "/tutorial/cloudflare/vite-spa": "/cloudflare/frontend/vite-spa",
71 "/tutorial/cloudflare/compute/workflows": "/cloudflare/compute/add-a-workflow",
72 "/tutorial/aws/compute/lambda": "/aws/compute/lambda",
73 "/tutorial/aws/data/dynamodb": "/aws/data/dynamodb",
74 "/tutorial/aws/messaging/sqs": "/aws/messaging/sqs",
75 "/tutorial/aws/data/s3": "/aws/data/s3",
76 "/tutorial/aws/messaging/kinesis": "/aws/messaging/kinesis",
77 "/tutorial/aws/api-gateway": "/aws/apis/api-gateway",
78 "/tutorial/aws/dynamodb-streams": "/aws/messaging/dynamodb-streams",
79 "/tutorial/aws/s3-events": "/aws/messaging/s3-events",
80 "/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
81 "/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
82 "/guides/effect-ai": "/cloudflare/ai/effect-ai",
83 "/guides/frontends": "/cloudflare/frontend/frontends",
84 "/guides/shared-database": "/cloudflare/data/shared-database",
85 // Absorbed into the Security & secrets block pages.
86 "/guides/secrets": "/cloudflare/security/secrets-env",
87 "/guides/stack-references": "/infrastructure-as-code/references",
88 // The CLI reference became its own hub tab.
89 "/guides/cli": "/cli",
90 "/aws/guides/secrets": "/aws/security/secrets-env",
91 // The Integrations tab was replaced by per-provider hubs (PlanetScale and
92 // Neon promoted to top-level tabs; Axiom and GitHub in the More menu).
93 // Section-folder alignment: file locations now mirror the sidebar taxonomy.
94 "/aws/choosing-a-runtime": "/aws/compute/choosing-a-runtime",
95 "/aws/cloudwatch": "/aws/observability/cloudwatch",
96 "/aws/dynamodb": "/aws/data/dynamodb",
97 "/aws/ec2": "/aws/compute/ec2",
98 "/aws/ecs": "/aws/compute/ecs",
99 "/aws/eks": "/aws/compute/eks",
100 "/aws/eventbridge": "/aws/messaging/eventbridge",
101 "/aws/guides/api-gateway": "/aws/apis/api-gateway",
102 "/aws/guides/custom-domains": "/aws/networking/custom-domains",
103 "/aws/guides/dynamodb-streams": "/aws/messaging/dynamodb-streams",
104 "/aws/guides/effect-http-api": "/aws/apis/effect-http-api",
105 "/aws/guides/effect-rpc": "/aws/apis/effect-rpc",
106 "/aws/guides/microvms": "/aws/compute/microvms",
107 "/aws/guides/s3-events": "/aws/messaging/s3-events",
108 "/aws/guides/static-site": "/aws/frontend/static-site",
109 "/aws/kinesis": "/aws/messaging/kinesis",
110 "/aws/lambda": "/aws/compute/lambda",
111 "/aws/rds": "/aws/data/rds",
112 "/aws/s3": "/aws/data/s3",
113 "/aws/secrets-env": "/aws/security/secrets-env",
114 "/aws/sns": "/aws/messaging/sns",
115 "/aws/sqs": "/aws/messaging/sqs",
116 "/aws/websites": "/aws/frontend/websites",
117 "/axiom/ingest": "/axiom/data/ingest",
118 "/cloudflare/containers": "/cloudflare/compute/containers",
119 "/cloudflare/d1": "/cloudflare/data/d1",
120 "/cloudflare/domains": "/cloudflare/networking/domains",
121 "/cloudflare/durable-objects": "/cloudflare/compute/durable-objects",
122 "/cloudflare/guides/ai-gateway": "/cloudflare/ai/ai-gateway",
123 "/cloudflare/guides/ai-search": "/cloudflare/ai/ai-search",
124 "/cloudflare/guides/analytics-engine": "/cloudflare/observability/analytics-engine",
125 "/cloudflare/guides/artifacts": "/cloudflare/data/artifacts",
126 "/cloudflare/guides/axiom-observability": "/cloudflare/observability/axiom-observability",
127 "/cloudflare/guides/branch-from-shared-database": "/cloudflare/data/branch-from-shared-database",
128 "/cloudflare/guides/browser-rendering": "/cloudflare/compute/browser-rendering",
129 "/cloudflare/guides/containers": "/cloudflare/compute/run-a-container",
130 "/cloudflare/guides/cron": "/cloudflare/messaging/cron",
131 "/cloudflare/guides/cross-worker-durable-object":
132 "/cloudflare/compute/cross-worker-durable-object",
133 "/cloudflare/guides/custom-domains": "/cloudflare/networking/custom-domains",
134 "/cloudflare/guides/drizzle": "/cloudflare/data/drizzle",
135 "/cloudflare/guides/effect-ai": "/cloudflare/ai/effect-ai",
136 "/cloudflare/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
137 "/cloudflare/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
138 "/cloudflare/guides/email": "/cloudflare/email/send-and-receive",
139 "/cloudflare/guides/frontends": "/cloudflare/frontend/frontends",
140 "/cloudflare/guides/github-events": "/cloudflare/messaging/github-events",
141 "/cloudflare/guides/hibernatable-websockets": "/cloudflare/compute/hibernatable-websockets",
142 "/cloudflare/guides/secrets-store": "/cloudflare/security/secrets-store",
143 "/cloudflare/guides/shared-database": "/cloudflare/data/shared-database",
144 "/cloudflare/guides/tunnel": "/cloudflare/networking/tunnel",
145 "/cloudflare/guides/turnstile": "/cloudflare/security/turnstile",
146 "/cloudflare/guides/vectorize": "/cloudflare/ai/vectorize",
147 "/cloudflare/guides/vite-spa": "/cloudflare/frontend/vite-spa",
148 "/cloudflare/guides/workers-for-platforms": "/cloudflare/compute/workers-for-platforms",
149 "/cloudflare/guides/workflows": "/cloudflare/compute/add-a-workflow",
150 "/cloudflare/hyperdrive": "/cloudflare/data/hyperdrive",
151 "/cloudflare/kv": "/cloudflare/data/kv",
152 "/cloudflare/queues": "/cloudflare/messaging/queues",
153 "/cloudflare/r2": "/cloudflare/data/r2",
154 "/cloudflare/secrets-env": "/cloudflare/security/secrets-env",
155 "/cloudflare/workers": "/cloudflare/compute/workers",
156 "/cloudflare/workflows": "/cloudflare/compute/workflows",
157 "/concepts/action": "/infrastructure-as-code/action",
158 "/concepts/binding": "/infrastructure-as-effects/binding",
159 "/concepts/layers": "/infrastructure-as-effects/layers",
160 "/concepts/local-development": "/environments/local-development",
161 "/concepts/observability": "/testing/observability",
162 "/concepts/outputs": "/infrastructure-as-code/outputs",
163 "/concepts/phases": "/infrastructure-as-effects/phases",
164 "/concepts/platform": "/infrastructure-as-effects/runtime",
165 "/infrastructure-as-effects/platform": "/infrastructure-as-effects/runtime",
166 "/infrastructure-as-effects/functions-and-servers": "/infrastructure-as-effects/runtime",
167 "/rpc": "/apis",
168 "/rpc/schemaless": "/apis/schemaless",
169 "/rpc/effect-rpc": "/apis/effect-rpc",
170 "/rpc/effect-http": "/apis/effect-http",
171 "/concepts/profiles": "/environments/profiles",
172 "/concepts/provider": "/infrastructure-as-code/provider",
173 "/concepts/references": "/infrastructure-as-code/references",
174 "/concepts/resource": "/infrastructure-as-code/resource",
175 "/concepts/resource-lifecycle": "/infrastructure-as-code/resource-lifecycle",
176 "/concepts/secrets": "/environments/secrets",
177 "/concepts/stack": "/infrastructure-as-code/stack",
178 "/concepts/stages": "/environments/stages",
179 "/concepts/state-store": "/state-store",
180 "/concepts/test-harness": "/testing/test-harness",
181 "/concepts/testing": "/testing",
182 "/fly/frontend": "/fly/frontend/websites",
183 "/fly/websites": "/fly/frontend/websites",
184 "/gcp/guides/bindings": "/gcp/security/iam",
185 "/gcp/guides/cloud-run-api": "/gcp/compute/cloud-run",
186 "/gcp/guides/event-pipeline": "/gcp/messaging/pubsub",
187 "/gcp/guides/memorystore": "/gcp/data/memorystore",
188 "/guides/ci": "/environments/ci",
189 "/guides/circular-bindings": "/infrastructure-as-effects/circular-bindings",
190 "/guides/custom-provider": "/infrastructure-as-code/custom-provider",
191 "/guides/custom-state-store": "/state-store/custom-state-store",
192 "/guides/file-layout": "/project-structure/file-layout",
193 "/guides/infrastructure-layers": "/infrastructure-as-effects/layers",
194 "/infrastructure-as-effects/infrastructure-layers": "/infrastructure-as-effects/layers",
195 "/guides/migrating-from-v1": "/migrating-from-v1",
196 "/guides/monorepo": "/project-structure/monorepo",
197 "/guides/monorepo-multi-stack": "/project-structure/monorepo-multi-stack",
198 "/guides/monorepo-single-stack": "/project-structure/monorepo-single-stack",
199 "/guides/testing-a-stack": "/testing/testing-a-stack",
200 "/guides/testing-providers": "/testing/testing-providers",
201 "/hetzner/frontend": "/hetzner/frontend/websites",
202 "/hetzner/websites": "/hetzner/frontend/websites",
203 "/neon/branching": "/neon/data/branching",
204 "/neon/connections": "/neon/data/connections",
205 "/neon/migrations": "/neon/data/migrations",
206 "/planetscale/backups": "/planetscale/data/backups",
207 "/planetscale/credentials": "/planetscale/data/credentials",
208 "/planetscale/migrations": "/planetscale/data/migrations",
209 "/planetscale/mysql": "/planetscale/data/mysql",
210 "/planetscale/postgres": "/planetscale/data/postgres",
211 "/railway/frontend": "/railway/frontend/websites",
212 "/railway/websites": "/railway/frontend/websites",
213 "/integrations": "/providers",
214 "/integrations/planetscale": "/planetscale",
215 "/integrations/neon": "/neon",
216 "/integrations/axiom": "/axiom",
217 "/integrations/github": "/github",
218 "/git/auth": "/git/blocks/auth",
219 "/git/storage": "/git/blocks/repositories",
220 "/git/assemblies": "/git/recipes",
221 "/git/assemblies/cloudflare": "/git/recipes/cloudflare",
222 "/git/assemblies/cloudflare-aws": "/git/recipes/cloudflare-aws",
223 "/git/assemblies/one-origin": "/git/tutorial/part-4",
224 "/git/build": "/git/tutorial/part-4",
225 "/git/scale": "/git/recipes/scaling",
226 "/git/scale/pushes": "/git/blocks/hasher",
227 "/git/scale/clones": "/git/blocks/repositories",
228 };
229
230 const resolveRedirect = (url: URL): string | undefined => {
231 const social = SOCIAL_REDIRECTS[url.pathname.replace(/\/$/, "")];
232 if (social) return social;
233 const reference = referenceDestination(url.pathname + url.search);
234 if (reference) return reference;
235 let p = url.pathname.replace(/\/$/, "");
236 const isMarkdown = p.endsWith(".md");
237 if (isMarkdown) p = p.slice(0, -".md".length);
238 const target = REDIRECTS[p];
239 if (!target) return undefined;
240 if (isMarkdown) return `${target.split("#")[0]}.md`;
241 return target;
242 };
243
244 export default {
245 fetch: async (request: Request, env: WorkerEnv) => {
246 const url = new URL(request.url);
247 const canonical = url.host === CANONICAL_HOST;
248 const redirect = resolveRedirect(url);
249 if (redirect !== undefined) {
250 return Response.redirect(new URL(redirect, request.url), 301);
251 }
252 if (url.pathname === "/robots.txt" && !canonical) {
253 return new Response(MIRROR_ROBOTS_TXT, {
254 headers: {
255 "content-type": "text/plain; charset=utf-8",
256 "x-robots-tag": "noindex",
257 },
258 });
259 }
260 if (request.method === "GET" && prefersMarkdown(request)) {
261 const mdUrl = toMarkdownUrl(url).toString();
262 const res = await env.ASSETS.fetch(new Request(mdUrl, request));
263 // Astro's asset server labels `.md` as `application/octet-stream`, which
264 // agents treat as a binary download instead of rendering. Force the
265 // correct text type + charset since this branch only ever serves markdown.
266 if (res.status !== 404)
267 return withoutIndexing(
268 withContentType(res, "text/markdown; charset=utf-8"),
269 url,
270 canonical,
271 );
272 }
273 const res = await env.ASSETS.fetch(request);
274 if (res.status === 404 && request.method === "GET") {
275 const v1 = await resolveV1Fallback(url);
276 if (v1 !== undefined) return Response.redirect(v1, 301);
277 }
278 return withoutIndexing(
279 withUtf8Charset(await rewriteAgentTextOrigin(request, rewriteSocialCardHost(request, res))),
280 url,
281 canonical,
282 );
283 },
284 };
285
286 /**
287 * Served as `robots.txt` on every non-canonical host. Crawlable on purpose:
288 * a `Disallow` would stop crawlers from ever seeing the `noindex` header, and
289 * a blocked URL can still be indexed by URL alone. No `Sitemap:` line, so the
290 * mirror never advertises itself.
291 */
292 const MIRROR_ROBOTS_TXT = `# Mirror of https://${CANONICAL_HOST} — every response is noindex.
293 User-agent: *
294 Allow: /
295 `;
296
297 /**
298 * Every page is mirrored as raw markdown (`/getting-started.md`, or the same
299 * URL requested with `Accept: text/markdown`) for agents. Search engines crawl
300 * those mirrors too and index them as separate documents — duplicating the HTML
301 * page and surfacing raw MDX (`import { Tabs } from "@astrojs/starlight/..."`)
302 * as the search result's snippet. `llms.txt` / `llms-full.txt` are the same
303 * kind of agent-facing text.
304 *
305 * Keep them crawlable (so the directive is actually seen) but out of the index.
306 * On a non-canonical host *everything* is noindex — the HTML included.
307 */
308 const withoutIndexing = (res: Response, url: URL, canonicalHost: boolean): Response => {
309 const ct = res.headers.get("content-type") ?? "";
310 const agentText =
311 url.pathname.endsWith(".md") || ct.includes("text/markdown") || ct.includes("text/plain");
312 if (canonicalHost && !agentText) return res;
313 const next = new Response(res.body, res);
314 next.headers.set("x-robots-tag", "noindex");
315 return next;
316 };
317
318 /**
319 * v1-era URL that still resolves on the v1 docs host, or `undefined`. Only
320 * consulted after production 404s, so the extra HEAD subrequest is confined
321 * to dead links.
322 */
323 const resolveV1Fallback = async (url: URL): Promise<string | undefined> => {
324 const pathname = url.pathname.replace(/\/$/, "");
325 if (pathname.endsWith(".md")) return undefined;
326 if (!V1_PREFIXES.some((prefix) => `${pathname}/`.startsWith(prefix))) {
327 return undefined;
328 }
329 const v1Path = pathname.startsWith("/docs/") ? pathname.slice("/docs".length) : pathname;
330 const target = `${V1_ORIGIN}${v1Path}${url.search}`;
331 try {
332 const probe = await fetch(target, { method: "HEAD", redirect: "follow" });
333 return probe.ok ? target : undefined;
334 } catch {
335 return undefined;
336 }
337 };
338
339 /**
340 * `llms.txt` / `llms-full.txt` are generated at build time with absolute
341 * canonical URLs, so a PR preview would hand agents an index that points back
342 * at production. Rewrite the baked origin to the request's own origin — the
343 * same treatment `rewriteSocialCardHost` gives the OG tags.
344 */
345 const AGENT_TEXT_PATHS = new Set(["/llms.txt", "/llms-full.txt"]);
346
347 const rewriteAgentTextOrigin = async (request: Request, res: Response): Promise<Response> => {
348 const reqUrl = new URL(request.url);
349 if (!AGENT_TEXT_PATHS.has(reqUrl.pathname) || reqUrl.host === CANONICAL_HOST || !res.ok) {
350 return res;
351 }
352 const body = (await res.text()).replaceAll(`${CANONICAL_ORIGIN}/`, `${reqUrl.origin}/`);
353 return new Response(body, res);
354 };
355
356 /**
357 * Astro's static asset server labels `.txt` as `text/plain` with no charset.
358 * UTF-8 bytes (em dashes, arrows in our docs and `llms.txt`) then get decoded
359 * as latin-1 by browsers and agents, showing up as mojibake (`â€"`). Stamp
360 * `charset=utf-8` on text responses that omit it.
361 */
362 const withUtf8Charset = (res: Response): Response => {
363 const ct = res.headers.get("content-type");
364 if (!ct || !ct.startsWith("text/") || /charset=/i.test(ct)) return res;
365 return withContentType(res, `${ct}; charset=utf-8`);
366 };
367
368 const withContentType = (res: Response, contentType: string): Response => {
369 const next = new Response(res.body, res);
370 next.headers.set("content-type", contentType);
371 return next;
372 };
373
374 /**
375 * Astro bakes absolute URLs into `og:image`, `og:url`, and `twitter:image`
376 * using the `site` config. Rewrite those to the request's host so each
377 * deployment unfurls itself. `<link rel="canonical">` is deliberately NOT
378 * rewritten — it must keep pointing at production on every mirror.
379 */
380 const rewriteSocialCardHost = (request: Request, res: Response): Response => {
381 const ct = res.headers.get("content-type") ?? "";
382 if (!ct.includes("text/html")) return res;
383 const reqUrl = new URL(request.url);
384 if (reqUrl.host === CANONICAL_HOST) return res;
385
386 const content = {
387 element(el: HTMLRewriterElement) {
388 const value = el.getAttribute("content");
389 if (!value) return;
390 let u: URL;
391 try {
392 u = new URL(value);
393 } catch {
394 return;
395 }
396 if (u.host !== CANONICAL_HOST) return;
397 u.protocol = reqUrl.protocol;
398 u.host = reqUrl.host;
399 el.setAttribute("content", u.toString());
400 },
401 };
402
403 return new HTMLRewriter()
404 .on('meta[property="og:image"]', content)
405 .on('meta[property="og:url"]', content)
406 .on('meta[name="twitter:image"]', content)
407 .transform(res);
408 };
409
410 /**
411 * Returns true if the accept header prefers markdown or plain text over HTML.
412 *
413 * Examples:
414 * - opencode - accept: text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, *\/*;q=0.1
415 * - claude code - accept: application/json, text/plain, *\/*
416 *
417 * Notes:
418 * - ChatGPT and Claude web don't set an accept header; maybe check the user agent instead?
419 * - Cursor's headers are too generic (accept: *, user-agent: https://github.com/sindresorhus/got)
420 */
421 const prefersMarkdown = (request: Request) => {
422 const accept = request.headers.get("accept");
423 if (!accept) return false;
424
425 // parse accept header and sort by quality; highest quality first
426 const types = accept
427 .split(",")
428 .map((part) => {
429 const type = part.split(";")[0].trim();
430 const q = part.match(/q=([^,]+)/)?.[1];
431 return { type, q: q ? Number.parseFloat(q) : 1 };
432 })
433 .sort((a, b) => b.q - a.q)
434 .map((type) => type.type);
435
436 const markdown = types.indexOf("text/markdown");
437 const plain = types.indexOf("text/plain");
438 const html = types.indexOf("text/html");
439
440 // if no HTML is specified, and either markdown or plain text is specified, prefer markdown
441 if (html === -1) {
442 return markdown !== -1 || plain !== -1;
443 }
444
445 // prefer markdown if higher quality than HTML
446 if ((markdown !== -1 && markdown < html) || (plain !== -1 && plain < html)) {
447 return true;
448 }
449
450 // otherwise, prefer HTML
451 return false;
452 };
453
454 function toMarkdownUrl(url: URL): URL {
455 const md = new URL(url.toString());
456 let p = md.pathname.replace(/\/$/, "");
457 if (p === "") p = "/index";
458 md.pathname = `${p}.md`;
459 return md;
460 }