The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { dotAlchemyDirectory } from "alchemy/AlchemyContext"; |
| 2 | import * as Artifacts from "alchemy/Artifacts"; |
| 3 | import * as Bundle from "alchemy/Bundle/Bundle"; |
| 4 | import { exec } from "alchemy/Util/exec"; |
| 5 | import { isPathWithin } from "alchemy/Util/isPathWithin"; |
| 6 | import { sha256 } from "alchemy/Util/sha256"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as FileSystem from "effect/FileSystem"; |
| 9 | import * as ChildProcess from "effect/process/ChildProcess"; |
| 10 | import path from "pathe"; |
| 11 | import { convertPathToPattern, glob } from "tinyglobby"; |
| 12 | import type { SourceContext, SourceProvider } from "../Source.ts"; |
| 13 | import { bundleSource, resolveMainPath, watchBundleDirectory } from "./shared.ts"; |
| 14 | |
| 15 | /** |
| 16 | * Whether a Worker `main` entry points at a Python module. Python Workers |
| 17 | * skip the rolldown pipeline entirely — Cloudflare interprets `.py` sources |
| 18 | * directly with Pyodide, so the "bundle" is just the source files plus the |
| 19 | * vendored `python_modules/` directory. |
| 20 | * |
| 21 | * The parameter is `unknown` on purpose. `getCompatibility` reaches this from |
| 22 | * `WorkerProvider.precreate`, which is handed *raw, unresolved* props so |
| 23 | * resources in a dependency cycle can signal early — and `main` is an |
| 24 | * `Input`, so it can legitimately be an `Output` there (e.g. an entry path |
| 25 | * derived from a `Command.Build` output). An unresolved Output is a callable |
| 26 | * Proxy: it passes a `!== undefined` guard, `.split` yields another proxy, |
| 27 | * and calling it returns `undefined` — so a value-shaped check explodes with |
| 28 | * a bare `TypeError`. Narrowing on `typeof === "string"` keeps the predicate |
| 29 | * total: an unresolved `main` is simply not Python yet, and `reconcile` |
| 30 | * re-derives compatibility from fully resolved props. |
| 31 | */ |
| 32 | export const isPythonMain = (main: unknown): main is string => |
| 33 | typeof main === "string" && main.split("?")[0].endsWith(".py"); |
| 34 | |
| 35 | /** |
| 36 | * The Pyodide cross-compilation targets uv understands, keyed by the Python |
| 37 | * version the Workers runtime selects from the compatibility date/flags. |
| 38 | * The wheel index does not have to match the exact Pyodide release the |
| 39 | * runtime uses — only the ABI (emscripten-wasm32) must be compatible. |
| 40 | * Mirrors pywrangler (cloudflare/workers-py `pywrangler/utils.py`). |
| 41 | */ |
| 42 | const PYODIDE_TARGETS = { |
| 43 | "3.12": { |
| 44 | interpreter: "cpython-3.12.7-emscripten-wasm32-musl", |
| 45 | index: "https://index.pyodide.org/0.27.7", |
| 46 | }, |
| 47 | "3.13": { |
| 48 | interpreter: "cpython-3.13.2-emscripten-wasm32-musl", |
| 49 | index: "https://index.pyodide.org/0.28.3", |
| 50 | }, |
| 51 | } as const; |
| 52 | |
| 53 | // `python_workers_20250116` (Python 3.13 / Pyodide 0.28) is implied by |
| 54 | // `python_workers` for compatibility dates on or after this day. |
| 55 | const PYTHON_3_13_DEFAULT_ON = "2025-09-29"; |
| 56 | |
| 57 | /** |
| 58 | * The Python version the Workers runtime will use for the given |
| 59 | * compatibility settings — the same selection pywrangler performs. |
| 60 | */ |
| 61 | export const pythonVersionForCompatibility = (compatibility: { |
| 62 | date: string; |
| 63 | flags: string[]; |
| 64 | }): keyof typeof PYODIDE_TARGETS => { |
| 65 | if (compatibility.flags.includes("no_python_workers_20250116")) { |
| 66 | return "3.12"; |
| 67 | } |
| 68 | if ( |
| 69 | compatibility.flags.includes("python_workers_20250116") || |
| 70 | // ISO dates compare lexically. |
| 71 | compatibility.date >= PYTHON_3_13_DEFAULT_ON |
| 72 | ) { |
| 73 | return "3.13"; |
| 74 | } |
| 75 | return "3.12"; |
| 76 | }; |
| 77 | |
| 78 | /** |
| 79 | * The PyPI package pywrangler always adds to the vendored dependencies: it |
| 80 | * provides the `workers` Python module (Request/Response/WorkerEntrypoint |
| 81 | * wrappers) plus the JS shims under `python_modules/workers/` that |
| 82 | * `import_from_javascript()` resolves. |
| 83 | */ |
| 84 | const MANAGED_SDK_PACKAGE = "workers-runtime-sdk"; |
| 85 | |
| 86 | export interface PythonWorkerBundleOptions { |
| 87 | id: string; |
| 88 | /** Namespace-qualified id — the display prefix for log lines. */ |
| 89 | fqn: string; |
| 90 | /** Path (or `file://` URL) to the Worker's `.py` entry module. */ |
| 91 | main: string; |
| 92 | compatibility: { |
| 93 | date: string; |
| 94 | flags: string[]; |
| 95 | }; |
| 96 | } |
| 97 | |
| 98 | const uvError = (message: string) => (cause: unknown) => |
| 99 | Bundle.BundleError.make({ message, cause }); |
| 100 | |
| 101 | /** |
| 102 | * Run a `uv` command, failing with a {@link Bundle.BundleError} that |
| 103 | * includes uv's output when the command exits non-zero, and with an |
| 104 | * actionable install hint when the binary is missing entirely. |
| 105 | */ |
| 106 | const runUv = Effect.fn(function* ( |
| 107 | args: string[], |
| 108 | options: { cwd: string; env?: Record<string, string> }, |
| 109 | ) { |
| 110 | const result = yield* exec( |
| 111 | ChildProcess.make("uv", args, { |
| 112 | cwd: options.cwd, |
| 113 | env: options.env, |
| 114 | extendEnv: true, |
| 115 | }), |
| 116 | ).pipe( |
| 117 | Effect.mapError( |
| 118 | uvError( |
| 119 | "Failed to run `uv` — Python Workers use uv (>= 0.8.10) to vendor " + |
| 120 | "dependencies for Pyodide. Install it from https://docs.astral.sh/uv/ " + |
| 121 | "and re-run.", |
| 122 | ), |
| 123 | ), |
| 124 | ); |
| 125 | if (result.exitCode !== 0) { |
| 126 | return yield* Bundle.BundleError.make({ |
| 127 | message: `\`uv ${args.join(" ")}\` failed with exit code ${result.exitCode}:\n${result.stdout}\n${result.stderr}`, |
| 128 | }); |
| 129 | } |
| 130 | return result; |
| 131 | }); |
| 132 | |
| 133 | /** |
| 134 | * Resolve the `python_modules/` directory to upload alongside a Python |
| 135 | * Worker's sources: |
| 136 | * |
| 137 | * - If the Worker's directory already contains `python_modules/` (the |
| 138 | * user vendored it themselves, e.g. with `pywrangler sync` or any other |
| 139 | * tool that produces Wrangler's vendored layout), it is used as-is. |
| 140 | * - Otherwise, the managed Workers runtime SDK and any dependencies from an |
| 141 | * optional `pyproject.toml` are vendored with uv into a staging directory |
| 142 | * under `.alchemy/`: |
| 143 | * resolve against the Pyodide wheel index for the runtime's Python |
| 144 | * version (`uv pip compile` → `pylock.toml`), install the prebuilt |
| 145 | * wheels into an emscripten cross-venv (`uv venv` + `uv pip install |
| 146 | * --no-build`), and copy the venv's `site-packages` out. Re-vendoring |
| 147 | * is skipped while the `pyproject.toml` hash is unchanged. |
| 148 | * |
| 149 | * workerd no longer bundles the `workers` Python module at current |
| 150 | * compatibility dates, so even a dependency-free Worker needs the managed |
| 151 | * SDK under `python_modules/`. |
| 152 | */ |
| 153 | const resolvePythonModulesDir = Effect.fn(function* ( |
| 154 | options: PythonWorkerBundleOptions & { root: string }, |
| 155 | ) { |
| 156 | const fs = yield* FileSystem.FileSystem; |
| 157 | |
| 158 | const userManaged = path.join(options.root, "python_modules"); |
| 159 | if (yield* orDefault(fs.exists(userManaged), false)) { |
| 160 | return userManaged; |
| 161 | } |
| 162 | |
| 163 | const pyproject = path.join(options.root, "pyproject.toml"); |
| 164 | const hasPyproject = yield* orDefault(fs.exists(pyproject), false); |
| 165 | |
| 166 | const pythonVersion = pythonVersionForCompatibility(options.compatibility); |
| 167 | const target = PYODIDE_TARGETS[pythonVersion]; |
| 168 | const pyprojectContent = hasPyproject |
| 169 | ? yield* fs |
| 170 | .readFileString(pyproject) |
| 171 | .pipe(Effect.mapError(uvError(`Failed to read "${pyproject}"`))) |
| 172 | : ""; |
| 173 | const inputHash = yield* sha256(`${pythonVersion}\0${target.index}\0${pyprojectContent}`); |
| 174 | |
| 175 | const runtimeBase = process.cwd(); |
| 176 | const dotAlchemy = yield* dotAlchemyDirectory; |
| 177 | // uv runs from the Worker root, so its arguments need absolute staging paths. |
| 178 | const staging = path.resolve(runtimeBase, dotAlchemy, "python", options.id); |
| 179 | const vendorDir = path.join(staging, "python_modules"); |
| 180 | const tokenFile = path.join(staging, ".synced"); |
| 181 | |
| 182 | const token = yield* orDefault(fs.readFileString(tokenFile), undefined); |
| 183 | if (token === inputHash && (yield* orDefault(fs.exists(vendorDir), false))) { |
| 184 | return vendorDir; |
| 185 | } |
| 186 | |
| 187 | yield* Effect.logDebug( |
| 188 | `Vendoring Python dependencies for ${pythonVersion} (emscripten-wasm32) with uv`, |
| 189 | ); |
| 190 | yield* fs |
| 191 | .makeDirectory(staging, { recursive: true }) |
| 192 | .pipe(Effect.mapError(uvError(`Failed to create "${staging}"`))); |
| 193 | |
| 194 | // pywrangler always vendors the managed SDK package alongside the |
| 195 | // project's own dependencies; passing it as a second requirements input |
| 196 | // avoids mutating the user's pyproject.toml. |
| 197 | const extraRequirements = path.join(staging, "requirements.extra.txt"); |
| 198 | yield* fs |
| 199 | .writeFileString(extraRequirements, `${MANAGED_SDK_PACKAGE}\n`) |
| 200 | .pipe(Effect.mapError(uvError(`Failed to write "${extraRequirements}"`))); |
| 201 | |
| 202 | // 1. Resolve to a pylock.toml against the Pyodide wheel index. `--no-build` |
| 203 | // restricts the resolution to prebuilt (pure-Python or emscripten) wheels |
| 204 | // — building a Pyodide-platformed wheel locally is not possible. |
| 205 | const lockfile = path.join(staging, "pylock.toml"); |
| 206 | yield* runUv( |
| 207 | [ |
| 208 | "pip", |
| 209 | "compile", |
| 210 | ...(hasPyproject ? [pyproject] : []), |
| 211 | extraRequirements, |
| 212 | "--python", |
| 213 | target.interpreter, |
| 214 | "--extra-index-url", |
| 215 | target.index, |
| 216 | "--index-strategy", |
| 217 | "unsafe-best-match", |
| 218 | "--no-header", |
| 219 | "--no-build", |
| 220 | "-o", |
| 221 | lockfile, |
| 222 | ], |
| 223 | { cwd: options.root }, |
| 224 | ); |
| 225 | |
| 226 | // 2. Install the locked wheels into a fresh emscripten cross-venv. |
| 227 | const venv = path.join(staging, "venv"); |
| 228 | yield* orDefault(fs.remove(venv, { recursive: true }), undefined); |
| 229 | yield* runUv(["venv", venv, "--python", target.interpreter], { |
| 230 | cwd: options.root, |
| 231 | }); |
| 232 | yield* runUv(["pip", "install", "--no-build", "-r", lockfile, "--preview-features", "pylock"], { |
| 233 | cwd: options.root, |
| 234 | env: { VIRTUAL_ENV: venv }, |
| 235 | }); |
| 236 | |
| 237 | // 3. The venv's site-packages IS the vendored python_modules directory. |
| 238 | const sitePackages = |
| 239 | process.platform === "win32" |
| 240 | ? path.join(venv, "Lib", "site-packages") |
| 241 | : path.join(venv, "lib", `python${pythonVersion}`, "site-packages"); |
| 242 | yield* orDefault(fs.remove(vendorDir, { recursive: true }), undefined); |
| 243 | yield* fs |
| 244 | .copy(sitePackages, vendorDir) |
| 245 | .pipe(Effect.mapError(uvError(`Failed to copy "${sitePackages}" to "${vendorDir}"`))); |
| 246 | // Mark the directory as a virtual environment (pywrangler parity). |
| 247 | yield* fs |
| 248 | .writeFileString(path.join(vendorDir, "pyvenv.cfg"), "") |
| 249 | .pipe(Effect.mapError(uvError("Failed to write pyvenv.cfg"))); |
| 250 | yield* fs |
| 251 | .writeFileString(tokenFile, inputHash) |
| 252 | .pipe(Effect.mapError(uvError(`Failed to write "${tokenFile}"`))); |
| 253 | |
| 254 | return vendorDir; |
| 255 | }); |
| 256 | |
| 257 | const orDefault = <A, B, R>( |
| 258 | effect: Effect.Effect<A, unknown, R>, |
| 259 | fallback: B, |
| 260 | ): Effect.Effect<A | B, never, R> => Effect.catchCause(effect, () => Effect.succeed(fallback)); |
| 261 | |
| 262 | const globFiles = ( |
| 263 | patterns: string[], |
| 264 | options: { cwd: string; ignore?: string[]; dot?: boolean }, |
| 265 | ) => |
| 266 | Effect.tryPromise({ |
| 267 | try: () => |
| 268 | glob(patterns, { |
| 269 | cwd: options.cwd, |
| 270 | onlyFiles: true, |
| 271 | expandDirectories: false, |
| 272 | dot: options.dot ?? false, |
| 273 | ignore: options.ignore, |
| 274 | }), |
| 275 | catch: (error) => |
| 276 | Bundle.BundleError.make({ |
| 277 | message: `Failed to list Python worker files in "${options.cwd}"`, |
| 278 | cause: error, |
| 279 | }), |
| 280 | }).pipe(Effect.map((names) => names.map((name) => name.replaceAll("\\", "/")).sort())); |
| 281 | |
| 282 | /** |
| 283 | * Read a Python Worker "bundle" from disk. There is no compilation step — |
| 284 | * the output is: |
| 285 | * |
| 286 | * 1. the entry module (first, so it becomes `main_module`), then every |
| 287 | * other `.py` file under the entry's directory, read as text; |
| 288 | * 2. every file of the resolved `python_modules/` directory (vendored |
| 289 | * wheels), read as bytes and named `python_modules/<relpath>` — |
| 290 | * Wrangler's vendored-module layout, which both the upload API and |
| 291 | * workerd resolve at runtime. |
| 292 | */ |
| 293 | export const readPythonWorkerBundle = Effect.fn(function* (options: PythonWorkerBundleOptions) { |
| 294 | const fs = yield* FileSystem.FileSystem; |
| 295 | const main = yield* resolveMainPath(options.main); |
| 296 | const root = path.dirname(main); |
| 297 | const runtimeBase = process.cwd(); |
| 298 | const dotAlchemy = yield* dotAlchemyDirectory; |
| 299 | const entryName = path.basename(main); |
| 300 | |
| 301 | const readTextModule = Effect.fn(function* (name: string) { |
| 302 | const content = yield* fs |
| 303 | .readFileString(path.join(root, name)) |
| 304 | .pipe( |
| 305 | Effect.mapError(uvError(`Failed to read Python worker module "${path.join(root, name)}"`)), |
| 306 | ); |
| 307 | const hash = yield* sha256(content); |
| 308 | return { path: name, content, hash } satisfies Bundle.BundleFile; |
| 309 | }); |
| 310 | |
| 311 | const sources = globFiles(["**/*.py"], { |
| 312 | cwd: root, |
| 313 | ignore: [ |
| 314 | "**/__pycache__/**", |
| 315 | "python_modules/**", |
| 316 | "**/.venv*/**", |
| 317 | "**/node_modules/**", |
| 318 | "**/.alchemy/**", |
| 319 | `${convertPathToPattern(path.resolve(runtimeBase, dotAlchemy))}/**`, |
| 320 | ], |
| 321 | }).pipe( |
| 322 | Effect.map((names) => |
| 323 | names.filter( |
| 324 | (name) => |
| 325 | name !== entryName && !isPathWithin(dotAlchemy, path.join(root, name), runtimeBase), |
| 326 | ), |
| 327 | ), |
| 328 | Effect.flatMap(Effect.forEach(readTextModule, { concurrency: "unbounded" })), |
| 329 | ); |
| 330 | |
| 331 | const vendored = resolvePythonModulesDir({ ...options, root }).pipe( |
| 332 | Effect.flatMap((vendorDir) => { |
| 333 | if (!vendorDir) { |
| 334 | return Effect.succeed([]); |
| 335 | } |
| 336 | const readVendoredModule = Effect.fn(function* (name: string) { |
| 337 | const content = yield* fs |
| 338 | .readFile(path.join(vendorDir, name)) |
| 339 | .pipe( |
| 340 | Effect.mapError( |
| 341 | uvError(`Failed to read vendored module "${path.join(vendorDir, name)}"`), |
| 342 | ), |
| 343 | ); |
| 344 | const hash = yield* sha256(content); |
| 345 | return { |
| 346 | path: `python_modules/${name}`, |
| 347 | content, |
| 348 | hash, |
| 349 | } satisfies Bundle.BundleFile; |
| 350 | }); |
| 351 | return globFiles(["**/*"], { |
| 352 | cwd: vendorDir, |
| 353 | dot: true, |
| 354 | // Compiled bytecode is host-specific and excluded by Wrangler too. |
| 355 | ignore: ["**/*.pyc", "**/__pycache__/**"], |
| 356 | }).pipe(Effect.flatMap(Effect.forEach(readVendoredModule, { concurrency: "unbounded" }))); |
| 357 | }), |
| 358 | ); |
| 359 | |
| 360 | const [entry, additional, vendoredFiles] = yield* Effect.all( |
| 361 | [readTextModule(entryName), sources, vendored], |
| 362 | { concurrency: "unbounded" }, |
| 363 | ); |
| 364 | return yield* Bundle.bundleOutputFromFiles([entry, ...additional, ...vendoredFiles]); |
| 365 | }); |
| 366 | |
| 367 | /** |
| 368 | * Watch a Python Worker for changes: emits the initial build, then |
| 369 | * rebuilds whenever a file under the entry's directory changes (debounced). |
| 370 | * Mirrors the {@link Bundle.BundleWatchEvent} protocol of the rolldown |
| 371 | * watcher so local dev can consume either stream interchangeably. |
| 372 | */ |
| 373 | export const watchPythonWorkerBundle = (options: PythonWorkerBundleOptions) => |
| 374 | watchBundleDirectory({ |
| 375 | main: options.main, |
| 376 | read: readPythonWorkerBundle(options), |
| 377 | ignore: (path) => path.includes("__pycache__") || path.includes("/python_modules/"), |
| 378 | }); |
| 379 | |
| 380 | /** |
| 381 | * Source provider for Python workers (`.py` entry): no bundling — the |
| 382 | * entry plus sibling `.py` files upload as Python modules with |
| 383 | * dependencies vendored via uv. The read is cached per run under the |
| 384 | * shared `"build"` artifact key so diff and reconcile vendor once. |
| 385 | */ |
| 386 | export const makePythonSource = (main: string): SourceProvider => { |
| 387 | const pythonOptions = (ctx: SourceContext) => ({ |
| 388 | id: ctx.id, |
| 389 | fqn: ctx.fqn, |
| 390 | main, |
| 391 | compatibility: ctx.compatibility, |
| 392 | }); |
| 393 | return bundleSource({ |
| 394 | build: (ctx) => readPythonWorkerBundle(pythonOptions(ctx)).pipe(Artifacts.cached("build")), |
| 395 | watch: (ctx) => Effect.succeed(watchPythonWorkerBundle(pythonOptions(ctx))), |
| 396 | }); |
| 397 | }; |