@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 397 lines 14.6 KB
Raw
1 import { dotAlchemyDirectory } from "alchemy/AlchemyContext";
2 import * as Artifacts from "alchemy/Artifacts";
3 import * as Bundle from "alchemy/Bundle/Bundle";
4 import { exec } from "alchemy/Util/exec";
5 import { isPathWithin } from "alchemy/Util/isPathWithin";
6 import { sha256 } from "alchemy/Util/sha256";
7 import * as Effect from "effect/Effect";
8 import * as FileSystem from "effect/FileSystem";
9 import * as ChildProcess from "effect/process/ChildProcess";
10 import path from "pathe";
11 import { convertPathToPattern, glob } from "tinyglobby";
12 import type { SourceContext, SourceProvider } from "../Source.ts";
13 import { bundleSource, resolveMainPath, watchBundleDirectory } from "./shared.ts";
14
15 /**
16 * Whether a Worker `main` entry points at a Python module. Python Workers
17 * skip the rolldown pipeline entirely — Cloudflare interprets `.py` sources
18 * directly with Pyodide, so the "bundle" is just the source files plus the
19 * vendored `python_modules/` directory.
20 *
21 * The parameter is `unknown` on purpose. `getCompatibility` reaches this from
22 * `WorkerProvider.precreate`, which is handed *raw, unresolved* props so
23 * resources in a dependency cycle can signal early — and `main` is an
24 * `Input`, so it can legitimately be an `Output` there (e.g. an entry path
25 * derived from a `Command.Build` output). An unresolved Output is a callable
26 * Proxy: it passes a `!== undefined` guard, `.split` yields another proxy,
27 * and calling it returns `undefined` — so a value-shaped check explodes with
28 * a bare `TypeError`. Narrowing on `typeof === "string"` keeps the predicate
29 * total: an unresolved `main` is simply not Python yet, and `reconcile`
30 * re-derives compatibility from fully resolved props.
31 */
32 export const isPythonMain = (main: unknown): main is string =>
33 typeof main === "string" && main.split("?")[0].endsWith(".py");
34
35 /**
36 * The Pyodide cross-compilation targets uv understands, keyed by the Python
37 * version the Workers runtime selects from the compatibility date/flags.
38 * The wheel index does not have to match the exact Pyodide release the
39 * runtime uses — only the ABI (emscripten-wasm32) must be compatible.
40 * Mirrors pywrangler (cloudflare/workers-py `pywrangler/utils.py`).
41 */
42 const PYODIDE_TARGETS = {
43 "3.12": {
44 interpreter: "cpython-3.12.7-emscripten-wasm32-musl",
45 index: "https://index.pyodide.org/0.27.7",
46 },
47 "3.13": {
48 interpreter: "cpython-3.13.2-emscripten-wasm32-musl",
49 index: "https://index.pyodide.org/0.28.3",
50 },
51 } as const;
52
53 // `python_workers_20250116` (Python 3.13 / Pyodide 0.28) is implied by
54 // `python_workers` for compatibility dates on or after this day.
55 const PYTHON_3_13_DEFAULT_ON = "2025-09-29";
56
57 /**
58 * The Python version the Workers runtime will use for the given
59 * compatibility settings — the same selection pywrangler performs.
60 */
61 export const pythonVersionForCompatibility = (compatibility: {
62 date: string;
63 flags: string[];
64 }): keyof typeof PYODIDE_TARGETS => {
65 if (compatibility.flags.includes("no_python_workers_20250116")) {
66 return "3.12";
67 }
68 if (
69 compatibility.flags.includes("python_workers_20250116") ||
70 // ISO dates compare lexically.
71 compatibility.date >= PYTHON_3_13_DEFAULT_ON
72 ) {
73 return "3.13";
74 }
75 return "3.12";
76 };
77
78 /**
79 * The PyPI package pywrangler always adds to the vendored dependencies: it
80 * provides the `workers` Python module (Request/Response/WorkerEntrypoint
81 * wrappers) plus the JS shims under `python_modules/workers/` that
82 * `import_from_javascript()` resolves.
83 */
84 const MANAGED_SDK_PACKAGE = "workers-runtime-sdk";
85
86 export interface PythonWorkerBundleOptions {
87 id: string;
88 /** Namespace-qualified id — the display prefix for log lines. */
89 fqn: string;
90 /** Path (or `file://` URL) to the Worker's `.py` entry module. */
91 main: string;
92 compatibility: {
93 date: string;
94 flags: string[];
95 };
96 }
97
98 const uvError = (message: string) => (cause: unknown) =>
99 Bundle.BundleError.make({ message, cause });
100
101 /**
102 * Run a `uv` command, failing with a {@link Bundle.BundleError} that
103 * includes uv's output when the command exits non-zero, and with an
104 * actionable install hint when the binary is missing entirely.
105 */
106 const runUv = Effect.fn(function* (
107 args: string[],
108 options: { cwd: string; env?: Record<string, string> },
109 ) {
110 const result = yield* exec(
111 ChildProcess.make("uv", args, {
112 cwd: options.cwd,
113 env: options.env,
114 extendEnv: true,
115 }),
116 ).pipe(
117 Effect.mapError(
118 uvError(
119 "Failed to run `uv` — Python Workers use uv (>= 0.8.10) to vendor " +
120 "dependencies for Pyodide. Install it from https://docs.astral.sh/uv/ " +
121 "and re-run.",
122 ),
123 ),
124 );
125 if (result.exitCode !== 0) {
126 return yield* Bundle.BundleError.make({
127 message: `\`uv ${args.join(" ")}\` failed with exit code ${result.exitCode}:\n${result.stdout}\n${result.stderr}`,
128 });
129 }
130 return result;
131 });
132
133 /**
134 * Resolve the `python_modules/` directory to upload alongside a Python
135 * Worker's sources:
136 *
137 * - If the Worker's directory already contains `python_modules/` (the
138 * user vendored it themselves, e.g. with `pywrangler sync` or any other
139 * tool that produces Wrangler's vendored layout), it is used as-is.
140 * - Otherwise, the managed Workers runtime SDK and any dependencies from an
141 * optional `pyproject.toml` are vendored with uv into a staging directory
142 * under `.alchemy/`:
143 * resolve against the Pyodide wheel index for the runtime's Python
144 * version (`uv pip compile` → `pylock.toml`), install the prebuilt
145 * wheels into an emscripten cross-venv (`uv venv` + `uv pip install
146 * --no-build`), and copy the venv's `site-packages` out. Re-vendoring
147 * is skipped while the `pyproject.toml` hash is unchanged.
148 *
149 * workerd no longer bundles the `workers` Python module at current
150 * compatibility dates, so even a dependency-free Worker needs the managed
151 * SDK under `python_modules/`.
152 */
153 const resolvePythonModulesDir = Effect.fn(function* (
154 options: PythonWorkerBundleOptions & { root: string },
155 ) {
156 const fs = yield* FileSystem.FileSystem;
157
158 const userManaged = path.join(options.root, "python_modules");
159 if (yield* orDefault(fs.exists(userManaged), false)) {
160 return userManaged;
161 }
162
163 const pyproject = path.join(options.root, "pyproject.toml");
164 const hasPyproject = yield* orDefault(fs.exists(pyproject), false);
165
166 const pythonVersion = pythonVersionForCompatibility(options.compatibility);
167 const target = PYODIDE_TARGETS[pythonVersion];
168 const pyprojectContent = hasPyproject
169 ? yield* fs
170 .readFileString(pyproject)
171 .pipe(Effect.mapError(uvError(`Failed to read "${pyproject}"`)))
172 : "";
173 const inputHash = yield* sha256(`${pythonVersion}\0${target.index}\0${pyprojectContent}`);
174
175 const runtimeBase = process.cwd();
176 const dotAlchemy = yield* dotAlchemyDirectory;
177 // uv runs from the Worker root, so its arguments need absolute staging paths.
178 const staging = path.resolve(runtimeBase, dotAlchemy, "python", options.id);
179 const vendorDir = path.join(staging, "python_modules");
180 const tokenFile = path.join(staging, ".synced");
181
182 const token = yield* orDefault(fs.readFileString(tokenFile), undefined);
183 if (token === inputHash && (yield* orDefault(fs.exists(vendorDir), false))) {
184 return vendorDir;
185 }
186
187 yield* Effect.logDebug(
188 `Vendoring Python dependencies for ${pythonVersion} (emscripten-wasm32) with uv`,
189 );
190 yield* fs
191 .makeDirectory(staging, { recursive: true })
192 .pipe(Effect.mapError(uvError(`Failed to create "${staging}"`)));
193
194 // pywrangler always vendors the managed SDK package alongside the
195 // project's own dependencies; passing it as a second requirements input
196 // avoids mutating the user's pyproject.toml.
197 const extraRequirements = path.join(staging, "requirements.extra.txt");
198 yield* fs
199 .writeFileString(extraRequirements, `${MANAGED_SDK_PACKAGE}\n`)
200 .pipe(Effect.mapError(uvError(`Failed to write "${extraRequirements}"`)));
201
202 // 1. Resolve to a pylock.toml against the Pyodide wheel index. `--no-build`
203 // restricts the resolution to prebuilt (pure-Python or emscripten) wheels
204 // — building a Pyodide-platformed wheel locally is not possible.
205 const lockfile = path.join(staging, "pylock.toml");
206 yield* runUv(
207 [
208 "pip",
209 "compile",
210 ...(hasPyproject ? [pyproject] : []),
211 extraRequirements,
212 "--python",
213 target.interpreter,
214 "--extra-index-url",
215 target.index,
216 "--index-strategy",
217 "unsafe-best-match",
218 "--no-header",
219 "--no-build",
220 "-o",
221 lockfile,
222 ],
223 { cwd: options.root },
224 );
225
226 // 2. Install the locked wheels into a fresh emscripten cross-venv.
227 const venv = path.join(staging, "venv");
228 yield* orDefault(fs.remove(venv, { recursive: true }), undefined);
229 yield* runUv(["venv", venv, "--python", target.interpreter], {
230 cwd: options.root,
231 });
232 yield* runUv(["pip", "install", "--no-build", "-r", lockfile, "--preview-features", "pylock"], {
233 cwd: options.root,
234 env: { VIRTUAL_ENV: venv },
235 });
236
237 // 3. The venv's site-packages IS the vendored python_modules directory.
238 const sitePackages =
239 process.platform === "win32"
240 ? path.join(venv, "Lib", "site-packages")
241 : path.join(venv, "lib", `python${pythonVersion}`, "site-packages");
242 yield* orDefault(fs.remove(vendorDir, { recursive: true }), undefined);
243 yield* fs
244 .copy(sitePackages, vendorDir)
245 .pipe(Effect.mapError(uvError(`Failed to copy "${sitePackages}" to "${vendorDir}"`)));
246 // Mark the directory as a virtual environment (pywrangler parity).
247 yield* fs
248 .writeFileString(path.join(vendorDir, "pyvenv.cfg"), "")
249 .pipe(Effect.mapError(uvError("Failed to write pyvenv.cfg")));
250 yield* fs
251 .writeFileString(tokenFile, inputHash)
252 .pipe(Effect.mapError(uvError(`Failed to write "${tokenFile}"`)));
253
254 return vendorDir;
255 });
256
257 const orDefault = <A, B, R>(
258 effect: Effect.Effect<A, unknown, R>,
259 fallback: B,
260 ): Effect.Effect<A | B, never, R> => Effect.catchCause(effect, () => Effect.succeed(fallback));
261
262 const globFiles = (
263 patterns: string[],
264 options: { cwd: string; ignore?: string[]; dot?: boolean },
265 ) =>
266 Effect.tryPromise({
267 try: () =>
268 glob(patterns, {
269 cwd: options.cwd,
270 onlyFiles: true,
271 expandDirectories: false,
272 dot: options.dot ?? false,
273 ignore: options.ignore,
274 }),
275 catch: (error) =>
276 Bundle.BundleError.make({
277 message: `Failed to list Python worker files in "${options.cwd}"`,
278 cause: error,
279 }),
280 }).pipe(Effect.map((names) => names.map((name) => name.replaceAll("\\", "/")).sort()));
281
282 /**
283 * Read a Python Worker "bundle" from disk. There is no compilation step —
284 * the output is:
285 *
286 * 1. the entry module (first, so it becomes `main_module`), then every
287 * other `.py` file under the entry's directory, read as text;
288 * 2. every file of the resolved `python_modules/` directory (vendored
289 * wheels), read as bytes and named `python_modules/<relpath>` —
290 * Wrangler's vendored-module layout, which both the upload API and
291 * workerd resolve at runtime.
292 */
293 export const readPythonWorkerBundle = Effect.fn(function* (options: PythonWorkerBundleOptions) {
294 const fs = yield* FileSystem.FileSystem;
295 const main = yield* resolveMainPath(options.main);
296 const root = path.dirname(main);
297 const runtimeBase = process.cwd();
298 const dotAlchemy = yield* dotAlchemyDirectory;
299 const entryName = path.basename(main);
300
301 const readTextModule = Effect.fn(function* (name: string) {
302 const content = yield* fs
303 .readFileString(path.join(root, name))
304 .pipe(
305 Effect.mapError(uvError(`Failed to read Python worker module "${path.join(root, name)}"`)),
306 );
307 const hash = yield* sha256(content);
308 return { path: name, content, hash } satisfies Bundle.BundleFile;
309 });
310
311 const sources = globFiles(["**/*.py"], {
312 cwd: root,
313 ignore: [
314 "**/__pycache__/**",
315 "python_modules/**",
316 "**/.venv*/**",
317 "**/node_modules/**",
318 "**/.alchemy/**",
319 `${convertPathToPattern(path.resolve(runtimeBase, dotAlchemy))}/**`,
320 ],
321 }).pipe(
322 Effect.map((names) =>
323 names.filter(
324 (name) =>
325 name !== entryName && !isPathWithin(dotAlchemy, path.join(root, name), runtimeBase),
326 ),
327 ),
328 Effect.flatMap(Effect.forEach(readTextModule, { concurrency: "unbounded" })),
329 );
330
331 const vendored = resolvePythonModulesDir({ ...options, root }).pipe(
332 Effect.flatMap((vendorDir) => {
333 if (!vendorDir) {
334 return Effect.succeed([]);
335 }
336 const readVendoredModule = Effect.fn(function* (name: string) {
337 const content = yield* fs
338 .readFile(path.join(vendorDir, name))
339 .pipe(
340 Effect.mapError(
341 uvError(`Failed to read vendored module "${path.join(vendorDir, name)}"`),
342 ),
343 );
344 const hash = yield* sha256(content);
345 return {
346 path: `python_modules/${name}`,
347 content,
348 hash,
349 } satisfies Bundle.BundleFile;
350 });
351 return globFiles(["**/*"], {
352 cwd: vendorDir,
353 dot: true,
354 // Compiled bytecode is host-specific and excluded by Wrangler too.
355 ignore: ["**/*.pyc", "**/__pycache__/**"],
356 }).pipe(Effect.flatMap(Effect.forEach(readVendoredModule, { concurrency: "unbounded" })));
357 }),
358 );
359
360 const [entry, additional, vendoredFiles] = yield* Effect.all(
361 [readTextModule(entryName), sources, vendored],
362 { concurrency: "unbounded" },
363 );
364 return yield* Bundle.bundleOutputFromFiles([entry, ...additional, ...vendoredFiles]);
365 });
366
367 /**
368 * Watch a Python Worker for changes: emits the initial build, then
369 * rebuilds whenever a file under the entry's directory changes (debounced).
370 * Mirrors the {@link Bundle.BundleWatchEvent} protocol of the rolldown
371 * watcher so local dev can consume either stream interchangeably.
372 */
373 export const watchPythonWorkerBundle = (options: PythonWorkerBundleOptions) =>
374 watchBundleDirectory({
375 main: options.main,
376 read: readPythonWorkerBundle(options),
377 ignore: (path) => path.includes("__pycache__") || path.includes("/python_modules/"),
378 });
379
380 /**
381 * Source provider for Python workers (`.py` entry): no bundling — the
382 * entry plus sibling `.py` files upload as Python modules with
383 * dependencies vendored via uv. The read is cached per run under the
384 * shared `"build"` artifact key so diff and reconcile vendor once.
385 */
386 export const makePythonSource = (main: string): SourceProvider => {
387 const pythonOptions = (ctx: SourceContext) => ({
388 id: ctx.id,
389 fqn: ctx.fqn,
390 main,
391 compatibility: ctx.compatibility,
392 });
393 return bundleSource({
394 build: (ctx) => readPythonWorkerBundle(pythonOptions(ctx)).pipe(Artifacts.cached("build")),
395 watch: (ctx) => Effect.succeed(watchPythonWorkerBundle(pythonOptions(ctx))),
396 });
397 };