A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[compile] Add fallback path to AOT compile when serialization fails. (#27350)
Signed-off-by: zhxchen17 <zhxchen17@fb.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Zhengxu Chen committed
Oct 28, 2025 at 12:54 UTC
e3d81866662e0ce398a2985b82e59075f327f51f
1 file changed
+11
-2
vllm/compilation/decorators.py
+11
-2
@@ -403,8 +403,17 @@ def _support_torch_compile(
403
output = self.aot_compiled_fn(self, *args, **kwargs)
404
assert aot_compilation_path is not None
405
assert cache_dir is not None
406
- os.makedirs(cache_dir, exist_ok=True)
407
- self.aot_compiled_fn.save_compiled_function(aot_compilation_path)
406
+ try:
407
+ os.makedirs(cache_dir, exist_ok=True)
408
+ self.aot_compiled_fn.save_compiled_function(
409
+ aot_compilation_path
410
+ )
411
+ except Exception as e:
412
+ logger.warning(
413
+ "Cannot save aot compilation to path %s, error: %s",
414
+ aot_compilation_path,
415
+ str(e),
416
+ )
417
else:
418
output = self.compiled_callable(*args, **kwargs)
419
return output