A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Fix torch.compile raise FileNotFoundError (#15278)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 22, 2025 at 13:49 UTC
2fa0e1396bf02c819f0b89762b8ca395eff4c757
1 file changed
+1
vllm/compilation/backends.py
+1
@@ -399,6 +399,7 @@ class VllmBackend:
399
rank = vllm_config.parallel_config.rank
400
dp_rank = vllm_config.parallel_config.data_parallel_rank
401
local_cache_dir = os.path.join(cache_dir, f"rank_{rank}_{dp_rank}")
402
+ os.makedirs(local_cache_dir, exist_ok=True)
403
self.compilation_config.local_cache_dir = local_cache_dir
404
405
disable_cache = envs.VLLM_DISABLE_COMPILE_CACHE