A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 04:45 UTC
[compile] Nest inductor cache under AOT compile dir (#39718)
Signed-off-by: Alessandro Sangiorgi <asangior@redhat.com>
Alessandro Sangiorgi committed
Apr 14, 2026 at 12:17 UTC
2faad08362ff50f254de27cb3c54272b9f3af4b8
1 file changed
+10
vllm/compilation/decorators.py
+10
@@ -507,6 +507,16 @@ def _support_torch_compile(
507
hash_key,
508
)
509
510
+ # Hash-level dir; shared across ranks on the same node.
511
+ self.compilation_config.local_cache_dir = cache_dir
512
+ inductor_cache = os.path.join(cache_dir, "inductor_cache")
513
+ os.makedirs(inductor_cache, exist_ok=True)
514
+ # Process-wide: post-load execution, CUDA-graph capture, and later
515
+ # autotune/recompile all need to write under {hash}/inductor_cache/.
516
+ # Unconditional because torch's cache_dir() may have pre-filled the
517
+ # /tmp default during import, making setdefault a no-op.
518
+ os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
519
+
520
rank = self.vllm_config.parallel_config.rank
521
dp_rank = self.vllm_config.parallel_config.data_parallel_index
522
cache_dir = os.path.join(cache_dir, f"rank_{rank}_{dp_rank}")