A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Disable the torch.compile cache checks when VLLM_DISABLE_COMPILE_CACHE=1 (#16573)
Signed-off-by: Lu Fang <lufang@fb.com>
Lu Fang committed
Apr 26, 2025 at 09:17 UTC
4d17e20310a256b42409c1b04166a60316a2f075
1 file changed
+9
-4
vllm/compilation/compiler_interface.py
+9
-4
@@ -11,6 +11,7 @@ import torch
11
import torch._inductor.compile_fx
12
import torch.fx as fx
13
14
+import vllm.envs as envs
15
from vllm.config import VllmConfig
16
from vllm.utils import is_torch_equal_or_newer
17
@@ -317,10 +318,14 @@ class InductorAdaptor(CompilerInterface):
318
inner_compile=hijacked_compile_fx_inner,
319
config_patches=current_config)
320
320
- assert hash_str is not None, (
321
- "failed to get the hash of the compiled graph")
322
- assert file_path is not None, (
323
- "failed to get the file path of the compiled graph")
321
+ # We treat VLLM_DISABLE_COMPILE_CACHE as the overall switch for torch
322
+ # compilation cache. So turn off the checks if we disable the
323
+ # compilation cache.
324
+ if not envs.VLLM_DISABLE_COMPILE_CACHE:
325
+ assert hash_str is not None, (
326
+ "failed to get the hash of the compiled graph")
327
+ assert file_path is not None, (
328
+ "failed to get the file path of the compiled graph")
329
return compiled_graph, (hash_str, file_path)
330
331
def load(self,