A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
refactor: abstract graph mode support into platform interface (#25161)
Signed-off-by: Yizhou Liu <liu_yizhou@outlook.com>
Yizhou committed
Sep 22, 2025 at 18:22 UTC
b6f01bd9a7941c3c06175feaf7588fa55c9db646
5 files changed
+23
-7
vllm/config/__init__.py
+1
-1
@@ -503,7 +503,7 @@ class VllmConfig:
503
if self.compilation_config.pass_config.enable_sequence_parallelism:
504
self.compilation_config.custom_ops.append("+rms_norm")
505
506
- if current_platform.is_cuda_alike() or current_platform.is_xpu():
506
+ if current_platform.support_static_graph_mode():
507
# if cudagraph_mode is not explicitly set by users, set default
508
# value
509
if self.compilation_config.cudagraph_mode is None:
vllm/platforms/cuda.py
+4
@@ -498,6 +498,10 @@ class CudaPlatformBase(Platform):
498
def support_hybrid_kv_cache(cls) -> bool:
499
return True
500
501
+ @classmethod
502
+ def support_static_graph_mode(cls) -> bool:
503
+ return True
504
+
505
506
# NVML utils
507
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/interface.py
+7
@@ -587,6 +587,13 @@ class Platform:
587
"""
588
return False
589
590
+ @classmethod
591
+ def support_static_graph_mode(cls) -> bool:
592
+ """
593
+ Returns if the graph mode is supported by the current platform.
594
+ """
595
+ return False
596
+
597
@classmethod
598
def use_sync_weight_loader(cls) -> bool:
599
"""
vllm/platforms/rocm.py
+4
@@ -477,3 +477,7 @@ class RocmPlatform(Platform):
477
@classmethod
478
def support_hybrid_kv_cache(cls) -> bool:
479
return True
480
+
481
+ @classmethod
482
+ def support_static_graph_mode(cls) -> bool:
483
+ return True
vllm/platforms/xpu.py
+7
-6
@@ -113,12 +113,9 @@ class XPUPlatform(Platform):
113
# lazy import to avoid circular import
114
from vllm.config import CompilationLevel, CUDAGraphMode
115
compilation_config = vllm_config.compilation_config
116
- if compilation_config.cudagraph_mode is None or \
117
- compilation_config.cudagraph_mode.max_cudagraph_mode() \
118
- != CUDAGraphMode.NONE:
119
- logger.info("[XPU] CUDA graph is not supported on XPU, disabling "
120
- "cudagraphs. Fallback to cudagraph_mode=NONE")
121
- compilation_config.cudagraph_mode = CUDAGraphMode.NONE
116
+
117
+ assert compilation_config.cudagraph_mode == CUDAGraphMode.NONE, \
118
+ "CUDA graph mode should be NONE on XPU"
119
120
if vllm_config.lora_config is not None:
121
compilation_config.level = CompilationLevel.NO_COMPILATION
@@ -169,6 +166,10 @@ class XPUPlatform(Platform):
166
def support_hybrid_kv_cache(cls) -> bool:
167
return True
168
169
+ @classmethod
170
+ def support_static_graph_mode(cls) -> bool:
171
+ return False
172
+
173
@classmethod
174
def is_pin_memory_available(cls):
175
return True