A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core] Make cudagraph check cuda platform only (#23005)
Signed-off-by: Chengji Yao <chengjiyao@gmail.com> Signed-off-by: Chengji Yao <chengjiyao@google.com> Co-authored-by: Chengji Yao <chengjiyao@gmail.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Chengji Yao committed
Aug 16, 2025 at 00:46 UTC
933f45334a79dcb69aa93178b3bbf3d9e0d46f09
1 file changed
+22
-17
vllm/config/__init__.py
+22
-17
@@ -3535,15 +3535,6 @@ class VllmConfig:
3535
# in V0 means the compilation level wins out.
3536
self.compilation_config.level = CompilationLevel.NO_COMPILATION
3537
3538
- # if cudagraph_mode is not explicitly set by users, set default value
3539
- if self.compilation_config.cudagraph_mode is None:
3540
- if envs.VLLM_USE_V1 and self.compilation_config.level \
3541
- == CompilationLevel.PIECEWISE:
3542
- self.compilation_config.cudagraph_mode = \
3543
- CUDAGraphMode.PIECEWISE
3544
- else:
3545
- self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
3546
-
3538
# async tp is built on top of sequence parallelism
3539
# and requires it to be enabled.
3540
if self.compilation_config.pass_config.enable_async_tp:
@@ -3552,14 +3543,28 @@ class VllmConfig:
3543
if self.compilation_config.pass_config.enable_sequence_parallelism:
3544
self.compilation_config.custom_ops.append("+rms_norm")
3545
3555
- # disable cudagraph when enforce eager execution
3556
- if self.model_config is not None and self.model_config.enforce_eager:
3557
- logger.info("Cudagraph is disabled under eager mode")
3558
- self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
3559
- elif envs.VLLM_USE_V1:
3560
- self.compilation_config.cudagraph_num_of_warmups = 1
3546
+ if current_platform.is_cuda_alike():
3547
+ # if cudagraph_mode is not explicitly set by users, set default
3548
+ # value
3549
+ if self.compilation_config.cudagraph_mode is None:
3550
+ if envs.VLLM_USE_V1 and self.compilation_config.level \
3551
+ == CompilationLevel.PIECEWISE:
3552
+ self.compilation_config.cudagraph_mode = \
3553
+ CUDAGraphMode.PIECEWISE
3554
+ else:
3555
+ self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
3556
3562
- self._set_cudagraph_sizes()
3557
+ # disable cudagraph when enforce eager execution
3558
+ if self.model_config is not None and \
3559
+ self.model_config.enforce_eager:
3560
+ logger.info("Cudagraph is disabled under eager mode")
3561
+ self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
3562
+ elif envs.VLLM_USE_V1:
3563
+ self.compilation_config.cudagraph_num_of_warmups = 1
3564
+
3565
+ self._set_cudagraph_sizes()
3566
+ else:
3567
+ self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
3568
3569
if self.cache_config.cpu_offload_gb > 0 and \
3570
self.compilation_config.level != CompilationLevel.NO_COMPILATION \
@@ -3618,7 +3623,7 @@ class VllmConfig:
3623
current_platform.check_and_update_config(self)
3624
3625
# final check of cudagraph mode after platform-specific update
3621
- if envs.VLLM_USE_V1:
3626
+ if envs.VLLM_USE_V1 and current_platform.is_cuda_alike():
3627
if self.compilation_config.cudagraph_mode == CUDAGraphMode.FULL \
3628
and self.model_config is not None and \
3629
not self.model_config.disable_cascade_attn: