A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
config check sleep mode support oot platforms (#16562)
Shuqiao Li committed
Apr 15, 2025 at 07:31 UTC
d2020acac748e5be1c4a37b06ab632b4e58e6606
2 files changed
+7
-2
vllm/config.py
+4
-2
@@ -417,8 +417,10 @@ class ModelConfig:
417
418
from vllm.platforms import current_platform
419
420
- if self.enable_sleep_mode and not current_platform.is_cuda():
421
- raise ValueError("Sleep mode is only supported on CUDA devices.")
420
+ if (self.enable_sleep_mode
421
+ and not current_platform.is_sleep_mode_available()):
422
+ raise ValueError(
423
+ "Sleep mode is not supported on current platform.")
424
425
hf_config = get_config(self.hf_config_path or self.model,
426
trust_remote_code, revision, code_revision,
vllm/platforms/interface.py
+3
@@ -148,6 +148,9 @@ class Platform:
148
"""Stateless version of :func:`torch.cuda.is_available`."""
149
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
150
151
+ def is_sleep_mode_available(self) -> bool:
152
+ return self._enum == PlatformEnum.CUDA
153
+
154
@classmethod
155
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
156
dtype: torch.dtype, kv_cache_dtype: Optional[str],