A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix][CUDA] fixes CUDA FP8 kv cache dtype supported (#21420)
Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
elvischenv committed
Jul 23, 2025 at 11:34 UTC
2dec7c1a5df93b125199206f8ecc4975de82e504
1 file changed
+13
-13
vllm/platforms/cuda.py
+13
-13
@@ -456,6 +456,19 @@ class CudaPlatformBase(Platform):
456
def device_count(cls) -> int:
457
return cuda_device_count_stateless()
458
459
+ @classmethod
460
+ def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str) -> bool:
461
+ fp8_attention = kv_cache_dtype.startswith("fp8")
462
+ will_use_fa = (not envs.is_set("VLLM_ATTENTION_BACKEND")
463
+ ) or envs.VLLM_ATTENTION_BACKEND == "FLASH_ATTN_VLLM_V1"
464
+ supported = False
465
+ if cls.is_device_capability(100):
466
+ supported = True
467
+ elif fp8_attention and will_use_fa:
468
+ from vllm.attention.utils.fa_utils import flash_attn_supports_fp8
469
+ supported = flash_attn_supports_fp8()
470
+ return supported
471
+
472
473
# NVML utils
474
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
@@ -583,19 +596,6 @@ class NonNvmlCudaPlatform(CudaPlatformBase):
596
" not found. Assuming no NVLink available.")
597
return False
598
586
- @classmethod
587
- def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str) -> bool:
588
- fp8_attention = kv_cache_dtype.startswith("fp8")
589
- will_use_fa = (not envs.is_set("VLLM_ATTENTION_BACKEND")
590
- ) or envs.VLLM_ATTENTION_BACKEND == "FLASH_ATTN_VLLM_V1"
591
- supported = False
592
- if cls.is_device_capability(100):
593
- supported = True
594
- elif fp8_attention and will_use_fa:
595
- from vllm.attention.utils.fa_utils import flash_attn_supports_fp8
596
- supported = flash_attn_supports_fp8()
597
- return supported
598
-
599
600
# Autodetect either NVML-enabled or non-NVML platform
601
# based on whether NVML is available.