A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI Failure] Fix fp8 kv cache on <SM90 (#25396)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Sep 22, 2025 at 14:27 UTC
239ef0c1ac0dfe68d8d2e28c54ecf9aa9bcd945b
1 file changed
+6
-2
vllm/platforms/cuda.py
+6
-2
@@ -286,6 +286,9 @@ class CudaPlatformBase(Platform):
286
TREE_ATTN_V1 = "vllm.v1.attention.backends.tree_attn.TreeAttentionBackend" # noqa: E501
287
XFORMERS_V1 = "vllm.v1.attention.backends.xformers.XFormersAttentionBackend" # noqa: E501
288
289
+ use_fp8_kv_cache = (kv_cache_dtype is not None
290
+ and kv_cache_dtype.startswith("fp8"))
291
+
292
if selected_backend == _Backend.FLASHINFER:
293
logger.info_once("Using FlashInfer backend on V1 engine.")
294
if cls.has_device_capability(100):
@@ -334,10 +337,11 @@ class CudaPlatformBase(Platform):
337
338
# FlashAttention is the default for SM 8.0+ GPUs
339
if cls.has_device_capability(80):
337
- if has_sink and not cls.is_device_capability(90):
340
+ if (has_sink or
341
+ use_fp8_kv_cache) and not cls.is_device_capability(90):
342
logger.info_once("Using Triton backend on V1 engine.")
343
return TRITON_ATTN_VLLM_V1
340
- if is_default_backend_supported := is_attn_backend_supported(
344
+ elif is_default_backend_supported := is_attn_backend_supported(
345
FLASH_ATTN_V1, head_size, dtype,
346
allow_import_error=False):
347
logger.info_once("Using Flash Attention backend on "