A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Switch bailout logic for kv-cache-dtype with SM100 Flashinfer (#20934)
Signed-off-by: Pavani Majety <pmajety@nvidia.com>
Pavani Majety committed
Jul 14, 2025 at 20:27 UTC
9ad0a4588ba4e9c979cda0d178dec4fcdb89fd0c
1 file changed
+4
-3
vllm/engine/arg_utils.py
+4
-3
@@ -1418,14 +1418,15 @@ class EngineArgs:
1418
and not envs.is_set("VLLM_ATTENTION_BACKEND")
1419
) or envs.VLLM_ATTENTION_BACKEND == "FLASH_ATTN_VLLM_V1"
1420
supported = False
1421
- if current_platform.is_rocm():
1421
+ if current_platform.is_rocm() or (
1422
+ current_platform.is_cuda()
1423
+ and current_platform.is_device_capability(100)):
1424
supported = True
1425
elif fp8_attention and will_use_fa:
1426
from vllm.attention.utils.fa_utils import (
1427
flash_attn_supports_fp8)
1428
supported = flash_attn_supports_fp8()
1427
- elif envs.VLLM_USE_TRTLLM_DECODE_ATTENTION:
1428
- supported = True
1429
+
1430
if not supported:
1431
_raise_or_fallback(feature_name="--kv-cache-dtype",
1432
recommend_to_remove=False)