A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V1] Enable V1 for compute capability < 8.0 + FP32 (#23614)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Aug 26, 2025 at 18:00 UTC
50fede6634a997f4e971ecb4eb4cce337340e394
1 file changed
+8
-8
vllm/engine/arg_utils.py
+8
-8
@@ -1433,15 +1433,15 @@ class EngineArgs:
1433
recommend_to_remove=True)
1434
return False
1435
1436
- # Need at least Ampere for now (FA support required).
1437
- # Skip this check if we are running on a non-GPU platform,
1438
- # or if the device capability is not available
1439
- # (e.g. in a Ray actor without GPUs).
1436
+ # Triton v3.3 has f16 conversion regression issue on Turing and Volta,
1437
+ # which broke fp16 inference
1438
+ # see: https://github.com/triton-lang/triton/issues/6698
1439
if (current_platform.is_cuda()
1441
- and current_platform.get_device_capability()
1442
- and current_platform.get_device_capability().major < 8):
1443
- _raise_or_fallback(feature_name="Compute Capability < 8.0",
1444
- recommend_to_remove=False)
1440
+ and not current_platform.has_device_capability(80)
1441
+ and model_config.dtype == torch.float16):
1442
+ _raise_or_fallback(
1443
+ feature_name="Compute Capability < 8.0 with FP16",
1444
+ recommend_to_remove=False)
1445
return False
1446
1447
if self.kv_cache_dtype != "auto":