A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 04:45 UTC
[Bugfix] flashinfer: fail fast when --kv-cache-dtype nvfp4 used on unsupported arch (#43669)
Signed-off-by: Kartavya Sonar <sonarkartavya@gmail.com>
Kartavya sonar committed
Jun 2, 2026 at 23:20 UTC
fe32e7830b2f3e799a571373b25fc4563fb05d19
1 file changed
+7
vllm/v1/attention/backends/flashinfer.py
+7
@@ -623,6 +623,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
623
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
624
self.is_kvcache_nvfp4 = self.cache_dtype == "nvfp4"
625
if self.is_kvcache_nvfp4:
626
+ # trtllm-gen FP4 FMHA kernels only exist for sm100f (sm_100/sm_103).
627
+ # Fail fast at init rather than crashing on the first request.
628
+ if not current_platform.is_device_capability_family(100):
629
+ raise ValueError(
630
+ "--kv-cache-dtype nvfp4 requires sm100f, "
631
+ "please try a different dtype or remove"
632
+ )
633
# For NVFP4, kv_cache_dtype stays as the string "nvfp4"
634
# which is passed to FlashInferImpl
635
self.kv_cache_dtype = self.cache_dtype