A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Enable FP8 KV cache for FlashInfer and Triton backend on non-sm100 GPUs (#24577)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Thien Tran committed
Sep 11, 2025 at 03:33 UTC
a0933c3bd67c1e210c7851690f22e79575f07d91
2 files changed
+9
-1
vllm/platforms/cuda.py
+4
@@ -530,6 +530,10 @@ class CudaPlatformBase(Platform):
530
supported = flash_attn_supports_fp8()
531
else:
532
supported = True
533
+ elif attention_backend == "FLASHINFER":
534
+ supported = True
535
+ elif attention_backend == "TRITON_ATTN_VLLM_V1":
536
+ supported = cls.supports_fp8()
537
return supported
538
539
@classmethod
vllm/v1/attention/backends/flashinfer.py
+5
-1
@@ -202,7 +202,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
202
else:
203
assert self.kv_cache_spec.dtype == self.model_config.dtype
204
self.kv_cache_dtype = self.kv_cache_spec.dtype
205
- self.q_data_type = self.kv_cache_dtype
205
+
206
+ if supports_trtllm_attention()[0]:
207
+ self.q_data_type = self.kv_cache_dtype
208
+ else:
209
+ self.q_data_type = self.model_config.dtype
210
211
self._cascade_wrapper = None # Wrapper for cascade attention
212