A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 20:15 UTC
[Bugfix] Fix DSV3.2 NVFP4 (#33932)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Matthew Bonanni committed
Feb 5, 2026 at 14:22 UTC
4145e50d854e3182c22bad99ec011c283b9c493f
1 file changed
+4
-2
vllm/model_executor/layers/attention/mla_attention.py
+4
-2
@@ -530,7 +530,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
530
scale=self._k_scale,
531
)
532
533
- if fp8_attention:
533
+ if fp8_attention and self.kv_cache_dtype != "fp8_ds_mla":
534
kv_cache = kv_cache.view(current_platform.fp8_dtype())
535
536
# Sparse MLA impls only support forward_mqa (decode-style attention)
@@ -614,7 +614,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
614
# Convert from (N, B, L) to (B, N, L)
615
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
616
617
- if fp8_attention:
617
+ if fp8_attention and self.impl.supports_quant_query_input:
618
assert mqa_ql_nope.shape[0] == mqa_q_pe.shape[0]
619
assert mqa_ql_nope.shape[1] == mqa_q_pe.shape[1]
620
mqa_q = self._decode_concat_quant_fp8_op(
@@ -1885,6 +1885,8 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
1885
self.indexer = indexer
1886
self.q_pad_num_heads = q_pad_num_heads
1887
1888
+ self.supports_quant_query_input = True
1889
+
1890
# Use flashinfer's optimized concat_mla_k kernel when available.
1891
# The kernel is optimized for DeepSeek V3 dimensions:
1892
# num_heads=128, nope_dim=128, rope_dim=64