A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 20:45 UTC
[Bugfix] change FlashMLA reorder_batch_threshold (#27777)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Matthew Bonanni committed
Nov 3, 2025 at 15:17 UTC
145c00a4d32b7a681f7fb936c9575812c7aa7880
1 file changed
+1
-1
vllm/v1/attention/backends/mla/flashmla.py
+1
-1
@@ -71,7 +71,7 @@ class FlashMLAMetadata(MLACommonMetadata[FlashMLADecodeMetadata]):
71
class FlashMLAMetadataBuilder(MLACommonMetadataBuilder[FlashMLAMetadata]):
72
cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
73
query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM
74
- reorder_batch_threshold: int = 512 # process small prefills with decode pathway
74
+ reorder_batch_threshold: int = 128 # process small prefills with decode pathway
75
# ^ TODO(matt): tune this
76
77
def __init__(