A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 19:15 UTC
fix(kv-cache): increase hybrid attention grouping threshold from 1.25 to 1.5 (#36684)
Signed-off-by: Jaime Campos Salas <jaime.campos.salas@gmail.com>
jaime campos salas committed
Mar 12, 2026 at 23:28 UTC
891c60dcd51b3a90704e76c90bf6d7796ec6f3c2
1 file changed
+8
-6
vllm/v1/core/kv_cache_utils.py
+8
-6
@@ -1040,12 +1040,14 @@ def _get_kv_cache_groups_uniform_page_size(
1040
min_num_layers = min([len(layers) for layers in same_type_layers.values()])
1041
group_size = min_num_layers
1042
max_num_layers = max([len(layers) for layers in same_type_layers.values()])
1043
- if max_num_layers < min_num_layers * 1.25:
1044
- # If the number of layers is not much larger than the minimum number of layers,
1045
- # use the maximum number of layers as the group size to avoid too many padding
1046
- # layers. A typical example is gpt-oss-20b + eagle, with 12 sw + 13 full. We
1047
- # pad it to (13 sw, 13 full) instead of (12 sw, 24 full). 1.25 is just a
1048
- # magic number to avoid too many padding layers.
1043
+ if max_num_layers < min_num_layers * 1.5:
1044
+ # If the number of layers is not much larger than the minimum number of
1045
+ # layers, use the maximum number of layers as the group size to avoid
1046
+ # too many padding layers. A typical example is gpt-oss-20b + eagle,
1047
+ # with 12 sw + 13 full. We pad it to (13 sw, 13 full) instead of
1048
+ # (12 sw, 24 full). 1.5 is a heuristic to avoid too many padding
1049
+ # layers while accommodating speculative decoding drafters that add
1050
+ # extra layers to one attention type.
1051
group_size = max_num_layers
1052
grouped_layers = []
1053
for layers in same_type_layers.values():