A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 07:00 UTC
[Bugfix] Disable --calculate-kv-scales for hybrid GDN/Mamba+Attention… (#37565)
Signed-off-by: Young-Leo <562593859@qq.com>
Le Yang committed
Mar 21, 2026 at 02:28 UTC
d7d2b5e405a24f716371cc7f9b488b14300b0991
1 file changed
+17
-1
vllm/model_executor/models/config.py
+17
-1
@@ -113,8 +113,24 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig):
113
Args:
114
vllm_config: vLLM Config
115
"""
116
+ cache_config = vllm_config.cache_config
117
+
118
+ # Disable calculate_kv_scales for hybrid models: uninitialized
119
+ # recurrent state corrupts scales during the calibration pass.
120
+ # See issue: https://github.com/vllm-project/vllm/issues/37554
121
+ if cache_config.calculate_kv_scales:
122
+ logger.warning(
123
+ "Disabling calculate_kv_scales for hybrid model '%s'. "
124
+ "Hybrid models with recurrent layers (GDN, Mamba, SSM) "
125
+ "produce unreliable KV cache scales during the "
126
+ "calibration pass because recurrent state is "
127
+ "uninitialized. Using default scale of 1.0 instead.",
128
+ vllm_config.model_config.model,
129
+ )
130
+ cache_config.calculate_kv_scales = False
131
+
132
# Save the user input before it gets modified by MambaModelConfig
117
- mamba_block_size = vllm_config.cache_config.mamba_block_size
133
+ mamba_block_size = cache_config.mamba_block_size
134
# Enable FULL_AND_PIECEWISE by default
135
MambaModelConfig.verify_and_update_config(vllm_config)
136