A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 12:30 UTC
[Bugfix] Fix mamba cache dtype for Qwen3.5 (#34200)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Feb 10, 2026 at 13:12 UTC
9615575afc0d9a7d5fe98b65ac2a7150b068472e
1 file changed
+2
-1
vllm/model_executor/models/qwen3_5.py
+2
-1
@@ -870,8 +870,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
870
cls,
871
vllm_config: "VllmConfig",
872
) -> tuple[torch.dtype, torch.dtype]:
873
+ mamba_ssm_dtype = vllm_config.model_config.hf_text_config.mamba_ssm_dtype
874
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
874
- vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
875
+ vllm_config.model_config.dtype, mamba_ssm_dtype
876
)
877
878
@classmethod