A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Stability fix] turn off HMA allocator when connector is set (#27592)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu> Signed-off-by: Kuntai Du <kuntai@uchicago.edu>
Kuntai Du committed
Oct 27, 2025 at 18:32 UTC
255e34ca50db3fc1f0ef8b66193b7c2fe47ca672
1 file changed
+14
vllm/config/vllm.py
+14
@@ -597,6 +597,20 @@ class VllmConfig:
597
if not current_platform.support_hybrid_kv_cache():
598
# Hybrid KV cache manager is not supported on non-GPU platforms.
599
self.scheduler_config.disable_hybrid_kv_cache_manager = True
600
+ if self.kv_transfer_config is not None:
601
+ # NOTE(Kuntai): turn HMA off for connector for now.
602
+ # TODO(Kuntai): have a more elegent solution to check and
603
+ # turn off HMA for connector that does not support HMA.
604
+ logger.warning(
605
+ "Turning off hybrid kv cache manager because "
606
+ "`--kv-transfer-config` is set. This will reduce the "
607
+ "performance of vLLM on LLMs with sliding window attention "
608
+ "or Mamba attention. If you are a developer of kv connector"
609
+ ", please consider supporting hybrid kv cache manager for "
610
+ "your connector by making sure your connector is a subclass"
611
+ " of `SupportsHMA` defined in kv_connector/v1/base.py."
612
+ )
613
+ self.scheduler_config.disable_hybrid_kv_cache_manager = True
614
if self.kv_events_config is not None:
615
# Hybrid KV cache manager is not compatible with KV events.
616
self.scheduler_config.disable_hybrid_kv_cache_manager = True