A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[HybridKVCache][Platform] Add support_hybrid_kv_cache for platform (#24646)
Signed-off-by: MengqingCao <cmq0113@163.com>
Mengqing Cao committed
Sep 11, 2025 at 21:47 UTC
4f6593b058dc7ba66d887442ba5763c6c1b3886e
5 files changed
+20
-2
vllm/config/__init__.py
+1
-2
@@ -3529,8 +3529,7 @@ class VllmConfig:
3529
# logger should only print warning message for hybrid models. As we
3530
# can't know whether the model is hybrid or not now, so we don't log
3531
# warning message here and will log it later.
3532
- if not (current_platform.is_cuda() or current_platform.is_rocm()
3533
- or current_platform.is_cpu()):
3532
+ if not current_platform.support_hybrid_kv_cache():
3533
# Hybrid KV cache manager is not supported on non-GPU platforms.
3534
self.scheduler_config.disable_hybrid_kv_cache_manager = True
3535
if self.kv_transfer_config is not None:
vllm/platforms/cpu.py
+4
@@ -347,3 +347,7 @@ class CpuPlatform(Platform):
347
@classmethod
348
def opaque_attention_op(cls) -> bool:
349
return True
350
+
351
+ @classmethod
352
+ def support_hybrid_kv_cache(cls) -> bool:
353
+ return True
vllm/platforms/cuda.py
+4
@@ -571,6 +571,10 @@ class CudaPlatformBase(Platform):
571
"You can use float16 instead by explicitly setting the "
572
"`dtype` flag in CLI, for example: --dtype=half.")
573
574
+ @classmethod
575
+ def support_hybrid_kv_cache(cls) -> bool:
576
+ return True
577
+
578
579
# NVML utils
580
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
vllm/platforms/interface.py
+7
@@ -586,6 +586,13 @@ class Platform:
586
"""
587
raise NotImplementedError
588
589
+ @classmethod
590
+ def support_hybrid_kv_cache(cls) -> bool:
591
+ """
592
+ Returns if the hybrid kv cache is supported by the current platform.
593
+ """
594
+ return False
595
+
596
597
class UnspecifiedPlatform(Platform):
598
_enum = PlatformEnum.UNSPECIFIED
vllm/platforms/rocm.py
+4
@@ -498,3 +498,7 @@ class RocmPlatform(Platform):
498
f"Your {gpu_name} GPU {compute_str}. "
499
"You can use float16 instead by explicitly setting the "
500
"`dtype` flag in CLI, for example: --dtype=half.")
501
+
502
+ @classmethod
503
+ def support_hybrid_kv_cache(cls) -> bool:
504
+ return True