A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix unable to run encoder model when disable_hybrid_kv_cache_manager is true (#24571)
Signed-off-by: lianyibo <lianyibo1@kunlunit.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
lianyibo committed
Sep 17, 2025 at 01:36 UTC
faa7a5daac8244376a3a182d6eee8d2e0f6d8127
1 file changed
+4
vllm/v1/core/kv_cache_utils.py
+4
@@ -754,6 +754,10 @@ def is_kv_cache_type_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
754
True if all layers have the same type, False otherwise.
755
"""
756
757
+ if not kv_cache_spec:
758
+ # Encoder-only models do not have KV cache, kv_cache_type can be
759
+ # regarded as uniform.
760
+ return True
761
try:
762
kv_cache_spec_values = list(kv_cache_spec.values())
763
_ = kv_cache_spec_values[0].merge(kv_cache_spec_values)