A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Bugfix] Fix cache block size calculation for CPU MLA (#15848)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Thien Tran committed
Apr 2, 2025 at 16:45 UTC
2edc87b161a17a71b00626800ad489ac8b563cb0
1 file changed
+1
-1
vllm/worker/cpu_worker.py
+1
-1
@@ -106,7 +106,7 @@ class CPUCacheEngine:
106
num_layers = model_config.get_num_layers(parallel_config)
107
108
key_cache_block = block_size * num_heads * head_size
109
- value_cache_block = key_cache_block
109
+ value_cache_block = key_cache_block if not model_config.use_mla else 0
110
total = num_layers * (key_cache_block + value_cache_block)
111
if cache_dtype == "auto":
112
dtype = model_config.dtype