A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
Fix KV Offloading + MLA AssertionError by using num_kv_heads=1 in cpu… (#37536)
Signed-off-by: xueliangyang-oeuler <yxl546827391@gmail.com> Co-authored-by: xueliangyang-oeuler <yxl546827391@gmail.com>
XueLiang Yang committed
Mar 19, 2026 at 20:05 UTC
e390742c5906df0edad8fc77b203e0623559ce79
1 file changed
+1
-1
vllm/v1/kv_offload/worker/cpu_gpu.py
+1
-1
@@ -240,7 +240,7 @@ class CpuGpuOffloadingHandlers:
240
gpu_shape = gpu_tensor.shape
241
attn_backend = attn_backends[layer_name]
242
test_shape = attn_backend.get_kv_cache_shape(
243
- num_blocks=1234, block_size=16, num_kv_heads=8, head_size=256
243
+ num_blocks=1234, block_size=16, num_kv_heads=1, head_size=256
244
)
245
246
has_layers_dim = False