A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 13:33 UTC
[Bugfix] Fix DeepseekV32 `AssertionError: num_kv_heads == 1` (#33090)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jan 27, 2026 at 16:03 UTC
492a7983dd71fe0869aa5f77dcd050db1e37e31c
1 file changed
+1
-1
vllm/distributed/kv_transfer/kv_connector/utils.py
+1
-1
@@ -322,7 +322,7 @@ class TpKVTopology:
322
# Figure out whether the first dimension of the cache is K/V
323
# or num_blocks. This is used to register the memory regions correctly.
324
kv_cache_shape = self.attn_backend.get_kv_cache_shape(
325
- num_blocks=1, block_size=16, num_kv_heads=4, head_size=1
325
+ num_blocks=1, block_size=16, num_kv_heads=1, head_size=1
326
)
327
# Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D],
328
# we just mock num_blocks to 1 for the dimension check below.