A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix num_heads value for simple connector when tp enabled (#12074)
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
shangmingc committed
Jan 20, 2025 at 10:56 UTC
df450aa5671bd3a48929686eb14d8a4324afd91a
1 file changed
+2
-1
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+2
-1
@@ -35,6 +35,7 @@ class SimpleConnector(KVConnectorBase):
35
):
36
37
self.config = config.kv_transfer_config
38
+ self.tp_size = config.parallel_config.tensor_parallel_size
39
40
if self.config.kv_connector == "PyNcclConnector":
41
from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import (
@@ -161,7 +162,7 @@ class SimpleConnector(KVConnectorBase):
162
end_layer = model_executable.model.end_layer
163
164
model_config = model_executable.model.config
164
- num_heads = model_config.num_key_value_heads
165
+ num_heads = int(model_config.num_key_value_heads / self.tp_size)
166
hidden_size = model_config.hidden_size
167
num_attention_heads = model_config.num_attention_heads
168
head_size = int(hidden_size / num_attention_heads)