@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df450aa567
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix num_heads value for simple connector when tp enabled (#12074)

Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>

shangmingc committed Jan 20, 2025 at 10:56 UTC df450aa5671bd3a48929686eb14d8a4324afd91a
1 file changed +2 -1
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+2 -1
@@ -35,6 +35,7 @@ class SimpleConnector(KVConnectorBase):
35 ):
36
37 self.config = config.kv_transfer_config
38 + self.tp_size = config.parallel_config.tensor_parallel_size
39
40 if self.config.kv_connector == "PyNcclConnector":
41 from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import (
@@ -161,7 +162,7 @@ class SimpleConnector(KVConnectorBase):
162 end_layer = model_executable.model.end_layer
163
164 model_config = model_executable.model.config
164 - num_heads = model_config.num_key_value_heads
165 + num_heads = int(model_config.num_key_value_heads / self.tp_size)
166 hidden_size = model_config.hidden_size
167 num_attention_heads = model_config.num_attention_heads
168 head_size = int(hidden_size / num_attention_heads)