A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix value unpack error of simple connector for KVCache transfer. (#11058)
Signed-off-by: ShangmingCai <csmthu@gmail.com>
shangmingc committed
Dec 13, 2024 at 05:19 UTC
db6c264a1e658e37782570f5155c77be0d41f312
1 file changed
+6
-2
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+6
-2
@@ -118,6 +118,12 @@ class SimpleConnector(KVConnectorBase):
118
start_layer = model_executable.model.start_layer
119
end_layer = model_executable.model.end_layer
120
121
+ model_config = model_executable.model.config
122
+ num_heads = model_config.num_key_value_heads
123
+ hidden_size = model_config.hidden_size
124
+ num_attention_heads = model_config.num_attention_heads
125
+ head_size = int(hidden_size / num_attention_heads)
126
+
127
# query_lens contains new KV caches that are added to vLLM.
128
# so we will send them to decode instance
129
# FIXME(Kuntai): This assume that all requests are prefill.
@@ -131,8 +137,6 @@ class SimpleConnector(KVConnectorBase):
137
for layer_id in range(start_layer, end_layer):
138
kv_cache = kv_caches[layer_id - start_layer]
139
134
- _, _, num_heads, head_size = kv_cache[0].shape
135
-
140
key_cache = kv_cache[0].reshape(-1, num_heads, head_size)
141
value_cache = kv_cache[1].reshape(-1, num_heads, head_size)
142