@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db6c264a1e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix value unpack error of simple connector for KVCache transfer. (#11058)

Signed-off-by: ShangmingCai <csmthu@gmail.com>

shangmingc committed Dec 13, 2024 at 05:19 UTC db6c264a1e658e37782570f5155c77be0d41f312
1 file changed +6 -2
vllm/distributed/kv_transfer/kv_connector/simple_connector.py
+6 -2
@@ -118,6 +118,12 @@ class SimpleConnector(KVConnectorBase):
118 start_layer = model_executable.model.start_layer
119 end_layer = model_executable.model.end_layer
120
121 + model_config = model_executable.model.config
122 + num_heads = model_config.num_key_value_heads
123 + hidden_size = model_config.hidden_size
124 + num_attention_heads = model_config.num_attention_heads
125 + head_size = int(hidden_size / num_attention_heads)
126 +
127 # query_lens contains new KV caches that are added to vLLM.
128 # so we will send them to decode instance
129 # FIXME(Kuntai): This assume that all requests are prefill.
@@ -131,8 +137,6 @@ class SimpleConnector(KVConnectorBase):
137 for layer_id in range(start_layer, end_layer):
138 kv_cache = kv_caches[layer_id - start_layer]
139
134 - _, _, num_heads, head_size = kv_cache[0].shape
135 -
140 key_cache = kv_cache[0].reshape(-1, num_heads, head_size)
141 value_cache = kv_cache[1].reshape(-1, num_heads, head_size)
142