A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix internlm after https://github.com/vllm-project/vllm/pull/2860 (#2861)
Philipp Moritz committed
Feb 13, 2024 at 18:01 UTC
0c48b37c310254e83cd2906230e87af97cb148ba
1 file changed
+2
-1
vllm/model_executor/models/llama.py
+2
-1
@@ -175,7 +175,8 @@ class LlamaDecoderLayer(nn.Module):
175
self.self_attn = LlamaAttention(
176
hidden_size=self.hidden_size,
177
num_heads=config.num_attention_heads,
178
- num_kv_heads=config.num_key_value_heads,
178
+ num_kv_heads=getattr(config, "num_key_value_heads",
179
+ config.num_attention_heads),
180
rope_theta=rope_theta,
181
rope_scaling=rope_scaling,
182
max_position_embeddings=max_position_embeddings,