A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix the issue for AquilaChat2-* models (#1339)
Lu Wang committed
Oct 13, 2023 at 11:51 UTC
de894728979411bc88ecc00ca6bb928c9a8f0231
3 files changed
+10
-2
vllm/model_executor/model_loader.py
+1
@@ -14,6 +14,7 @@ from vllm.model_executor.weight_utils import (get_quant_config,
14
# TODO(woosuk): Lazy-load the model classes.
15
_MODEL_REGISTRY = {
16
"AquilaModel": AquilaForCausalLM,
17
+ "AquilaForCausalLM": AquilaForCausalLM, # AquilaChat2
18
"BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
19
"BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
20
"BloomForCausalLM": BloomForCausalLM,
vllm/model_executor/models/aquila.py
+3
-2
@@ -147,6 +147,7 @@ class AquilaAttention(nn.Module):
147
rotary_dim=self.head_dim,
148
base=self.rope_theta,
149
max_position=self.max_position_embeddings,
150
+ num_kv_heads=self.num_kv_heads,
151
)
152
153
def forward(
@@ -177,7 +178,7 @@ class AquilaDecoderLayer(nn.Module):
178
self.self_attn = AquilaAttention(
179
hidden_size=self.hidden_size,
180
num_heads=config.num_attention_heads,
180
- num_kv_heads=config.num_attention_heads,
181
+ num_kv_heads=config.num_key_value_heads,
182
rope_theta=rope_theta,
183
max_position_embeddings=max_position_embeddings,
184
)
@@ -308,7 +309,7 @@ class AquilaForCausalLM(nn.Module):
309
q_proj_shard_size = (self.config.hidden_size // tp_size)
310
kv_proj_shard_size = (self.config.hidden_size //
311
self.config.num_attention_heads *
311
- self.config.num_attention_heads // tp_size)
312
+ self.config.num_key_value_heads // tp_size)
313
attention_weight_specs = [
314
# (weight_name, shard_size, offset)
315
("q_proj", q_proj_shard_size, 0),
vllm/transformers_utils/configs/aquila.py
+6
@@ -33,6 +33,7 @@ class AquilaConfig(PretrainedConfig):
33
intermediate_size=11008,
34
num_hidden_layers=32,
35
num_attention_heads=32,
36
+ num_key_value_heads=None,
37
hidden_act="silu",
38
max_position_embeddings=2048,
39
initializer_range=0.006,
@@ -49,6 +50,11 @@ class AquilaConfig(PretrainedConfig):
50
self.hidden_size = hidden_size
51
self.intermediate_size = intermediate_size
52
self.num_hidden_layers = num_hidden_layers
53
+ # for backward compatibility
54
+ if num_key_value_heads is None:
55
+ num_key_value_heads = num_attention_heads
56
+
57
+ self.num_key_value_heads = num_key_value_heads
58
self.num_attention_heads = num_attention_heads
59
self.hidden_act = hidden_act
60
self.initializer_range = initializer_range