@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de89472897
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix the issue for AquilaChat2-* models (#1339)

Lu Wang committed Oct 13, 2023 at 11:51 UTC de894728979411bc88ecc00ca6bb928c9a8f0231
3 files changed +10 -2
vllm/model_executor/model_loader.py
+1
@@ -14,6 +14,7 @@ from vllm.model_executor.weight_utils import (get_quant_config,
14 # TODO(woosuk): Lazy-load the model classes.
15 _MODEL_REGISTRY = {
16 "AquilaModel": AquilaForCausalLM,
17 + "AquilaForCausalLM": AquilaForCausalLM, # AquilaChat2
18 "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
19 "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
20 "BloomForCausalLM": BloomForCausalLM,
vllm/model_executor/models/aquila.py
+3 -2
@@ -147,6 +147,7 @@ class AquilaAttention(nn.Module):
147 rotary_dim=self.head_dim,
148 base=self.rope_theta,
149 max_position=self.max_position_embeddings,
150 + num_kv_heads=self.num_kv_heads,
151 )
152
153 def forward(
@@ -177,7 +178,7 @@ class AquilaDecoderLayer(nn.Module):
178 self.self_attn = AquilaAttention(
179 hidden_size=self.hidden_size,
180 num_heads=config.num_attention_heads,
180 - num_kv_heads=config.num_attention_heads,
181 + num_kv_heads=config.num_key_value_heads,
182 rope_theta=rope_theta,
183 max_position_embeddings=max_position_embeddings,
184 )
@@ -308,7 +309,7 @@ class AquilaForCausalLM(nn.Module):
309 q_proj_shard_size = (self.config.hidden_size // tp_size)
310 kv_proj_shard_size = (self.config.hidden_size //
311 self.config.num_attention_heads *
311 - self.config.num_attention_heads // tp_size)
312 + self.config.num_key_value_heads // tp_size)
313 attention_weight_specs = [
314 # (weight_name, shard_size, offset)
315 ("q_proj", q_proj_shard_size, 0),
vllm/transformers_utils/configs/aquila.py
+6
@@ -33,6 +33,7 @@ class AquilaConfig(PretrainedConfig):
33 intermediate_size=11008,
34 num_hidden_layers=32,
35 num_attention_heads=32,
36 + num_key_value_heads=None,
37 hidden_act="silu",
38 max_position_embeddings=2048,
39 initializer_range=0.006,
@@ -49,6 +50,11 @@ class AquilaConfig(PretrainedConfig):
50 self.hidden_size = hidden_size
51 self.intermediate_size = intermediate_size
52 self.num_hidden_layers = num_hidden_layers
53 + # for backward compatibility
54 + if num_key_value_heads is None:
55 + num_key_value_heads = num_attention_heads
56 +
57 + self.num_key_value_heads = num_key_value_heads
58 self.num_attention_heads = num_attention_heads
59 self.hidden_act = hidden_act
60 self.initializer_range = initializer_range