A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
fix qwen-14b model (#1173)
Qing committed
Sep 28, 2023 at 07:33 UTC
28e616c4e3ef24d3763de5c5210f2ee20be56f5e
2 files changed
+32
-43
vllm/model_executor/models/qwen.py
+7
-7
@@ -141,17 +141,17 @@ class QWenBlock(nn.Module):
141
142
def __init__(self, config: QWenConfig):
143
super().__init__()
144
- self.ln_1 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
144
+ self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
145
146
rope_theta = getattr(config, "rope_theta", 10000)
147
- self.attn = QWenAttention(config.n_embd,
147
+ self.attn = QWenAttention(config.hidden_size,
148
config.num_attention_heads,
149
config.max_position_embeddings,
150
rope_theta=rope_theta)
151
152
- self.ln_2 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
152
+ self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
153
154
- self.mlp = QWenMLP(config.n_embd, config.ffn_hidden_size // 2)
154
+ self.mlp = QWenMLP(config.hidden_size, config.intermediate_size // 2)
155
156
def forward(
157
self,
@@ -190,11 +190,11 @@ class QWenModel(nn.Module):
190
191
vocab_size = ((config.vocab_size + 63) // 64) * 64
192
self.wte = VocabParallelEmbedding(vocab_size,
193
- config.n_embd,
193
+ config.hidden_size,
194
perform_initialization=False)
195
self.h = nn.ModuleList(
196
[QWenBlock(config) for _ in range(config.num_hidden_layers)])
197
- self.ln_f = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
197
+ self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
198
199
def forward(
200
self,
@@ -230,7 +230,7 @@ class QWenLMHeadModel(nn.Module):
230
self.transformer = QWenModel(config)
231
vocab_size = ((config.vocab_size + 63) // 64) * 64
232
self.lm_head = ColumnParallelLinear(
233
- config.n_embd,
233
+ config.hidden_size,
234
vocab_size,
235
bias=False,
236
gather_output=False,
vllm/transformers_utils/configs/qwen.py
+25
-36
@@ -7,65 +7,54 @@ from transformers import PretrainedConfig
7
class QWenConfig(PretrainedConfig):
8
model_type = "qwen"
9
keys_to_ignore_at_inference = ["past_key_values"]
10
- attribute_map = {
11
- "hidden_size": "n_embd",
12
- "num_attention_heads": "n_head",
13
- "max_position_embeddings": "n_positions",
14
- "num_hidden_layers": "n_layer",
15
- }
10
11
def __init__(
12
self,
19
- vocab_size=151851,
20
- n_embd=4096,
21
- n_layer=32,
22
- n_head=32,
23
- n_inner=None,
24
- embd_pdrop=0.0,
25
- attn_pdrop=0.0,
26
- layer_norm_epsilon=1e-5,
13
+ vocab_size=151936,
14
+ hidden_size=4096,
15
+ num_hidden_layers=32,
16
+ num_attention_heads=32,
17
+ emb_dropout_prob=0.0,
18
+ attn_dropout_prob=0.0,
19
+ layer_norm_epsilon=1e-6,
20
initializer_range=0.02,
21
+ max_position_embeddings=8192,
22
scale_attn_weights=True,
23
use_cache=True,
30
- eos_token_id=151643,
31
- apply_residual_connection_post_layernorm=False,
32
- bf16=True,
24
+ bf16=False,
25
+ fp16=False,
26
+ fp32=False,
27
kv_channels=128,
28
rotary_pct=1.0,
29
rotary_emb_base=10000,
36
- use_dynamic_ntk=False,
37
- use_logn_attn=False,
38
- use_flash_attn=True,
39
- ffn_hidden_size=22016,
30
+ use_dynamic_ntk=True,
31
+ use_logn_attn=True,
32
+ use_flash_attn="auto",
33
+ intermediate_size=22016,
34
no_bias=True,
35
tie_word_embeddings=False,
36
**kwargs,
37
):
44
- self.eos_token_id = eos_token_id
45
- super().__init__(eos_token_id=eos_token_id,
46
- tie_word_embeddings=tie_word_embeddings,
47
- **kwargs)
48
-
38
self.vocab_size = vocab_size
50
- self.n_embd = n_embd
51
- self.n_layer = n_layer
52
- self.n_head = n_head
53
- self.n_inner = n_inner
54
- self.embd_pdrop = embd_pdrop
55
- self.attn_pdrop = attn_pdrop
39
+ self.hidden_size = hidden_size
40
+ self.intermediate_size = intermediate_size
41
+ self.num_hidden_layers = num_hidden_layers
42
+ self.num_attention_heads = num_attention_heads
43
+ self.emb_dropout_prob = emb_dropout_prob
44
+ self.attn_dropout_prob = attn_dropout_prob
45
self.layer_norm_epsilon = layer_norm_epsilon
46
self.initializer_range = initializer_range
47
self.scale_attn_weights = scale_attn_weights
48
self.use_cache = use_cache
60
- self.apply_residual_connection_post_layernorm = (
61
- apply_residual_connection_post_layernorm)
49
+ self.max_position_embeddings = max_position_embeddings
50
self.bf16 = bf16
51
+ self.fp16 = fp16
52
+ self.fp32 = fp32
53
self.kv_channels = kv_channels
54
self.rotary_pct = rotary_pct
55
self.rotary_emb_base = rotary_emb_base
56
self.use_dynamic_ntk = use_dynamic_ntk
57
self.use_logn_attn = use_logn_attn
58
self.use_flash_attn = use_flash_attn
69
- self.ffn_hidden_size = ffn_hidden_size
59
self.no_bias = no_bias
71
- self.tie_word_embeddings = tie_word_embeddings
60
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)