A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
add QWen-7b (#685)
Co-authored-by: wq.chu <wq.chu@tianrang-inc.com>
Qing committed
Aug 9, 2023 at 04:50 UTC
a57d13cc967d04478cebcfa2c3440fb167b21c6d
6 files changed
+396
-11
vllm/model_executor/model_loader.py
+1
@@ -23,6 +23,7 @@ _MODEL_REGISTRY = {
23
"LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
24
"MPTForCausalLM": MPTForCausalLM,
25
"OPTForCausalLM": OPTForCausalLM,
26
+ "QWenLMHeadModel": QWenLMHeadModel,
27
"RWForCausalLM": FalconForCausalLM,
28
}
29
vllm/model_executor/models/__init__.py
+5
-11
@@ -9,17 +9,11 @@ from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
9
from vllm.model_executor.models.llama import LlamaForCausalLM
10
from vllm.model_executor.models.mpt import MPTForCausalLM
11
from vllm.model_executor.models.opt import OPTForCausalLM
12
+from vllm.model_executor.models.qwen import QWenLMHeadModel
13
14
__all__ = [
14
- "BaiChuanForCausalLM",
15
- "BaichuanForCausalLM",
16
- "BloomForCausalLM",
17
- "FalconForCausalLM",
18
- "GPT2LMHeadModel",
19
- "GPTBigCodeForCausalLM",
20
- "GPTJForCausalLM",
21
- "GPTNeoXForCausalLM",
22
- "LlamaForCausalLM",
23
- "MPTForCausalLM",
24
- "OPTForCausalLM",
15
+ "BaiChuanForCausalLM", "BaichuanForCausalLM", "BloomForCausalLM",
16
+ "FalconForCausalLM", "GPT2LMHeadModel", "GPTBigCodeForCausalLM",
17
+ "GPTJForCausalLM", "GPTNeoXForCausalLM", "LlamaForCausalLM",
18
+ "MPTForCausalLM", "OPTForCausalLM", "QWenLMHeadModel"
19
]
vllm/model_executor/models/qwen.py
new
+316
@@ -0,0 +1,316 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
4
+# Copyright (c) Alibaba Cloud.
5
+# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6
+"""Inference-only QWen model compatible with HuggingFace weights.
7
+
8
+The input of the model is flattened to a 1D tensor of tokens. The model uses
9
+InputMetadata to extract the original 2D shape of the input.
10
+"""
11
+from typing import Dict, List, Optional, Tuple
12
+
13
+import torch
14
+from torch import nn
15
+
16
+from vllm.model_executor.input_metadata import InputMetadata
17
+from vllm.model_executor.layers.activation import SiluAndMul
18
+from vllm.model_executor.layers.layernorm import RMSNorm
19
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
20
+from vllm.model_executor.layers.sampler import Sampler
21
+from vllm.model_executor.weight_utils import (
22
+ hf_model_weights_iterator,
23
+ load_tensor_parallel_weights,
24
+)
25
+from vllm.model_executor.parallel_utils.parallel_state import (
26
+ get_tensor_model_parallel_rank,
27
+ get_tensor_model_parallel_world_size,
28
+)
29
+from vllm.model_executor.parallel_utils.tensor_parallel import (
30
+ VocabParallelEmbedding,
31
+ ColumnParallelLinear,
32
+ RowParallelLinear,
33
+)
34
+from vllm.sequence import SequenceOutputs
35
+from vllm.transformers_utils.configs.qwen import QWenConfig
36
+
37
+KVCache = Tuple[torch.Tensor, torch.Tensor]
38
+
39
+
40
+class QWenMLP(nn.Module):
41
+
42
+ def __init__(
43
+ self,
44
+ hidden_size: int,
45
+ intermediate_size: int,
46
+ hidden_act: str = "silu",
47
+ ):
48
+ super().__init__()
49
+ self.gate_up_proj = ColumnParallelLinear(
50
+ hidden_size,
51
+ 2 * intermediate_size,
52
+ bias=False,
53
+ gather_output=False,
54
+ perform_initialization=False,
55
+ )
56
+ self.c_proj = RowParallelLinear(
57
+ intermediate_size,
58
+ hidden_size,
59
+ bias=False,
60
+ input_is_parallel=True,
61
+ perform_initialization=False,
62
+ )
63
+ if hidden_act != "silu":
64
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
65
+ "Only silu is supported for now.")
66
+ self.act_fn = SiluAndMul()
67
+
68
+ def forward(self, x):
69
+ gate_up, _ = self.gate_up_proj(x)
70
+ x = self.act_fn(gate_up)
71
+ x, _ = self.c_proj(x)
72
+ return x
73
+
74
+
75
+class QWenAttention(nn.Module):
76
+
77
+ def __init__(self, hidden_size: int, num_heads: int,
78
+ max_position_embeddings: int):
79
+ super().__init__()
80
+ self.hidden_size = hidden_size
81
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
82
+ )
83
+ self.total_num_heads = num_heads
84
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
85
+ self.num_heads = (self.total_num_heads //
86
+ tensor_model_parallel_world_size)
87
+ self.head_dim = hidden_size // self.total_num_heads
88
+
89
+ # pylint: disable=invalid-name
90
+ self.c_attn = ColumnParallelLinear(
91
+ hidden_size,
92
+ 3 * hidden_size,
93
+ bias=True,
94
+ gather_output=False,
95
+ perform_initialization=False,
96
+ )
97
+ self.c_proj = RowParallelLinear(
98
+ self.total_num_heads * self.head_dim,
99
+ hidden_size,
100
+ bias=False,
101
+ input_is_parallel=True,
102
+ perform_initialization=False,
103
+ )
104
+ self.scaling = self.head_dim**-0.5
105
+ self.attn = PagedAttentionWithRoPE(
106
+ self.num_heads,
107
+ self.head_dim,
108
+ self.scaling,
109
+ rotary_dim=self.head_dim,
110
+ max_position=max_position_embeddings,
111
+ )
112
+
113
+ def forward(
114
+ self,
115
+ positions: torch.Tensor,
116
+ hidden_states: torch.Tensor,
117
+ kv_cache: KVCache,
118
+ input_metadata: InputMetadata,
119
+ cache_event: Optional[torch.cuda.Event],
120
+ ) -> torch.Tensor:
121
+ qkv, _ = self.c_attn(hidden_states)
122
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
123
+
124
+ k_cache, v_cache = kv_cache
125
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
126
+ input_metadata, cache_event)
127
+
128
+ output, _ = self.c_proj(attn_output)
129
+ return output
130
+
131
+
132
+class QWenBlock(nn.Module):
133
+
134
+ def __init__(self, config: QWenConfig):
135
+ super().__init__()
136
+ self.ln_1 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
137
+
138
+ self.attn = QWenAttention(config.n_embd, config.num_attention_heads,
139
+ config.max_position_embeddings)
140
+
141
+ self.ln_2 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
142
+
143
+ self.mlp = QWenMLP(config.n_embd, config.ffn_hidden_size // 2)
144
+
145
+ def forward(
146
+ self,
147
+ positions: torch.Tensor,
148
+ hidden_states: torch.Tensor,
149
+ kv_cache: KVCache,
150
+ input_metadata: InputMetadata,
151
+ cache_event: Optional[torch.cuda.Event],
152
+ ) -> torch.Tensor:
153
+ # Self Attention
154
+ residual = hidden_states
155
+ hidden_states = self.ln_1(hidden_states)
156
+ hidden_states = self.attn(
157
+ positions=positions,
158
+ hidden_states=hidden_states,
159
+ kv_cache=kv_cache,
160
+ input_metadata=input_metadata,
161
+ cache_event=cache_event,
162
+ )
163
+ hidden_states = residual + hidden_states
164
+
165
+ # Fully Connected
166
+ residual = hidden_states
167
+ hidden_states = self.ln_2(hidden_states)
168
+ hidden_states = self.mlp(hidden_states)
169
+ hidden_states = residual + hidden_states
170
+ return hidden_states
171
+
172
+
173
+class QWenModel(nn.Module):
174
+
175
+ def __init__(self, config: QWenConfig):
176
+ super().__init__()
177
+ self.config = config
178
+ self.vocab_size = config.vocab_size
179
+
180
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
181
+ self.wte = VocabParallelEmbedding(vocab_size,
182
+ config.n_embd,
183
+ perform_initialization=False)
184
+ self.h = nn.ModuleList(
185
+ [QWenBlock(config) for _ in range(config.num_hidden_layers)])
186
+ self.ln_f = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
187
+
188
+ def forward(
189
+ self,
190
+ input_ids: torch.Tensor,
191
+ positions: torch.Tensor,
192
+ kv_caches: List[KVCache],
193
+ input_metadata: InputMetadata,
194
+ cache_events: Optional[List[torch.cuda.Event]],
195
+ ) -> torch.Tensor:
196
+ hidden_states = self.wte(input_ids)
197
+ for i in range(len(self.h)):
198
+ if cache_events is None:
199
+ cache_event = None
200
+ else:
201
+ cache_event = cache_events[i]
202
+ layer = self.h[i]
203
+ hidden_states = layer(
204
+ positions,
205
+ hidden_states,
206
+ kv_caches[i],
207
+ input_metadata,
208
+ cache_event,
209
+ )
210
+ hidden_states = self.ln_f(hidden_states)
211
+ return hidden_states
212
+
213
+
214
+class QWenLMHeadModel(nn.Module):
215
+
216
+ def __init__(self, config: QWenConfig):
217
+ super().__init__()
218
+ self.config = config
219
+ self.transformer = QWenModel(config)
220
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
221
+ self.lm_head = ColumnParallelLinear(
222
+ config.n_embd,
223
+ vocab_size,
224
+ bias=False,
225
+ gather_output=False,
226
+ perform_initialization=False,
227
+ )
228
+ self.sampler = Sampler(config.vocab_size)
229
+
230
+ def forward(
231
+ self,
232
+ input_ids: torch.Tensor,
233
+ positions: torch.Tensor,
234
+ kv_caches: List[KVCache],
235
+ input_metadata: InputMetadata,
236
+ cache_events: Optional[List[torch.cuda.Event]],
237
+ ) -> Dict[int, SequenceOutputs]:
238
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
239
+ input_metadata, cache_events)
240
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
241
+ input_metadata)
242
+ return next_tokens
243
+
244
+ _column_parallel_weights = ["wte.weight", "lm_head.weight"]
245
+ _row_parallel_weights = ["c_proj.weight"]
246
+
247
+ def load_weights(
248
+ self,
249
+ model_name_or_path: str,
250
+ cache_dir: Optional[str] = None,
251
+ use_np_cache: bool = False,
252
+ ):
253
+ tp_world_size = get_tensor_model_parallel_world_size()
254
+ tp_rank = get_tensor_model_parallel_rank()
255
+ state_dict = self.state_dict()
256
+
257
+ for name, loaded_weight in hf_model_weights_iterator(
258
+ model_name_or_path, cache_dir, use_np_cache):
259
+ if "rotary_emb.inv_freq" in name:
260
+ continue
261
+
262
+ if "wte" in name or "lm_head" in name:
263
+ # Consider padding in the vocab size.
264
+ param = state_dict[name]
265
+ padded_vocab_size = param.shape[0] * tp_world_size
266
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
267
+ extra_rows = torch.empty(num_extra_rows,
268
+ loaded_weight.shape[1])
269
+ extra_rows = extra_rows.to(loaded_weight)
270
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
271
+
272
+ if "c_attn" in name:
273
+ total_num_heads = self.config.num_attention_heads
274
+ hidden_size = self.config.hidden_size
275
+ head_size = hidden_size // total_num_heads
276
+ num_heads = total_num_heads // tp_world_size
277
+ head_start = tp_rank * num_heads
278
+ head_end = (tp_rank + 1) * num_heads
279
+
280
+ if "weight" in name:
281
+ loaded_weight = loaded_weight.view(3, total_num_heads,
282
+ head_size, hidden_size)
283
+ loaded_weight = loaded_weight[:, head_start:head_end, :, :]
284
+ loaded_weight = loaded_weight.reshape(-1, hidden_size)
285
+ elif "bias" in name:
286
+ loaded_weight = loaded_weight.view(3, total_num_heads,
287
+ head_size)
288
+ loaded_weight = loaded_weight[:, head_start:head_end, :]
289
+ loaded_weight = loaded_weight.reshape(-1)
290
+
291
+ is_gate_up_weight = False
292
+ for stride_id, weight_name in enumerate(["w2", "w1"]):
293
+ if weight_name not in name:
294
+ continue
295
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
296
+ shard_size = param.shape[0] // 2
297
+ loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
298
+ (tp_rank + 1)]
299
+ param_slice = param.data[shard_size * stride_id:shard_size *
300
+ (stride_id + 1)]
301
+ assert param_slice.shape == loaded_weight.shape
302
+ param_slice.copy_(loaded_weight)
303
+ is_gate_up_weight = True
304
+ break
305
+ if is_gate_up_weight:
306
+ continue
307
+
308
+ param = state_dict[name]
309
+ load_tensor_parallel_weights(
310
+ param,
311
+ loaded_weight,
312
+ name,
313
+ self._column_parallel_weights,
314
+ self._row_parallel_weights,
315
+ tp_rank,
316
+ )
vllm/transformers_utils/config.py
+1
@@ -5,6 +5,7 @@ from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
5
_CONFIG_REGISTRY = {
6
"mpt": MPTConfig,
7
"baichuan": BaiChuanConfig,
8
+ "qwen": QWenConfig,
9
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
10
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
11
}
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,5 +1,6 @@
1
from vllm.transformers_utils.configs.mpt import MPTConfig
2
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3
+from vllm.transformers_utils.configs.qwen import QWenConfig
4
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
5
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
6
# `FalconConfig` class from the official HuggingFace transformers library.
@@ -8,5 +9,6 @@ from vllm.transformers_utils.configs.falcon import RWConfig
9
__all__ = [
10
"MPTConfig",
11
"BaiChuanConfig",
12
+ "QWenConfig",
13
"RWConfig",
14
]
vllm/transformers_utils/configs/qwen.py
new
+71
@@ -0,0 +1,71 @@
1
+# Copyright (c) Alibaba Cloud.
2
+# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
3
+
4
+from transformers import PretrainedConfig
5
+
6
+
7
+class QWenConfig(PretrainedConfig):
8
+ model_type = "qwen"
9
+ keys_to_ignore_at_inference = ["past_key_values"]
10
+ attribute_map = {
11
+ "hidden_size": "n_embd",
12
+ "num_attention_heads": "n_head",
13
+ "max_position_embeddings": "n_positions",
14
+ "num_hidden_layers": "n_layer",
15
+ }
16
+
17
+ def __init__(
18
+ self,
19
+ vocab_size=151851,
20
+ n_embd=4096,
21
+ n_layer=32,
22
+ n_head=32,
23
+ n_inner=None,
24
+ embd_pdrop=0.0,
25
+ attn_pdrop=0.0,
26
+ layer_norm_epsilon=1e-5,
27
+ initializer_range=0.02,
28
+ scale_attn_weights=True,
29
+ use_cache=True,
30
+ eos_token_id=151643,
31
+ apply_residual_connection_post_layernorm=False,
32
+ bf16=True,
33
+ kv_channels=128,
34
+ rotary_pct=1.0,
35
+ rotary_emb_base=10000,
36
+ use_dynamic_ntk=False,
37
+ use_logn_attn=False,
38
+ use_flash_attn=True,
39
+ ffn_hidden_size=22016,
40
+ no_bias=True,
41
+ tie_word_embeddings=False,
42
+ **kwargs,
43
+ ):
44
+ self.eos_token_id = eos_token_id
45
+ super().__init__(eos_token_id=eos_token_id,
46
+ tie_word_embeddings=tie_word_embeddings,
47
+ **kwargs)
48
+
49
+ self.vocab_size = vocab_size
50
+ self.n_embd = n_embd
51
+ self.n_layer = n_layer
52
+ self.n_head = n_head
53
+ self.n_inner = n_inner
54
+ self.embd_pdrop = embd_pdrop
55
+ self.attn_pdrop = attn_pdrop
56
+ self.layer_norm_epsilon = layer_norm_epsilon
57
+ self.initializer_range = initializer_range
58
+ self.scale_attn_weights = scale_attn_weights
59
+ self.use_cache = use_cache
60
+ self.apply_residual_connection_post_layernorm = (
61
+ apply_residual_connection_post_layernorm)
62
+ self.bf16 = bf16
63
+ self.kv_channels = kv_channels
64
+ self.rotary_pct = rotary_pct
65
+ self.rotary_emb_base = rotary_emb_base
66
+ self.use_dynamic_ntk = use_dynamic_ntk
67
+ self.use_logn_attn = use_logn_attn
68
+ self.use_flash_attn = use_flash_attn
69
+ self.ffn_hidden_size = ffn_hidden_size
70
+ self.no_bias = no_bias
71
+ self.tie_word_embeddings = tie_word_embeddings