A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add qwen2 (#2495)
Junyang Lin committed
Jan 23, 2024 at 06:34 UTC
94b5edeb5384ea2a46533a11dd5938b2c859bf5c
6 files changed
+344
-2
README.md
+1
@@ -77,6 +77,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
77
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
78
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
79
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
80
+- Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
81
- StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
82
- Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
83
docs/source/models/supported_models.rst
+3
@@ -69,6 +69,9 @@ Alongside each architecture, we include some popular models that use it.
69
- Qwen
70
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
71
* - :code:`StableLMEpochForCausalLM`
72
+ * - :code:`Qwen2ForCausalLM`
73
+ - Qwen2
74
+ - :code:`Qwen/Qwen2-7B-beta`, :code:`Qwen/Qwen-7B-Chat-beta`, etc.
75
- StableLM
76
- :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
77
* - :code:`YiForCausalLM`
requirements-rocm.txt
+1
-1
@@ -6,7 +6,7 @@ ray >= 2.5.1
6
sentencepiece # Required for LLaMA tokenizer.
7
numpy
8
tokenizers>=0.15.0
9
-transformers >= 4.36.0 # Required for Mixtral.
9
+transformers >= 4.37.0 # Required for Mixtral.
10
fastapi
11
uvicorn[standard]
12
pydantic >= 2.0 # Required for OpenAI server.
requirements.txt
+1
-1
@@ -4,7 +4,7 @@ ray >= 2.5.1
4
sentencepiece # Required for LLaMA tokenizer.
5
numpy
6
torch == 2.1.2
7
-transformers >= 4.36.0 # Required for Mixtral.
7
+transformers >= 4.37.0 # Required for Qwen2
8
xformers == 0.0.23.post1 # Required for CUDA 12.1.
9
fastapi
10
uvicorn[standard]
vllm/model_executor/models/__init__.py
+3
@@ -35,6 +35,7 @@ _MODELS = {
35
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
36
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
37
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38
+ "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
39
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
40
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
41
"YiForCausalLM": ("yi", "YiForCausalLM")
@@ -46,6 +47,8 @@ _ROCM_UNSUPPORTED_MODELS = []
47
# Models partially supported by ROCm.
48
# Architecture -> Reason.
49
_ROCM_PARTIALLY_SUPPORTED_MODELS = {
50
+ "Qwen2ForCausalLM":
51
+ "Sliding window attention is not yet supported in ROCm's flash attention",
52
"MistralForCausalLM":
53
"Sliding window attention is not yet supported in ROCm's flash attention",
54
"MixtralForCausalLM":
vllm/model_executor/models/qwen2.py
new
+335
@@ -0,0 +1,335 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2/modeling_qwen2.py
4
+# Copyright 2024 The Qwen team and the HuggingFace Inc. team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Mistral model compatible with HuggingFace weights."""
24
+from typing import List, Optional, Tuple
25
+
26
+import torch
27
+from torch import nn
28
+from transformers import Qwen2Config
29
+
30
+from vllm.model_executor.input_metadata import InputMetadata
31
+from vllm.model_executor.layers.activation import SiluAndMul
32
+from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.layernorm import RMSNorm
34
+from vllm.model_executor.layers.linear import (LinearMethodBase,
35
+ MergedColumnParallelLinear,
36
+ QKVParallelLinear,
37
+ RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
+from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.vocab_parallel_embedding import (
41
+ VocabParallelEmbedding, ParallelLMHead)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
+from vllm.sequence import SamplerOutput
48
+
49
+KVCache = Tuple[torch.Tensor, torch.Tensor]
50
+
51
+
52
+class Qwen2MLP(nn.Module):
53
+
54
+ def __init__(
55
+ self,
56
+ hidden_size: int,
57
+ intermediate_size: int,
58
+ hidden_act: str,
59
+ linear_method: Optional[LinearMethodBase] = None,
60
+ ) -> None:
61
+ super().__init__()
62
+ self.gate_up_proj = MergedColumnParallelLinear(
63
+ hidden_size, [intermediate_size] * 2,
64
+ bias=False,
65
+ linear_method=linear_method)
66
+ self.down_proj = RowParallelLinear(intermediate_size,
67
+ hidden_size,
68
+ bias=False,
69
+ linear_method=linear_method)
70
+ if hidden_act != "silu":
71
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
72
+ "Only silu is supported for now.")
73
+ self.act_fn = SiluAndMul()
74
+
75
+ def forward(self, x):
76
+ gate_up, _ = self.gate_up_proj(x)
77
+ x = self.act_fn(gate_up)
78
+ x, _ = self.down_proj(x)
79
+ return x
80
+
81
+
82
+class Qwen2Attention(nn.Module):
83
+
84
+ def __init__(self,
85
+ hidden_size: int,
86
+ num_heads: int,
87
+ num_kv_heads: int,
88
+ max_position: int = 4096 * 32,
89
+ rope_theta: float = 10000,
90
+ use_sliding_window: bool = False,
91
+ linear_method: Optional[LinearMethodBase] = None,
92
+ sliding_window: Optional[int] = None) -> None:
93
+ super().__init__()
94
+ self.hidden_size = hidden_size
95
+ tp_size = get_tensor_model_parallel_world_size()
96
+ self.total_num_heads = num_heads
97
+ assert self.total_num_heads % tp_size == 0
98
+ self.num_heads = self.total_num_heads // tp_size
99
+ self.total_num_kv_heads = num_kv_heads
100
+ if self.total_num_kv_heads >= tp_size:
101
+ # Number of KV heads is greater than TP size, so we partition
102
+ # the KV heads across multiple tensor parallel GPUs.
103
+ assert self.total_num_kv_heads % tp_size == 0
104
+ else:
105
+ # Number of KV heads is less than TP size, so we replicate
106
+ # the KV heads across multiple tensor parallel GPUs.
107
+ assert tp_size % self.total_num_kv_heads == 0
108
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
109
+ self.head_dim = hidden_size // self.total_num_heads
110
+ self.q_size = self.num_heads * self.head_dim
111
+ self.kv_size = self.num_kv_heads * self.head_dim
112
+ self.scaling = self.head_dim**-0.5
113
+ self.rope_theta = rope_theta
114
+ self.sliding_window = sliding_window if use_sliding_window else None
115
+
116
+ self.qkv_proj = QKVParallelLinear(
117
+ hidden_size,
118
+ self.head_dim,
119
+ self.total_num_heads,
120
+ self.total_num_kv_heads,
121
+ bias=True,
122
+ linear_method=linear_method,
123
+ )
124
+ self.o_proj = RowParallelLinear(
125
+ self.total_num_heads * self.head_dim,
126
+ hidden_size,
127
+ bias=False,
128
+ linear_method=linear_method,
129
+ )
130
+
131
+ self.rotary_emb = get_rope(
132
+ self.head_dim,
133
+ rotary_dim=self.head_dim,
134
+ max_position=max_position,
135
+ base=self.rope_theta,
136
+ )
137
+ self.attn = PagedAttention(self.num_heads,
138
+ self.head_dim,
139
+ self.scaling,
140
+ num_kv_heads=self.num_kv_heads,
141
+ sliding_window=self.sliding_window)
142
+
143
+ def forward(
144
+ self,
145
+ positions: torch.Tensor,
146
+ hidden_states: torch.Tensor,
147
+ kv_cache: KVCache,
148
+ input_metadata: InputMetadata,
149
+ ) -> torch.Tensor:
150
+ qkv, _ = self.qkv_proj(hidden_states)
151
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
+ q, k = self.rotary_emb(positions, q, k)
153
+ k_cache, v_cache = kv_cache
154
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
155
+ output, _ = self.o_proj(attn_output)
156
+ return output
157
+
158
+
159
+class Qwen2DecoderLayer(nn.Module):
160
+
161
+ def __init__(
162
+ self,
163
+ config: Qwen2Config,
164
+ layer_idx: int,
165
+ linear_method: Optional[LinearMethodBase] = None,
166
+ ) -> None:
167
+ super().__init__()
168
+ self.hidden_size = config.hidden_size
169
+ # Requires transformers > 4.32.0
170
+ rope_theta = getattr(config, "rope_theta", 1000000)
171
+ use_sliding_window = config.use_sliding_window and layer_idx < config.max_window_layers
172
+ self.self_attn = Qwen2Attention(
173
+ hidden_size=self.hidden_size,
174
+ num_heads=config.num_attention_heads,
175
+ max_position=config.max_position_embeddings,
176
+ num_kv_heads=config.num_key_value_heads,
177
+ rope_theta=rope_theta,
178
+ use_sliding_window=use_sliding_window,
179
+ linear_method=linear_method,
180
+ sliding_window=config.sliding_window)
181
+ self.mlp = Qwen2MLP(
182
+ hidden_size=self.hidden_size,
183
+ intermediate_size=config.intermediate_size,
184
+ hidden_act=config.hidden_act,
185
+ linear_method=linear_method,
186
+ )
187
+ self.input_layernorm = RMSNorm(config.hidden_size,
188
+ eps=config.rms_norm_eps)
189
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
190
+ eps=config.rms_norm_eps)
191
+
192
+ def forward(
193
+ self,
194
+ positions: torch.Tensor,
195
+ hidden_states: torch.Tensor,
196
+ kv_cache: KVCache,
197
+ input_metadata: InputMetadata,
198
+ residual: Optional[torch.Tensor],
199
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
200
+ # Self Attention
201
+ if residual is None:
202
+ residual = hidden_states
203
+ hidden_states = self.input_layernorm(hidden_states)
204
+ else:
205
+ hidden_states, residual = self.input_layernorm(
206
+ hidden_states, residual)
207
+ hidden_states = self.self_attn(
208
+ positions=positions,
209
+ hidden_states=hidden_states,
210
+ kv_cache=kv_cache,
211
+ input_metadata=input_metadata,
212
+ )
213
+
214
+ # Fully Connected
215
+ hidden_states, residual = self.post_attention_layernorm(
216
+ hidden_states, residual)
217
+ hidden_states = self.mlp(hidden_states)
218
+ return hidden_states, residual
219
+
220
+
221
+class Qwen2Model(nn.Module):
222
+
223
+ def __init__(
224
+ self,
225
+ config: Qwen2Config,
226
+ linear_method: Optional[LinearMethodBase] = None,
227
+ ) -> None:
228
+ super().__init__()
229
+ self.config = config
230
+ self.padding_idx = config.pad_token_id
231
+ self.vocab_size = config.vocab_size
232
+
233
+ self.embed_tokens = VocabParallelEmbedding(
234
+ config.vocab_size,
235
+ config.hidden_size,
236
+ )
237
+ self.layers = nn.ModuleList([
238
+ Qwen2DecoderLayer(config, layer_idx, linear_method)
239
+ for layer_idx in range(config.num_hidden_layers)
240
+ ])
241
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
242
+
243
+ def forward(
244
+ self,
245
+ input_ids: torch.Tensor,
246
+ positions: torch.Tensor,
247
+ kv_caches: List[KVCache],
248
+ input_metadata: InputMetadata,
249
+ ) -> torch.Tensor:
250
+ hidden_states = self.embed_tokens(input_ids)
251
+ residual = None
252
+ for i in range(len(self.layers)):
253
+ layer = self.layers[i]
254
+ hidden_states, residual = layer(
255
+ positions,
256
+ hidden_states,
257
+ kv_caches[i],
258
+ input_metadata,
259
+ residual,
260
+ )
261
+ hidden_states, _ = self.norm(hidden_states, residual)
262
+ return hidden_states
263
+
264
+
265
+class Qwen2ForCausalLM(nn.Module):
266
+
267
+ def __init__(
268
+ self,
269
+ config: Qwen2Config,
270
+ linear_method: Optional[LinearMethodBase] = None,
271
+ ) -> None:
272
+ super().__init__()
273
+ self.config = config
274
+ self.linear_method = linear_method
275
+ self.model = Qwen2Model(config, linear_method)
276
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
277
+ self.sampler = Sampler(config.vocab_size)
278
+
279
+ def forward(
280
+ self,
281
+ input_ids: torch.Tensor,
282
+ positions: torch.Tensor,
283
+ kv_caches: List[KVCache],
284
+ input_metadata: InputMetadata,
285
+ ) -> torch.Tensor:
286
+ hidden_states = self.model(input_ids, positions, kv_caches,
287
+ input_metadata)
288
+ return hidden_states
289
+
290
+ def sample(
291
+ self,
292
+ hidden_states: torch.Tensor,
293
+ sampling_metadata: SamplingMetadata,
294
+ ) -> Optional[SamplerOutput]:
295
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
296
+ sampling_metadata)
297
+ return next_tokens
298
+
299
+ def load_weights(self,
300
+ model_name_or_path: str,
301
+ cache_dir: Optional[str] = None,
302
+ load_format: str = "auto",
303
+ revision: Optional[str] = None):
304
+ stacked_params_mapping = [
305
+ # (param_name, shard_name, shard_id)
306
+ ("qkv_proj", "q_proj", "q"),
307
+ ("qkv_proj", "k_proj", "k"),
308
+ ("qkv_proj", "v_proj", "v"),
309
+ ("gate_up_proj", "gate_proj", 0),
310
+ ("gate_up_proj", "up_proj", 1),
311
+ ]
312
+ params_dict = dict(self.named_parameters())
313
+ for name, loaded_weight in hf_model_weights_iterator(
314
+ model_name_or_path, cache_dir, load_format, revision):
315
+ if "rotary_emb.inv_freq" in name:
316
+ continue
317
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
318
+ if weight_name not in name:
319
+ continue
320
+ name = name.replace(weight_name, param_name)
321
+ # Skip loading extra bias for GPTQ models.
322
+ if name.endswith(".bias") and name not in params_dict:
323
+ continue
324
+ param = params_dict[name]
325
+ weight_loader = param.weight_loader
326
+ weight_loader(param, loaded_weight, shard_id)
327
+ break
328
+ else:
329
+ # Skip loading extra bias for GPTQ models.
330
+ if name.endswith(".bias") and name not in params_dict:
331
+ continue
332
+ param = params_dict[name]
333
+ weight_loader = getattr(param, "weight_loader",
334
+ default_weight_loader)
335
+ weight_loader(param, loaded_weight)