@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6ea427f04
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Model] Add support for Qwen2MoeModel (#3346)

wenyujin333 committed Mar 28, 2024 at 23:19 UTC d6ea427f04890014fa9b0ab8eb6d87e244ff1a48
4 files changed +463
README.md
+1
@@ -89,6 +89,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
89 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
90 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
91 - Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
92 +- Qwen2MoE (`Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc.)
93 - StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
94 - Starcoder2(`bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc.)
95 - Xverse (`xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc.)
docs/source/models/supported_models.rst
+4
@@ -119,6 +119,10 @@ Alongside each architecture, we include some popular models that use it.
119 - Qwen2
120 - :code:`Qwen/Qwen2-beta-7B`, :code:`Qwen/Qwen2-beta-7B-Chat`, etc.
121 - ✅︎
122 + * - :code:`Qwen2MoeForCausalLM`
123 + - Qwen2MoE
124 + - :code:`Qwen/Qwen1.5-MoE-A2.7B`, :code:`Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc.
125 + -
126 * - :code:`StableLmForCausalLM`
127 - StableLM
128 - :code:`stabilityai/stablelm-3b-4e1t/` , :code:`stabilityai/stablelm-base-alpha-7b-v2`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -47,6 +47,7 @@ _MODELS = {
47 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
48 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
49 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
50 + "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
51 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
52 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
53 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
vllm/model_executor/models/qwen2_moe.py new
+457
@@ -0,0 +1,457 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py
4 +# Copyright 2024 The Qwen team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only Qwen2MoE model compatible with HuggingFace weights."""
25 +from typing import Any, Dict, List, Optional
26 +
27 +import torch
28 +import torch.nn.functional as F
29 +from torch import nn
30 +from transformers import PretrainedConfig
31 +
32 +from vllm.attention import Attention, AttentionMetadata
33 +from vllm.model_executor.layers.activation import SiluAndMul
34 +from vllm.model_executor.layers.fused_moe import fused_moe
35 +from vllm.model_executor.layers.layernorm import RMSNorm
36 +from vllm.model_executor.layers.linear import (LinearMethodBase,
37 + MergedColumnParallelLinear,
38 + QKVParallelLinear,
39 + ReplicatedLinear,
40 + RowParallelLinear)
41 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 +from vllm.model_executor.layers.rotary_embedding import get_rope
43 +from vllm.model_executor.layers.sampler import Sampler
44 +from vllm.model_executor.layers.vocab_parallel_embedding import (
45 + ParallelLMHead, VocabParallelEmbedding)
46 +from vllm.model_executor.parallel_utils.communication_op import (
47 + tensor_model_parallel_all_reduce)
48 +from vllm.model_executor.parallel_utils.parallel_state import (
49 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
50 +from vllm.model_executor.sampling_metadata import SamplingMetadata
51 +from vllm.model_executor.weight_utils import (default_weight_loader,
52 + hf_model_weights_iterator)
53 +from vllm.sequence import SamplerOutput
54 +
55 +
56 +class Qwen2MoeMLP(nn.Module):
57 +
58 + def __init__(
59 + self,
60 + hidden_size: int,
61 + intermediate_size: int,
62 + hidden_act: str,
63 + linear_method: Optional[LinearMethodBase] = None,
64 + reduce_results: bool = True,
65 + ) -> None:
66 + super().__init__()
67 + self.gate_up_proj = MergedColumnParallelLinear(
68 + hidden_size, [intermediate_size] * 2,
69 + bias=False,
70 + linear_method=linear_method)
71 + self.down_proj = RowParallelLinear(intermediate_size,
72 + hidden_size,
73 + bias=False,
74 + linear_method=linear_method,
75 + reduce_results=reduce_results)
76 + if hidden_act != "silu":
77 + raise ValueError(f"Unsupported activation: {hidden_act}. "
78 + "Only silu is supported for now.")
79 + self.act_fn = SiluAndMul()
80 +
81 + def forward(self, x):
82 + gate_up, _ = self.gate_up_proj(x)
83 + x = self.act_fn(gate_up)
84 + x, _ = self.down_proj(x)
85 + return x
86 +
87 +
88 +class Qwen2MoeSparseMoeBlock(nn.Module):
89 +
90 + def __init__(
91 + self,
92 + config: PretrainedConfig,
93 + linear_method: Optional[LinearMethodBase] = None,
94 + ):
95 + super().__init__()
96 + self.config = config
97 + self.rank = get_tensor_model_parallel_rank()
98 + self.tp_size = get_tensor_model_parallel_world_size()
99 + self.n_routed_experts = config.num_experts
100 + self.top_k = config.num_experts_per_tok
101 + if self.tp_size > self.n_routed_experts:
102 + raise ValueError(
103 + f"Tensor parallel size {self.tp_size} is greater than "
104 + f"the number of experts {self.n_routed_experts}.")
105 +
106 + self.experts = nn.ModuleList([
107 + Qwen2MoeMLP(hidden_size=config.hidden_size,
108 + intermediate_size=config.moe_intermediate_size,
109 + hidden_act=config.hidden_act,
110 + linear_method=linear_method,
111 + reduce_results=False)
112 + for idx in range(self.n_routed_experts)
113 + ])
114 + self.pack_params()
115 +
116 + self.gate = ReplicatedLinear(config.hidden_size,
117 + self.n_routed_experts,
118 + bias=False,
119 + linear_method=None)
120 + if config.shared_expert_intermediate_size > 0:
121 + self.shared_expert = Qwen2MoeMLP(
122 + hidden_size=config.hidden_size,
123 + intermediate_size=config.shared_expert_intermediate_size,
124 + hidden_act=config.hidden_act,
125 + linear_method=linear_method,
126 + reduce_results=False,
127 + )
128 + else:
129 + self.shared_expert = None
130 + self.shared_expert_gate = torch.nn.Linear(config.hidden_size,
131 + 1,
132 + bias=False)
133 +
134 + def pack_params(self):
135 + w1 = []
136 + w2 = []
137 + for expert in self.experts:
138 + w1.append(expert.gate_up_proj.weight)
139 + w2.append(expert.down_proj.weight)
140 + self.w1 = torch._utils._flatten_dense_tensors(w1)
141 + w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
142 + for data, param in zip(w1s, w1):
143 + param.data = data
144 + self.w1 = self.w1.view(len(w1), *w1s[0].shape)
145 +
146 + self.w2 = torch._utils._flatten_dense_tensors(w2)
147 + w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
148 + for data, param in zip(w2s, w2):
149 + param.data = data
150 +
151 + self.w2 = self.w2.view(len(w2), *w2s[0].shape)
152 +
153 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
154 + num_tokens, hidden_dim = hidden_states.shape
155 + hidden_states = hidden_states.view(-1, hidden_dim)
156 + shared_output = None
157 + if self.shared_expert is not None:
158 + shared_output = self.shared_expert(hidden_states)
159 + if self.shared_expert_gate is not None:
160 + shared_output = F.sigmoid(
161 + self.shared_expert_gate(hidden_states)) * shared_output
162 +
163 + # router_logits: (num_tokens, n_experts)
164 + router_logits, _ = self.gate(hidden_states)
165 + final_hidden_states = fused_moe(hidden_states,
166 + self.w1,
167 + self.w2,
168 + router_logits,
169 + self.top_k,
170 + renormalize=self.config.norm_topk_prob,
171 + inplace=True)
172 +
173 + if shared_output is not None:
174 + final_hidden_states = final_hidden_states + shared_output
175 + final_hidden_states = tensor_model_parallel_all_reduce(
176 + final_hidden_states)
177 +
178 + return final_hidden_states.view(num_tokens, hidden_dim)
179 +
180 +
181 +class Qwen2MoeAttention(nn.Module):
182 +
183 + def __init__(
184 + self,
185 + hidden_size: int,
186 + num_heads: int,
187 + num_kv_heads: int,
188 + rope_theta: float = 10000,
189 + rope_scaling: Optional[Dict[str, Any]] = None,
190 + max_position_embeddings: int = 8192,
191 + linear_method: Optional[LinearMethodBase] = None,
192 + ) -> None:
193 + super().__init__()
194 + self.hidden_size = hidden_size
195 + tp_size = get_tensor_model_parallel_world_size()
196 + self.total_num_heads = num_heads
197 + assert self.total_num_heads % tp_size == 0
198 + self.num_heads = self.total_num_heads // tp_size
199 + self.total_num_kv_heads = num_kv_heads
200 + if self.total_num_kv_heads >= tp_size:
201 + # Number of KV heads is greater than TP size, so we partition
202 + # the KV heads across multiple tensor parallel GPUs.
203 + assert self.total_num_kv_heads % tp_size == 0
204 + else:
205 + # Number of KV heads is less than TP size, so we replicate
206 + # the KV heads across multiple tensor parallel GPUs.
207 + assert tp_size % self.total_num_kv_heads == 0
208 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
209 + self.head_dim = hidden_size // self.total_num_heads
210 + self.q_size = self.num_heads * self.head_dim
211 + self.kv_size = self.num_kv_heads * self.head_dim
212 + self.scaling = self.head_dim**-0.5
213 + self.rope_theta = rope_theta
214 + self.max_position_embeddings = max_position_embeddings
215 +
216 + self.qkv_proj = QKVParallelLinear(
217 + hidden_size,
218 + self.head_dim,
219 + self.total_num_heads,
220 + self.total_num_kv_heads,
221 + bias=True,
222 + linear_method=linear_method,
223 + )
224 +
225 + self.o_proj = RowParallelLinear(
226 + self.total_num_heads * self.head_dim,
227 + hidden_size,
228 + bias=False,
229 + linear_method=linear_method,
230 + )
231 +
232 + self.rotary_emb = get_rope(
233 + self.head_dim,
234 + rotary_dim=self.head_dim,
235 + max_position=max_position_embeddings,
236 + base=rope_theta,
237 + rope_scaling=rope_scaling,
238 + )
239 + self.attn = Attention(self.num_heads,
240 + self.head_dim,
241 + self.scaling,
242 + num_kv_heads=self.num_kv_heads)
243 +
244 + def forward(
245 + self,
246 + positions: torch.Tensor,
247 + hidden_states: torch.Tensor,
248 + kv_cache: torch.Tensor,
249 + attn_metadata: AttentionMetadata,
250 + ) -> torch.Tensor:
251 + qkv, _ = self.qkv_proj(hidden_states)
252 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
253 + q, k = self.rotary_emb(positions, q, k)
254 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
255 + output, _ = self.o_proj(attn_output)
256 + return output
257 +
258 +
259 +class Qwen2MoeDecoderLayer(nn.Module):
260 +
261 + def __init__(
262 + self,
263 + config: PretrainedConfig,
264 + layer_idx: int,
265 + linear_method: Optional[LinearMethodBase] = None,
266 + ) -> None:
267 + super().__init__()
268 + self.hidden_size = config.hidden_size
269 + rope_theta = getattr(config, "rope_theta", 10000)
270 + rope_scaling = getattr(config, "rope_scaling", None)
271 + max_position_embeddings = getattr(config, "max_position_embeddings",
272 + 8192)
273 + self.self_attn = Qwen2MoeAttention(
274 + hidden_size=self.hidden_size,
275 + num_heads=config.num_attention_heads,
276 + num_kv_heads=config.num_key_value_heads,
277 + rope_theta=rope_theta,
278 + rope_scaling=rope_scaling,
279 + max_position_embeddings=max_position_embeddings,
280 + linear_method=linear_method,
281 + )
282 + if (config.num_experts is not None
283 + and (layer_idx + 1) % config.decoder_sparse_step == 0):
284 + self.mlp = Qwen2MoeSparseMoeBlock(config=config,
285 + linear_method=linear_method)
286 + else:
287 + self.mlp = Qwen2MoeMLP(
288 + hidden_size=config.hidden_size,
289 + intermediate_size=config.intermediate_size,
290 + hidden_act=config.hidden_act,
291 + linear_method=linear_method,
292 + )
293 + self.input_layernorm = RMSNorm(config.hidden_size,
294 + eps=config.rms_norm_eps)
295 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
296 + eps=config.rms_norm_eps)
297 +
298 + def forward(
299 + self,
300 + positions: torch.Tensor,
301 + hidden_states: torch.Tensor,
302 + kv_cache: torch.Tensor,
303 + attn_metadata: AttentionMetadata,
304 + residual: Optional[torch.Tensor],
305 + ) -> torch.Tensor:
306 + # Self Attention
307 + if residual is None:
308 + residual = hidden_states
309 + hidden_states = self.input_layernorm(hidden_states)
310 + else:
311 + hidden_states, residual = self.input_layernorm(
312 + hidden_states, residual)
313 + hidden_states = self.self_attn(
314 + positions=positions,
315 + hidden_states=hidden_states,
316 + kv_cache=kv_cache,
317 + attn_metadata=attn_metadata,
318 + )
319 +
320 + # Fully Connected
321 + hidden_states, residual = self.post_attention_layernorm(
322 + hidden_states, residual)
323 + hidden_states = self.mlp(hidden_states)
324 + return hidden_states, residual
325 +
326 +
327 +class Qwen2MoeModel(nn.Module):
328 +
329 + def __init__(
330 + self,
331 + config: PretrainedConfig,
332 + linear_method: Optional[LinearMethodBase] = None,
333 + ) -> None:
334 + super().__init__()
335 + self.padding_idx = config.pad_token_id
336 + self.vocab_size = config.vocab_size
337 +
338 + self.embed_tokens = VocabParallelEmbedding(
339 + config.vocab_size,
340 + config.hidden_size,
341 + )
342 + self.layers = nn.ModuleList([
343 + Qwen2MoeDecoderLayer(config,
344 + layer_idx,
345 + linear_method=linear_method)
346 + for layer_idx in range(config.num_hidden_layers)
347 + ])
348 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
349 +
350 + def forward(
351 + self,
352 + input_ids: torch.Tensor,
353 + positions: torch.Tensor,
354 + kv_caches: List[torch.Tensor],
355 + attn_metadata: AttentionMetadata,
356 + ) -> torch.Tensor:
357 + hidden_states = self.embed_tokens(input_ids)
358 + residual = None
359 + for i in range(len(self.layers)):
360 + layer = self.layers[i]
361 + hidden_states, residual = layer(positions, hidden_states,
362 + kv_caches[i], attn_metadata,
363 + residual)
364 + hidden_states, _ = self.norm(hidden_states, residual)
365 + return hidden_states
366 +
367 +
368 +class Qwen2MoeForCausalLM(nn.Module):
369 +
370 + def __init__(
371 + self,
372 + config: PretrainedConfig,
373 + linear_method: Optional[LinearMethodBase] = None,
374 + ) -> None:
375 + super().__init__()
376 + self.config = config
377 + self.linear_method = linear_method
378 + self.model = Qwen2MoeModel(config, linear_method)
379 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
380 + self.logits_processor = LogitsProcessor(config.vocab_size)
381 + self.sampler = Sampler()
382 +
383 + def forward(
384 + self,
385 + input_ids: torch.Tensor,
386 + positions: torch.Tensor,
387 + kv_caches: List[torch.Tensor],
388 + attn_metadata: AttentionMetadata,
389 + ) -> torch.Tensor:
390 + hidden_states = self.model(input_ids, positions, kv_caches,
391 + attn_metadata)
392 + return hidden_states
393 +
394 + def compute_logits(self, hidden_states: torch.Tensor,
395 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
396 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
397 + sampling_metadata)
398 + return logits
399 +
400 + def sample(
401 + self,
402 + logits: Optional[torch.Tensor],
403 + sampling_metadata: SamplingMetadata,
404 + ) -> Optional[SamplerOutput]:
405 + next_tokens = self.sampler(logits, sampling_metadata)
406 + return next_tokens
407 +
408 + def load_weights(self,
409 + model_name_or_path: str,
410 + cache_dir: Optional[str] = None,
411 + load_format: str = "auto",
412 + revision: Optional[str] = None):
413 + stacked_params_mapping = [
414 + # (param_name, shard_name, shard_id)
415 + ("qkv_proj", "q_proj", "q"),
416 + ("qkv_proj", "k_proj", "k"),
417 + ("qkv_proj", "v_proj", "v"),
418 + ("gate_up_proj", "gate_proj", 0),
419 + ("gate_up_proj", "up_proj", 1),
420 + ]
421 +
422 + params_dict = dict(self.named_parameters())
423 + for name, loaded_weight in hf_model_weights_iterator(
424 + model_name_or_path,
425 + cache_dir,
426 + load_format,
427 + revision,
428 + fall_back_to_pt=False):
429 + if "rotary_emb.inv_freq" in name:
430 + continue
431 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
432 + if weight_name not in name:
433 + continue
434 + name = name.replace(weight_name, param_name)
435 + # Skip loading extra bias for GPTQ models.
436 + if name.endswith(".bias") and name not in params_dict:
437 + continue
438 + # Skip experts that are not assigned to this worker.
439 + if (("mlp.experts." in name or "mlp.shared_expert." in name)
440 + and name not in params_dict):
441 + continue
442 + param = params_dict[name]
443 + weight_loader = param.weight_loader
444 + weight_loader(param, loaded_weight, shard_id)
445 + break
446 + else:
447 + # Skip loading extra bias for GPTQ models.
448 + if name.endswith(".bias") and name not in params_dict:
449 + continue
450 + # Skip experts that are not assigned to this worker.
451 + if (("mlp.experts." in name or "mlp.shared_expert." in name)
452 + and name not in params_dict):
453 + continue
454 + param = params_dict[name]
455 + weight_loader = getattr(param, "weight_loader",
456 + default_weight_loader)
457 + weight_loader(param, loaded_weight)