A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add quantized mixtral support (#2673)
Woosuk Kwon committed
Jan 30, 2024 at 16:34 UTC
3dad94448583a835230ae68c726d4b67e46845f2
3 files changed
+422
-4
vllm/model_executor/model_loader.py
+9
-4
@@ -4,7 +4,6 @@ from typing import Optional, Type
4
5
import torch
6
import torch.nn as nn
7
-from transformers import PretrainedConfig
7
8
from vllm.config import ModelConfig, LoRAConfig
9
from vllm.model_executor.models import ModelRegistry
@@ -21,8 +20,14 @@ def _set_default_torch_dtype(dtype: torch.dtype):
20
torch.set_default_dtype(old_dtype)
21
22
24
-def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
25
- architectures = getattr(config, "architectures", [])
23
+def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
24
+ architectures = getattr(model_config.hf_config, "architectures", [])
25
+ # Special handling for quantized Mixtral.
26
+ # FIXME(woosuk): This is a temporary hack.
27
+ if (model_config.quantization is not None
28
+ and "MixtralForCausalLM" in architectures):
29
+ architectures = ["QuantMixtralForCausalLM"]
30
+
31
for arch in architectures:
32
model_cls = ModelRegistry.load_model_cls(arch)
33
if model_cls is not None:
@@ -34,7 +39,7 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
39
40
def get_model(model_config: ModelConfig,
41
lora_config: Optional[LoRAConfig] = None) -> nn.Module:
37
- model_class = _get_model_architecture(model_config.hf_config)
42
+ model_class = _get_model_architecture(model_config)
43
44
# Get the (maybe quantized) linear method.
45
linear_method = None
vllm/model_executor/models/__init__.py
+1
@@ -30,6 +30,7 @@ _MODELS = {
30
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
31
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
32
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
33
+ "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
34
# transformers's mpt class has lower case
35
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
36
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
vllm/model_executor/models/mixtral_quant.py
new
+412
@@ -0,0 +1,412 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only Mixtral model."""
24
+from typing import List, Optional, Tuple
25
+
26
+import numpy as np
27
+
28
+import torch
29
+import torch.nn.functional as F
30
+
31
+from torch import nn
32
+from transformers import MixtralConfig
33
+
34
+from vllm.model_executor.input_metadata import InputMetadata
35
+from vllm.model_executor.layers.attention import PagedAttention
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.linear import (LinearMethodBase,
38
+ ReplicatedLinear,
39
+ QKVParallelLinear,
40
+ RowParallelLinear)
41
+from vllm.model_executor.layers.rotary_embedding import get_rope
42
+from vllm.model_executor.layers.sampler import Sampler
43
+from vllm.model_executor.layers.vocab_parallel_embedding import (
44
+ VocabParallelEmbedding, ParallelLMHead)
45
+from vllm.model_executor.parallel_utils.communication_op import (
46
+ tensor_model_parallel_all_reduce)
47
+from vllm.model_executor.parallel_utils.parallel_state import (
48
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49
+from vllm.model_executor.sampling_metadata import SamplingMetadata
50
+from vllm.model_executor.weight_utils import (default_weight_loader,
51
+ hf_model_weights_iterator)
52
+from vllm.sequence import SamplerOutput
53
+
54
+KVCache = Tuple[torch.Tensor, torch.Tensor]
55
+
56
+
57
+class MixtralMLP(nn.Module):
58
+
59
+ def __init__(
60
+ self,
61
+ num_experts: int,
62
+ hidden_size: int,
63
+ intermediate_size: int,
64
+ linear_method: Optional[LinearMethodBase] = None,
65
+ ) -> None:
66
+ super().__init__()
67
+ self.num_experts = num_experts
68
+ self.ffn_dim = intermediate_size
69
+ self.hidden_dim = hidden_size
70
+
71
+ self.w1 = ReplicatedLinear(self.hidden_dim,
72
+ self.ffn_dim,
73
+ bias=False,
74
+ linear_method=linear_method)
75
+ self.w2 = ReplicatedLinear(self.ffn_dim,
76
+ self.hidden_dim,
77
+ bias=False,
78
+ linear_method=linear_method)
79
+ self.w3 = ReplicatedLinear(self.hidden_dim,
80
+ self.ffn_dim,
81
+ bias=False,
82
+ linear_method=linear_method)
83
+
84
+ # TODO: Use vllm's SiluAndMul
85
+ self.act_fn = nn.SiLU()
86
+
87
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
88
+ w1_out, _ = self.w1(hidden_states)
89
+ w1_out = self.act_fn(w1_out)
90
+ w3_out, _ = self.w3(hidden_states)
91
+ current_hidden_states = w1_out * w3_out
92
+ current_hidden_states, _ = self.w2(current_hidden_states)
93
+ return current_hidden_states
94
+
95
+
96
+class MixtralMoE(nn.Module):
97
+
98
+ def __init__(
99
+ self,
100
+ config: MixtralConfig,
101
+ linear_method: Optional[LinearMethodBase] = None,
102
+ ):
103
+ super().__init__()
104
+ self.config = config
105
+ self.rank = get_tensor_model_parallel_rank()
106
+ self.tp_size = get_tensor_model_parallel_world_size()
107
+ self.num_total_experts = config.num_local_experts
108
+ self.top_k = config.num_experts_per_tok
109
+ if self.tp_size > self.num_total_experts:
110
+ raise ValueError(
111
+ f"Tensor parallel size {self.tp_size} is greater than "
112
+ f"the number of experts {self.num_total_experts}.")
113
+ # Split experts equally between ranks
114
+ self.expert_indicies = np.array_split(range(
115
+ self.num_total_experts), self.tp_size)[self.rank].tolist()
116
+ if not self.expert_indicies:
117
+ raise ValueError(
118
+ f"Rank {self.rank} has no experts assigned to it.")
119
+
120
+ self.experts = nn.ModuleList([
121
+ MixtralMLP(self.num_total_experts,
122
+ config.hidden_size,
123
+ config.intermediate_size,
124
+ linear_method=linear_method)
125
+ if idx in self.expert_indicies else None
126
+ for idx in range(self.num_total_experts)
127
+ ])
128
+ self.gate = ReplicatedLinear(config.hidden_size,
129
+ self.num_total_experts,
130
+ bias=False,
131
+ linear_method=None)
132
+
133
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
134
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
135
+ hidden_states = hidden_states.view(-1, hidden_dim)
136
+ # router_logits: (batch * sequence_length, n_experts)
137
+ router_logits, _ = self.gate(hidden_states)
138
+
139
+ routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
140
+ routing_weights, selected_experts = torch.topk(routing_weights,
141
+ self.top_k,
142
+ dim=-1)
143
+ routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
144
+
145
+ final_hidden_states = None
146
+ for expert_idx in self.expert_indicies:
147
+ expert_layer = self.experts[expert_idx]
148
+ expert_mask = (selected_experts == expert_idx)
149
+ expert_weights = (routing_weights * expert_mask).sum(dim=-1,
150
+ keepdim=True)
151
+
152
+ current_hidden_states = expert_layer(hidden_states).mul_(
153
+ expert_weights)
154
+ if final_hidden_states is None:
155
+ final_hidden_states = current_hidden_states
156
+ else:
157
+ final_hidden_states.add_(current_hidden_states)
158
+
159
+ return tensor_model_parallel_all_reduce(final_hidden_states).view(
160
+ batch_size, sequence_length, hidden_dim)
161
+
162
+
163
+class MixtralAttention(nn.Module):
164
+
165
+ def __init__(self,
166
+ hidden_size: int,
167
+ num_heads: int,
168
+ num_kv_heads: int,
169
+ max_position: int = 4096 * 32,
170
+ rope_theta: float = 10000,
171
+ linear_method: Optional[LinearMethodBase] = None,
172
+ sliding_window: Optional[int] = None) -> None:
173
+ super().__init__()
174
+ self.hidden_size = hidden_size
175
+ tp_size = get_tensor_model_parallel_world_size()
176
+ self.total_num_heads = num_heads
177
+ assert self.total_num_heads % tp_size == 0
178
+ self.num_heads = self.total_num_heads // tp_size
179
+ self.total_num_kv_heads = num_kv_heads
180
+ if self.total_num_kv_heads >= tp_size:
181
+ # Number of KV heads is greater than TP size, so we partition
182
+ # the KV heads across multiple tensor parallel GPUs.
183
+ assert self.total_num_kv_heads % tp_size == 0
184
+ else:
185
+ # Number of KV heads is less than TP size, so we replicate
186
+ # the KV heads across multiple tensor parallel GPUs.
187
+ assert tp_size % self.total_num_kv_heads == 0
188
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
189
+ self.head_dim = hidden_size // self.total_num_heads
190
+ self.q_size = self.num_heads * self.head_dim
191
+ self.kv_size = self.num_kv_heads * self.head_dim
192
+ self.scaling = self.head_dim**-0.5
193
+ self.rope_theta = rope_theta
194
+ self.sliding_window = sliding_window
195
+
196
+ self.qkv_proj = QKVParallelLinear(
197
+ hidden_size,
198
+ self.head_dim,
199
+ self.total_num_heads,
200
+ self.total_num_kv_heads,
201
+ bias=False,
202
+ linear_method=linear_method,
203
+ )
204
+ self.o_proj = RowParallelLinear(
205
+ self.total_num_heads * self.head_dim,
206
+ hidden_size,
207
+ bias=False,
208
+ linear_method=linear_method,
209
+ )
210
+ self.rotary_emb = get_rope(
211
+ self.head_dim,
212
+ rotary_dim=self.head_dim,
213
+ max_position=max_position,
214
+ base=int(self.rope_theta),
215
+ is_neox_style=True,
216
+ )
217
+ self.attn = PagedAttention(
218
+ self.num_heads,
219
+ self.head_dim,
220
+ self.scaling,
221
+ num_kv_heads=self.num_kv_heads,
222
+ sliding_window=self.sliding_window,
223
+ )
224
+
225
+ def forward(
226
+ self,
227
+ positions: torch.Tensor,
228
+ hidden_states: torch.Tensor,
229
+ kv_cache: KVCache,
230
+ input_metadata: InputMetadata,
231
+ ) -> torch.Tensor:
232
+ qkv, _ = self.qkv_proj(hidden_states)
233
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
234
+ q, k = self.rotary_emb(positions, q, k)
235
+ k_cache, v_cache = kv_cache
236
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
237
+ output, _ = self.o_proj(attn_output)
238
+ return output
239
+
240
+
241
+class MixtralDecoderLayer(nn.Module):
242
+
243
+ def __init__(
244
+ self,
245
+ config: MixtralConfig,
246
+ linear_method: Optional[LinearMethodBase] = None,
247
+ ) -> None:
248
+ super().__init__()
249
+ self.hidden_size = config.hidden_size
250
+ # Requires transformers > 4.32.0
251
+ rope_theta = getattr(config, "rope_theta", 10000)
252
+ self.self_attn = MixtralAttention(
253
+ hidden_size=self.hidden_size,
254
+ num_heads=config.num_attention_heads,
255
+ max_position=config.max_position_embeddings,
256
+ num_kv_heads=config.num_key_value_heads,
257
+ rope_theta=rope_theta,
258
+ sliding_window=config.sliding_window,
259
+ linear_method=linear_method)
260
+ self.block_sparse_moe = MixtralMoE(config=config,
261
+ linear_method=linear_method)
262
+ self.input_layernorm = RMSNorm(config.hidden_size,
263
+ eps=config.rms_norm_eps)
264
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
265
+ eps=config.rms_norm_eps)
266
+
267
+ def forward(
268
+ self,
269
+ positions: torch.Tensor,
270
+ hidden_states: torch.Tensor,
271
+ kv_cache: KVCache,
272
+ input_metadata: InputMetadata,
273
+ residual: Optional[torch.Tensor],
274
+ ) -> torch.Tensor:
275
+ # Self Attention
276
+ if residual is None:
277
+ residual = hidden_states
278
+ hidden_states = self.input_layernorm(hidden_states)
279
+ else:
280
+ hidden_states, residual = self.input_layernorm(
281
+ hidden_states, residual)
282
+ hidden_states = self.self_attn(
283
+ positions=positions,
284
+ hidden_states=hidden_states,
285
+ kv_cache=kv_cache,
286
+ input_metadata=input_metadata,
287
+ )
288
+
289
+ # Fully Connected
290
+ hidden_states, residual = self.post_attention_layernorm(
291
+ hidden_states, residual)
292
+ hidden_states = self.block_sparse_moe(hidden_states)
293
+ return hidden_states, residual
294
+
295
+
296
+class MixtralModel(nn.Module):
297
+
298
+ def __init__(
299
+ self,
300
+ config: MixtralConfig,
301
+ linear_method: Optional[LinearMethodBase] = None,
302
+ ) -> None:
303
+ super().__init__()
304
+ self.padding_idx = config.pad_token_id
305
+ self.vocab_size = config.vocab_size
306
+
307
+ self.embed_tokens = VocabParallelEmbedding(
308
+ config.vocab_size,
309
+ config.hidden_size,
310
+ )
311
+ self.layers = nn.ModuleList([
312
+ MixtralDecoderLayer(config, linear_method=linear_method)
313
+ for _ in range(config.num_hidden_layers)
314
+ ])
315
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
316
+
317
+ def forward(
318
+ self,
319
+ input_ids: torch.Tensor,
320
+ positions: torch.Tensor,
321
+ kv_caches: List[KVCache],
322
+ input_metadata: InputMetadata,
323
+ ) -> torch.Tensor:
324
+ hidden_states = self.embed_tokens(input_ids)
325
+ residual = None
326
+ for i in range(len(self.layers)):
327
+ layer = self.layers[i]
328
+ hidden_states, residual = layer(positions, hidden_states,
329
+ kv_caches[i], input_metadata,
330
+ residual)
331
+ hidden_states, _ = self.norm(hidden_states, residual)
332
+ return hidden_states
333
+
334
+
335
+class MixtralForCausalLM(nn.Module):
336
+
337
+ def __init__(
338
+ self,
339
+ config: MixtralConfig,
340
+ linear_method: Optional[LinearMethodBase] = None,
341
+ ) -> None:
342
+ super().__init__()
343
+ self.config = config
344
+ self.linear_method = linear_method
345
+ self.model = MixtralModel(config, linear_method)
346
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347
+ self.sampler = Sampler(config.vocab_size)
348
+
349
+ def forward(
350
+ self,
351
+ input_ids: torch.Tensor,
352
+ positions: torch.Tensor,
353
+ kv_caches: List[KVCache],
354
+ input_metadata: InputMetadata,
355
+ ) -> torch.Tensor:
356
+ hidden_states = self.model(input_ids, positions, kv_caches,
357
+ input_metadata)
358
+ return hidden_states
359
+
360
+ def sample(
361
+ self,
362
+ hidden_states: Optional[torch.Tensor],
363
+ sampling_metadata: SamplingMetadata,
364
+ ) -> Optional[SamplerOutput]:
365
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366
+ sampling_metadata)
367
+ return next_tokens
368
+
369
+ def load_weights(self,
370
+ model_name_or_path: str,
371
+ cache_dir: Optional[str] = None,
372
+ load_format: str = "auto",
373
+ revision: Optional[str] = None):
374
+ stacked_params_mapping = [
375
+ # (param_name, shard_name, shard_id)
376
+ ("qkv_proj", "q_proj", "q"),
377
+ ("qkv_proj", "k_proj", "k"),
378
+ ("qkv_proj", "v_proj", "v"),
379
+ ]
380
+
381
+ params_dict = dict(self.named_parameters())
382
+ for name, loaded_weight in hf_model_weights_iterator(
383
+ model_name_or_path,
384
+ cache_dir,
385
+ load_format,
386
+ revision,
387
+ fall_back_to_pt=False):
388
+ if "rotary_emb.inv_freq" in name:
389
+ continue
390
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
391
+ if weight_name not in name:
392
+ continue
393
+ name = name.replace(weight_name, param_name)
394
+ # Skip loading extra bias for GPTQ models.
395
+ if name.endswith(".bias") and name not in params_dict:
396
+ continue
397
+ param = params_dict[name]
398
+ weight_loader = param.weight_loader
399
+ weight_loader(param, loaded_weight, shard_id)
400
+ break
401
+ else:
402
+ # Skip loading extra bias for GPTQ models.
403
+ if name.endswith(".bias") and name not in params_dict:
404
+ continue
405
+ # Skip experts that are not assigned to this worker.
406
+ if ("block_sparse_moe.experts." in name
407
+ and name not in params_dict):
408
+ continue
409
+ param = params_dict[name]
410
+ weight_loader = getattr(param, "weight_loader",
411
+ default_weight_loader)
412
+ weight_loader(param, loaded_weight)