@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3dad944485
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add quantized mixtral support (#2673)

Woosuk Kwon committed Jan 30, 2024 at 16:34 UTC 3dad94448583a835230ae68c726d4b67e46845f2
3 files changed +422 -4
vllm/model_executor/model_loader.py
+9 -4
@@ -4,7 +4,6 @@ from typing import Optional, Type
4
5 import torch
6 import torch.nn as nn
7 -from transformers import PretrainedConfig
7
8 from vllm.config import ModelConfig, LoRAConfig
9 from vllm.model_executor.models import ModelRegistry
@@ -21,8 +20,14 @@ def _set_default_torch_dtype(dtype: torch.dtype):
20 torch.set_default_dtype(old_dtype)
21
22
24 -def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
25 - architectures = getattr(config, "architectures", [])
23 +def _get_model_architecture(model_config: ModelConfig) -> Type[nn.Module]:
24 + architectures = getattr(model_config.hf_config, "architectures", [])
25 + # Special handling for quantized Mixtral.
26 + # FIXME(woosuk): This is a temporary hack.
27 + if (model_config.quantization is not None
28 + and "MixtralForCausalLM" in architectures):
29 + architectures = ["QuantMixtralForCausalLM"]
30 +
31 for arch in architectures:
32 model_cls = ModelRegistry.load_model_cls(arch)
33 if model_cls is not None:
@@ -34,7 +39,7 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
39
40 def get_model(model_config: ModelConfig,
41 lora_config: Optional[LoRAConfig] = None) -> nn.Module:
37 - model_class = _get_model_architecture(model_config.hf_config)
42 + model_class = _get_model_architecture(model_config)
43
44 # Get the (maybe quantized) linear method.
45 linear_method = None
vllm/model_executor/models/__init__.py
+1
@@ -30,6 +30,7 @@ _MODELS = {
30 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
31 "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
32 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
33 + "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
34 # transformers's mpt class has lower case
35 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
36 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
vllm/model_executor/models/mixtral_quant.py new
+412
@@ -0,0 +1,412 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only Mixtral model."""
24 +from typing import List, Optional, Tuple
25 +
26 +import numpy as np
27 +
28 +import torch
29 +import torch.nn.functional as F
30 +
31 +from torch import nn
32 +from transformers import MixtralConfig
33 +
34 +from vllm.model_executor.input_metadata import InputMetadata
35 +from vllm.model_executor.layers.attention import PagedAttention
36 +from vllm.model_executor.layers.layernorm import RMSNorm
37 +from vllm.model_executor.layers.linear import (LinearMethodBase,
38 + ReplicatedLinear,
39 + QKVParallelLinear,
40 + RowParallelLinear)
41 +from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.sampler import Sampler
43 +from vllm.model_executor.layers.vocab_parallel_embedding import (
44 + VocabParallelEmbedding, ParallelLMHead)
45 +from vllm.model_executor.parallel_utils.communication_op import (
46 + tensor_model_parallel_all_reduce)
47 +from vllm.model_executor.parallel_utils.parallel_state import (
48 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.model_executor.weight_utils import (default_weight_loader,
51 + hf_model_weights_iterator)
52 +from vllm.sequence import SamplerOutput
53 +
54 +KVCache = Tuple[torch.Tensor, torch.Tensor]
55 +
56 +
57 +class MixtralMLP(nn.Module):
58 +
59 + def __init__(
60 + self,
61 + num_experts: int,
62 + hidden_size: int,
63 + intermediate_size: int,
64 + linear_method: Optional[LinearMethodBase] = None,
65 + ) -> None:
66 + super().__init__()
67 + self.num_experts = num_experts
68 + self.ffn_dim = intermediate_size
69 + self.hidden_dim = hidden_size
70 +
71 + self.w1 = ReplicatedLinear(self.hidden_dim,
72 + self.ffn_dim,
73 + bias=False,
74 + linear_method=linear_method)
75 + self.w2 = ReplicatedLinear(self.ffn_dim,
76 + self.hidden_dim,
77 + bias=False,
78 + linear_method=linear_method)
79 + self.w3 = ReplicatedLinear(self.hidden_dim,
80 + self.ffn_dim,
81 + bias=False,
82 + linear_method=linear_method)
83 +
84 + # TODO: Use vllm's SiluAndMul
85 + self.act_fn = nn.SiLU()
86 +
87 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
88 + w1_out, _ = self.w1(hidden_states)
89 + w1_out = self.act_fn(w1_out)
90 + w3_out, _ = self.w3(hidden_states)
91 + current_hidden_states = w1_out * w3_out
92 + current_hidden_states, _ = self.w2(current_hidden_states)
93 + return current_hidden_states
94 +
95 +
96 +class MixtralMoE(nn.Module):
97 +
98 + def __init__(
99 + self,
100 + config: MixtralConfig,
101 + linear_method: Optional[LinearMethodBase] = None,
102 + ):
103 + super().__init__()
104 + self.config = config
105 + self.rank = get_tensor_model_parallel_rank()
106 + self.tp_size = get_tensor_model_parallel_world_size()
107 + self.num_total_experts = config.num_local_experts
108 + self.top_k = config.num_experts_per_tok
109 + if self.tp_size > self.num_total_experts:
110 + raise ValueError(
111 + f"Tensor parallel size {self.tp_size} is greater than "
112 + f"the number of experts {self.num_total_experts}.")
113 + # Split experts equally between ranks
114 + self.expert_indicies = np.array_split(range(
115 + self.num_total_experts), self.tp_size)[self.rank].tolist()
116 + if not self.expert_indicies:
117 + raise ValueError(
118 + f"Rank {self.rank} has no experts assigned to it.")
119 +
120 + self.experts = nn.ModuleList([
121 + MixtralMLP(self.num_total_experts,
122 + config.hidden_size,
123 + config.intermediate_size,
124 + linear_method=linear_method)
125 + if idx in self.expert_indicies else None
126 + for idx in range(self.num_total_experts)
127 + ])
128 + self.gate = ReplicatedLinear(config.hidden_size,
129 + self.num_total_experts,
130 + bias=False,
131 + linear_method=None)
132 +
133 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
134 + batch_size, sequence_length, hidden_dim = hidden_states.shape
135 + hidden_states = hidden_states.view(-1, hidden_dim)
136 + # router_logits: (batch * sequence_length, n_experts)
137 + router_logits, _ = self.gate(hidden_states)
138 +
139 + routing_weights = F.softmax(router_logits, dim=1, dtype=torch.float)
140 + routing_weights, selected_experts = torch.topk(routing_weights,
141 + self.top_k,
142 + dim=-1)
143 + routing_weights /= routing_weights.sum(dim=-1, keepdim=True)
144 +
145 + final_hidden_states = None
146 + for expert_idx in self.expert_indicies:
147 + expert_layer = self.experts[expert_idx]
148 + expert_mask = (selected_experts == expert_idx)
149 + expert_weights = (routing_weights * expert_mask).sum(dim=-1,
150 + keepdim=True)
151 +
152 + current_hidden_states = expert_layer(hidden_states).mul_(
153 + expert_weights)
154 + if final_hidden_states is None:
155 + final_hidden_states = current_hidden_states
156 + else:
157 + final_hidden_states.add_(current_hidden_states)
158 +
159 + return tensor_model_parallel_all_reduce(final_hidden_states).view(
160 + batch_size, sequence_length, hidden_dim)
161 +
162 +
163 +class MixtralAttention(nn.Module):
164 +
165 + def __init__(self,
166 + hidden_size: int,
167 + num_heads: int,
168 + num_kv_heads: int,
169 + max_position: int = 4096 * 32,
170 + rope_theta: float = 10000,
171 + linear_method: Optional[LinearMethodBase] = None,
172 + sliding_window: Optional[int] = None) -> None:
173 + super().__init__()
174 + self.hidden_size = hidden_size
175 + tp_size = get_tensor_model_parallel_world_size()
176 + self.total_num_heads = num_heads
177 + assert self.total_num_heads % tp_size == 0
178 + self.num_heads = self.total_num_heads // tp_size
179 + self.total_num_kv_heads = num_kv_heads
180 + if self.total_num_kv_heads >= tp_size:
181 + # Number of KV heads is greater than TP size, so we partition
182 + # the KV heads across multiple tensor parallel GPUs.
183 + assert self.total_num_kv_heads % tp_size == 0
184 + else:
185 + # Number of KV heads is less than TP size, so we replicate
186 + # the KV heads across multiple tensor parallel GPUs.
187 + assert tp_size % self.total_num_kv_heads == 0
188 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
189 + self.head_dim = hidden_size // self.total_num_heads
190 + self.q_size = self.num_heads * self.head_dim
191 + self.kv_size = self.num_kv_heads * self.head_dim
192 + self.scaling = self.head_dim**-0.5
193 + self.rope_theta = rope_theta
194 + self.sliding_window = sliding_window
195 +
196 + self.qkv_proj = QKVParallelLinear(
197 + hidden_size,
198 + self.head_dim,
199 + self.total_num_heads,
200 + self.total_num_kv_heads,
201 + bias=False,
202 + linear_method=linear_method,
203 + )
204 + self.o_proj = RowParallelLinear(
205 + self.total_num_heads * self.head_dim,
206 + hidden_size,
207 + bias=False,
208 + linear_method=linear_method,
209 + )
210 + self.rotary_emb = get_rope(
211 + self.head_dim,
212 + rotary_dim=self.head_dim,
213 + max_position=max_position,
214 + base=int(self.rope_theta),
215 + is_neox_style=True,
216 + )
217 + self.attn = PagedAttention(
218 + self.num_heads,
219 + self.head_dim,
220 + self.scaling,
221 + num_kv_heads=self.num_kv_heads,
222 + sliding_window=self.sliding_window,
223 + )
224 +
225 + def forward(
226 + self,
227 + positions: torch.Tensor,
228 + hidden_states: torch.Tensor,
229 + kv_cache: KVCache,
230 + input_metadata: InputMetadata,
231 + ) -> torch.Tensor:
232 + qkv, _ = self.qkv_proj(hidden_states)
233 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
234 + q, k = self.rotary_emb(positions, q, k)
235 + k_cache, v_cache = kv_cache
236 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
237 + output, _ = self.o_proj(attn_output)
238 + return output
239 +
240 +
241 +class MixtralDecoderLayer(nn.Module):
242 +
243 + def __init__(
244 + self,
245 + config: MixtralConfig,
246 + linear_method: Optional[LinearMethodBase] = None,
247 + ) -> None:
248 + super().__init__()
249 + self.hidden_size = config.hidden_size
250 + # Requires transformers > 4.32.0
251 + rope_theta = getattr(config, "rope_theta", 10000)
252 + self.self_attn = MixtralAttention(
253 + hidden_size=self.hidden_size,
254 + num_heads=config.num_attention_heads,
255 + max_position=config.max_position_embeddings,
256 + num_kv_heads=config.num_key_value_heads,
257 + rope_theta=rope_theta,
258 + sliding_window=config.sliding_window,
259 + linear_method=linear_method)
260 + self.block_sparse_moe = MixtralMoE(config=config,
261 + linear_method=linear_method)
262 + self.input_layernorm = RMSNorm(config.hidden_size,
263 + eps=config.rms_norm_eps)
264 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
265 + eps=config.rms_norm_eps)
266 +
267 + def forward(
268 + self,
269 + positions: torch.Tensor,
270 + hidden_states: torch.Tensor,
271 + kv_cache: KVCache,
272 + input_metadata: InputMetadata,
273 + residual: Optional[torch.Tensor],
274 + ) -> torch.Tensor:
275 + # Self Attention
276 + if residual is None:
277 + residual = hidden_states
278 + hidden_states = self.input_layernorm(hidden_states)
279 + else:
280 + hidden_states, residual = self.input_layernorm(
281 + hidden_states, residual)
282 + hidden_states = self.self_attn(
283 + positions=positions,
284 + hidden_states=hidden_states,
285 + kv_cache=kv_cache,
286 + input_metadata=input_metadata,
287 + )
288 +
289 + # Fully Connected
290 + hidden_states, residual = self.post_attention_layernorm(
291 + hidden_states, residual)
292 + hidden_states = self.block_sparse_moe(hidden_states)
293 + return hidden_states, residual
294 +
295 +
296 +class MixtralModel(nn.Module):
297 +
298 + def __init__(
299 + self,
300 + config: MixtralConfig,
301 + linear_method: Optional[LinearMethodBase] = None,
302 + ) -> None:
303 + super().__init__()
304 + self.padding_idx = config.pad_token_id
305 + self.vocab_size = config.vocab_size
306 +
307 + self.embed_tokens = VocabParallelEmbedding(
308 + config.vocab_size,
309 + config.hidden_size,
310 + )
311 + self.layers = nn.ModuleList([
312 + MixtralDecoderLayer(config, linear_method=linear_method)
313 + for _ in range(config.num_hidden_layers)
314 + ])
315 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
316 +
317 + def forward(
318 + self,
319 + input_ids: torch.Tensor,
320 + positions: torch.Tensor,
321 + kv_caches: List[KVCache],
322 + input_metadata: InputMetadata,
323 + ) -> torch.Tensor:
324 + hidden_states = self.embed_tokens(input_ids)
325 + residual = None
326 + for i in range(len(self.layers)):
327 + layer = self.layers[i]
328 + hidden_states, residual = layer(positions, hidden_states,
329 + kv_caches[i], input_metadata,
330 + residual)
331 + hidden_states, _ = self.norm(hidden_states, residual)
332 + return hidden_states
333 +
334 +
335 +class MixtralForCausalLM(nn.Module):
336 +
337 + def __init__(
338 + self,
339 + config: MixtralConfig,
340 + linear_method: Optional[LinearMethodBase] = None,
341 + ) -> None:
342 + super().__init__()
343 + self.config = config
344 + self.linear_method = linear_method
345 + self.model = MixtralModel(config, linear_method)
346 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
347 + self.sampler = Sampler(config.vocab_size)
348 +
349 + def forward(
350 + self,
351 + input_ids: torch.Tensor,
352 + positions: torch.Tensor,
353 + kv_caches: List[KVCache],
354 + input_metadata: InputMetadata,
355 + ) -> torch.Tensor:
356 + hidden_states = self.model(input_ids, positions, kv_caches,
357 + input_metadata)
358 + return hidden_states
359 +
360 + def sample(
361 + self,
362 + hidden_states: Optional[torch.Tensor],
363 + sampling_metadata: SamplingMetadata,
364 + ) -> Optional[SamplerOutput]:
365 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
366 + sampling_metadata)
367 + return next_tokens
368 +
369 + def load_weights(self,
370 + model_name_or_path: str,
371 + cache_dir: Optional[str] = None,
372 + load_format: str = "auto",
373 + revision: Optional[str] = None):
374 + stacked_params_mapping = [
375 + # (param_name, shard_name, shard_id)
376 + ("qkv_proj", "q_proj", "q"),
377 + ("qkv_proj", "k_proj", "k"),
378 + ("qkv_proj", "v_proj", "v"),
379 + ]
380 +
381 + params_dict = dict(self.named_parameters())
382 + for name, loaded_weight in hf_model_weights_iterator(
383 + model_name_or_path,
384 + cache_dir,
385 + load_format,
386 + revision,
387 + fall_back_to_pt=False):
388 + if "rotary_emb.inv_freq" in name:
389 + continue
390 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
391 + if weight_name not in name:
392 + continue
393 + name = name.replace(weight_name, param_name)
394 + # Skip loading extra bias for GPTQ models.
395 + if name.endswith(".bias") and name not in params_dict:
396 + continue
397 + param = params_dict[name]
398 + weight_loader = param.weight_loader
399 + weight_loader(param, loaded_weight, shard_id)
400 + break
401 + else:
402 + # Skip loading extra bias for GPTQ models.
403 + if name.endswith(".bias") and name not in params_dict:
404 + continue
405 + # Skip experts that are not assigned to this worker.
406 + if ("block_sparse_moe.experts." in name
407 + and name not in params_dict):
408 + continue
409 + param = params_dict[name]
410 + weight_loader = getattr(param, "weight_loader",
411 + default_weight_loader)
412 + weight_loader(param, loaded_weight)