@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c008c51a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Refactor MoE to isolate Fp8 From Mixtral (#5970)

Co-authored-by: Robert Shaw <rshaw@neuralmagic> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Robert Shaw committed Jul 2, 2024 at 17:54 UTC 7c008c51a9aa4c0d53d09ab3a1ba61ecec354565
10 files changed +537 -306
.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic -b "auto" -l 250 -f 5 -t 8
2 +model_name: "neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.86
8 + - name: "exact_match,flexible-extract"
9 + value: 0.86
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 -b "auto" -l 250 -f 5 -t 4
2 +model_name: "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.624
8 + - name: "exact_match,flexible-extract"
9 + value: 0.624
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash ./run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4
2 +model_name: "Qwen/Qwen2-57B-A14B-Instruct"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.792
8 + - name: "exact_match,flexible-extract"
9 + value: 0.824
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,2 +1,3 @@
1 Meta-Llama-3-70B-Instruct.yaml
2 Mixtral-8x7B-Instruct-v0.1.yaml
3 +Qwen2-57B-A14-Instruct.yaml
tests/kernels/test_moe.py
+2 -2
@@ -77,8 +77,8 @@ def test_mixtral_moe(dtype: torch.dtype):
77 for i in range(config.num_local_experts):
78 weights = (hf_moe.experts[i].w1.weight.data,
79 hf_moe.experts[i].w3.weight.data)
80 - vllm_moe.w13_weight[i][:] = torch.cat(weights, dim=0)
81 - vllm_moe.w2_weight[i][:] = hf_moe.experts[i].w2.weight.data
80 + vllm_moe.experts.w13_weight[i][:] = torch.cat(weights, dim=0)
81 + vllm_moe.experts.w2_weight[i][:] = hf_moe.experts[i].w2.weight.data
82
83 # Generate input batch of dimensions [batch_size, seq_len, hidden_dim]
84 hf_inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
vllm/model_executor/layers/fused_moe/__init__.py
+4
@@ -1,5 +1,7 @@
1 from vllm.model_executor.layers.fused_moe.fused_moe import (
2 fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
3 +from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
4 + FusedMoEMethodBase)
5
6 __all__ = [
7 "fused_moe",
@@ -7,4 +9,6 @@ __all__ = [
9 "fused_experts",
10 "get_config_file_name",
11 "grouped_topk",
12 + "FusedMoE",
13 + "FusedMoEMethodBase",
14 ]
vllm/model_executor/layers/fused_moe/layer.py new
+197
@@ -0,0 +1,197 @@
1 +from abc import abstractmethod
2 +from typing import Optional
3 +
4 +import torch
5 +
6 +from vllm.distributed import (get_tensor_model_parallel_rank,
7 + get_tensor_model_parallel_world_size,
8 + tensor_model_parallel_all_reduce)
9 +from vllm.logger import init_logger
10 +from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
11 +from vllm.model_executor.layers.quantization.base_config import (
12 + QuantizationConfig, QuantizeMethodBase)
13 +from vllm.model_executor.utils import set_weight_attrs
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +class FusedMoEMethodBase(QuantizeMethodBase):
19 +
20 + @abstractmethod
21 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
22 + hidden_size: int, intermediate_size: int,
23 + params_dtype: torch.dtype, **extra_weight_attrs):
24 + raise NotImplementedError
25 +
26 + @abstractmethod
27 + def apply(self,
28 + layer: torch.nn.Module,
29 + x: torch.Tensor,
30 + router_logits: torch.Tensor,
31 + top_k: int,
32 + renormalize: bool = True) -> torch.Tensor:
33 + raise NotImplementedError
34 +
35 +
36 +class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
37 + """MoE method without quantization."""
38 +
39 + def create_weights(self, layer: torch.nn.Module, num_experts: int,
40 + hidden_size: int, intermediate_size: int,
41 + params_dtype: torch.dtype, **extra_weight_attrs):
42 +
43 + # Fused gate_up_proj (column parallel)
44 + w13_weight = torch.nn.Parameter(torch.empty(num_experts,
45 + 2 * intermediate_size,
46 + hidden_size,
47 + dtype=params_dtype),
48 + requires_grad=False)
49 + layer.register_parameter("w13_weight", w13_weight)
50 + set_weight_attrs(w13_weight, extra_weight_attrs)
51 +
52 + # down_proj (row parallel)
53 + w2_weight = torch.nn.Parameter(torch.empty(num_experts,
54 + hidden_size,
55 + intermediate_size,
56 + dtype=params_dtype),
57 + requires_grad=False)
58 + layer.register_parameter("w2_weight", w2_weight)
59 + set_weight_attrs(w2_weight, extra_weight_attrs)
60 +
61 + def apply(self,
62 + layer: torch.nn.Module,
63 + x: torch.Tensor,
64 + router_logits: torch.Tensor,
65 + top_k: int,
66 + renormalize: bool = True) -> torch.Tensor:
67 +
68 + return fused_moe(x,
69 + layer.w13_weight,
70 + layer.w2_weight,
71 + router_logits,
72 + top_k,
73 + renormalize=renormalize,
74 + inplace=True)
75 +
76 +
77 +class FusedMoE(torch.nn.Module):
78 + """FusedMoE layer for MoE models.
79 +
80 + This layer contains both MergedColumnParallel weights (gate_up_proj /
81 + w13) and RowParallelLinear weights (down_proj/ w2).
82 +
83 + Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We
84 + copy that naming convention here and handle any remapping in the
85 + load_weights function in each model implementation.
86 +
87 + Args:
88 + num_experts: Number of experts in the model
89 + top_k: Number of experts selected for each token
90 + hidden_size: Input hidden state size of the transformer
91 + intermediate_size: Intermediate size of the experts
92 + params_dtype: Data type for the parameters.
93 + reduce_results: Whether to all all_reduce on the output of the layer
94 + renomalize: Whether to renormalize the logits in the fused_moe kernel
95 + quant_config: Quantization configure.
96 + """
97 +
98 + def __init__(
99 + self,
100 + num_experts: int,
101 + top_k: int,
102 + hidden_size: int,
103 + intermediate_size: int,
104 + params_dtype: Optional[torch.dtype] = None,
105 + reduce_results: bool = False,
106 + renormalize: bool = True,
107 + quant_config: Optional[QuantizationConfig] = None,
108 + tp_size: Optional[int] = None,
109 + ):
110 + super().__init__()
111 +
112 + if params_dtype is None:
113 + params_dtype = torch.get_default_dtype()
114 +
115 + self.tp_size = (tp_size if tp_size is not None else
116 + get_tensor_model_parallel_world_size())
117 + self.top_k = top_k
118 + self.num_experts = num_experts
119 + self.intermediate_size_per_partition = intermediate_size // self.tp_size
120 + self.reduce_results = reduce_results
121 + self.renormalize = renormalize
122 +
123 + if quant_config is None:
124 + self.quant_method: Optional[QuantizeMethodBase] = (
125 + UnquantizedFusedMoEMethod())
126 + else:
127 + self.quant_method = quant_config.get_quant_method(self)
128 + assert self.quant_method is not None
129 +
130 + self.quant_method.create_weights(
131 + layer=self,
132 + num_experts=num_experts,
133 + hidden_size=hidden_size,
134 + intermediate_size=self.intermediate_size_per_partition,
135 + params_dtype=params_dtype,
136 + weight_loader=self.weight_loader)
137 +
138 + def weight_loader(self, param: torch.nn.Parameter,
139 + loaded_weight: torch.Tensor, weight_name: str,
140 + shard_id: int, expert_id: int):
141 + param_data = param.data
142 +
143 + # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
144 + # Follow up PR to enable fp8 for other MoE models.
145 + if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
146 + if param_data[expert_id] != 1 and (param_data[expert_id] -
147 + loaded_weight).abs() > 1e-5:
148 + raise ValueError(
149 + "input_scales of w1 and w3 of a layer "
150 + f"must be equal. But got {param_data[expert_id]} "
151 + f"vs. {loaded_weight}")
152 + param_data[expert_id] = loaded_weight
153 + # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
154 + # Follow up PR to enable fp8 for other MoE models.
155 + elif "weight_scale" in weight_name:
156 + # We have to keep the weight scales of w1 and w3 because
157 + # we need to re-quantize w1/w3 weights after weight loading.
158 + assert "w1" in weight_name or "w3" in weight_name
159 + shard_id = 0 if "w1" in weight_name else 1
160 + param_data[expert_id][shard_id] = loaded_weight
161 + else:
162 + tp_rank = get_tensor_model_parallel_rank()
163 + shard_size = self.intermediate_size_per_partition
164 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
165 +
166 + # w1, gate_proj case: Load into first shard of w13.
167 + if shard_id == 0:
168 + param_data[expert_id,
169 + 0:shard_size, :] = loaded_weight[shard, :]
170 + # w3, up_proj case: Load into second shard of w13.
171 + elif shard_id == 2:
172 + param_data[expert_id, shard_size:2 *
173 + shard_size, :] = loaded_weight[shard, :]
174 + # w2, down_proj case: Load into only shard of w2.
175 + elif shard_id == 1:
176 + param_data[expert_id, :, :] = loaded_weight[:, shard]
177 + else:
178 + raise ValueError(
179 + f"Shard id must be in [0,1,2] but got {shard_id}")
180 +
181 + def forward(self, hidden_states: torch.Tensor,
182 + router_logits: torch.Tensor):
183 + assert self.quant_method is not None
184 +
185 + # Matrix multiply.
186 + final_hidden_states = self.quant_method.apply(
187 + self,
188 + x=hidden_states,
189 + router_logits=router_logits,
190 + top_k=self.top_k,
191 + renormalize=self.renormalize)
192 +
193 + if self.reduce_results and self.tp_size > 1:
194 + final_hidden_states = tensor_model_parallel_all_reduce(
195 + final_hidden_states)
196 +
197 + return final_hidden_states
vllm/model_executor/layers/quantization/fp8.py
+191 -1
@@ -6,6 +6,8 @@ from torch.nn.parameter import Parameter
6
7 from vllm import _custom_ops as ops
8 from vllm.logger import init_logger
9 +from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
10 + fused_moe)
11 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
@@ -71,7 +73,9 @@ class Fp8Config(QuantizationConfig):
73
74 if isinstance(layer, LinearBase):
75 return Fp8LinearMethod(self)
74 - if isinstance(layer, Attention):
76 + elif isinstance(layer, FusedMoE):
77 + return Fp8MoEMethod(self)
78 + elif isinstance(layer, Attention):
79 return Fp8KVCacheMethod(self)
80 return None
81
@@ -270,6 +274,187 @@ class Fp8LinearMethod(LinearMethodBase):
274 return torch.narrow(output, 0, 0, x.shape[0])
275
276
277 +class Fp8MoEMethod(FusedMoEMethodBase):
278 + """MoE method for FP8.
279 + Supports loading FP8 checkpoints with static weight scale and
280 + dynamic/static activation scale.
281 +
282 + Also supports loading quantized FP16/BF16 model checkpoints with dynamic
283 + activation scaling. The weight scaling factor will be initialized after
284 + the model weights are loaded.
285 +
286 + Args:
287 + quant_config: The quantization config.
288 + """
289 +
290 + def __init__(self, quant_config: Fp8Config):
291 + self.quant_config = quant_config
292 +
293 + def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
294 + intermediate_size: int, params_dtype: torch.dtype,
295 + **extra_weight_attrs):
296 +
297 + layer.process_after_load = True
298 +
299 + if self.quant_config.is_checkpoint_fp8_serialized:
300 + params_dtype = torch.float8_e4m3fn
301 +
302 + # WEIGHTS
303 + w13_weight = torch.nn.Parameter(torch.empty(num_experts,
304 + 2 * intermediate_size,
305 + hidden_size,
306 + dtype=params_dtype),
307 + requires_grad=False)
308 + layer.register_parameter("w13_weight", w13_weight)
309 + set_weight_attrs(w13_weight, extra_weight_attrs)
310 +
311 + w2_weight = torch.nn.Parameter(torch.empty(num_experts,
312 + hidden_size,
313 + intermediate_size,
314 + dtype=params_dtype),
315 + requires_grad=False)
316 + layer.register_parameter("w2_weight", w2_weight)
317 + set_weight_attrs(w2_weight, extra_weight_attrs)
318 +
319 + # WEIGHT_SCALES
320 + # Allocate 2 scales for w1 and w3 respectively.
321 + # They will be combined to a single scale after weight loading.
322 + w13_scale = torch.nn.Parameter(torch.ones(num_experts,
323 + 2,
324 + dtype=torch.float32),
325 + requires_grad=False)
326 + layer.register_parameter("w13_scale", w13_scale)
327 +
328 + w2_scale = torch.nn.Parameter(torch.ones(num_experts,
329 + dtype=torch.float32),
330 + requires_grad=False)
331 + layer.register_parameter("w2_scale", w2_scale)
332 +
333 + # If loading fp8 checkpoint, pass the weight loaders.
334 + # If loading an fp16 checkpoint, do not (we will quantize in
335 + # process_weights_after_loading()
336 + if self.quant_config.is_checkpoint_fp8_serialized:
337 + set_weight_attrs(w13_scale, extra_weight_attrs)
338 + set_weight_attrs(w2_scale, extra_weight_attrs)
339 +
340 + # INPUT_SCALES
341 + if self.quant_config.activation_scheme == "static":
342 + if not self.quant_config.is_checkpoint_fp8_serialized:
343 + raise ValueError(
344 + "Found static activation scheme for checkpoint that "
345 + "was not serialized fp8.")
346 +
347 + a13_scale = torch.nn.Parameter(torch.ones(num_experts,
348 + dtype=torch.float32),
349 + requires_grad=False)
350 + layer.register_parameter("a13_scale", a13_scale)
351 + set_weight_attrs(a13_scale, extra_weight_attrs)
352 +
353 + a2_scale = torch.nn.Parameter(torch.ones(num_experts,
354 + dtype=torch.float32),
355 + requires_grad=False)
356 + layer.register_parameter("a2_scale", a2_scale)
357 + set_weight_attrs(a2_scale, extra_weight_attrs)
358 + else:
359 + layer.a13_scale = None
360 + layer.a2_scale = None
361 +
362 + def process_weights_after_loading(self, layer: Module) -> None:
363 + if (not hasattr(layer, "process_after_load")
364 + or not layer.process_after_load):
365 + return
366 +
367 + # If checkpoint is fp16, quantize in place.
368 + if not self.quant_config.is_checkpoint_fp8_serialized:
369 + w13_weight = torch.empty_like(layer.w13_weight.data,
370 + dtype=torch.float8_e4m3fn)
371 + w2_weight = torch.empty_like(layer.w2_weight.data,
372 + dtype=torch.float8_e4m3fn)
373 +
374 + # Re-initialize w13_scale because we directly quantize
375 + # merged w13 weights and generate a single scaling factor.
376 + layer.w13_scale = torch.nn.Parameter(torch.ones(
377 + layer.num_experts,
378 + dtype=torch.float32,
379 + device=w13_weight.device),
380 + requires_grad=False)
381 + for expert in range(layer.num_experts):
382 + w13_weight[expert, :, :], layer.w13_scale[
383 + expert] = ops.scaled_fp8_quant(
384 + layer.w13_weight.data[expert, :, :])
385 + w2_weight[expert, :, :], layer.w2_scale[
386 + expert] = ops.scaled_fp8_quant(
387 + layer.w2_weight.data[expert, :, :])
388 + layer.w13_weight = torch.nn.Parameter(w13_weight,
389 + requires_grad=False)
390 + layer.w2_weight = torch.nn.Parameter(w2_weight,
391 + requires_grad=False)
392 + return
393 +
394 + # If checkpoint is fp8, we need to handle that the
395 + # MoE kernels require single activation scale and single weight
396 + # scale for w13 per expert.
397 + else:
398 + # Fp8 moe kernels require a single activation scale.
399 + # We take the max of all the scales in case they differ.
400 + if self.quant_config.activation_scheme == "static":
401 + if layer.a13_scale is None or layer.a2_scale is None:
402 + raise ValueError(
403 + "QuantConfig has static quantization, but found "
404 + "activation scales are None.")
405 + if (not all_close_1d(layer.a13_scale)
406 + or not all_close_1d(layer.a2_scale)):
407 + print_warning_once(
408 + "Found input_scales that are not equal for "
409 + "fp8 MoE layer. Using the maximum across experts "
410 + "for each layer. ")
411 + layer.a13_scale = torch.nn.Parameter(layer.a13_scale.max(),
412 + requires_grad=False)
413 + layer.a2_scale = torch.nn.Parameter(layer.a2_scale.max(),
414 + requires_grad=False)
415 +
416 + # Fp8 moe kernel needs single weight scale for w13 per expert.
417 + # We take the max then dequant and requant each expert.
418 + assert layer.w13_scale is not None
419 + shard_size = layer.intermediate_size_per_partition
420 + max_w13_scales = layer.w13_scale.max(dim=1).values
421 + for expert_id in range(layer.num_experts):
422 + start = 0
423 + for shard_id in range(2):
424 + dq_weight = per_tensor_dequantize(
425 + layer.w13_weight[expert_id][start:start +
426 + shard_size, :],
427 + layer.w13_scale[expert_id][shard_id])
428 + layer.w13_weight[expert_id][
429 + start:start + shard_size, :] = per_tensor_quantize(
430 + dq_weight, max_w13_scales[expert_id])
431 + start += shard_size
432 +
433 + layer.w13_scale = torch.nn.Parameter(max_w13_scales,
434 + requires_grad=False)
435 + return
436 +
437 + def apply(self,
438 + layer: torch.nn.Module,
439 + x: torch.Tensor,
440 + router_logits: torch.Tensor,
441 + top_k: int,
442 + renormalize: bool = True) -> torch.Tensor:
443 +
444 + return fused_moe(x,
445 + layer.w13_weight,
446 + layer.w2_weight,
447 + router_logits,
448 + top_k,
449 + renormalize=renormalize,
450 + inplace=True,
451 + use_fp8=True,
452 + w1_scale=layer.w13_scale,
453 + w2_scale=layer.w2_scale,
454 + a1_scale=layer.a13_scale,
455 + a2_scale=layer.a2_scale)
456 +
457 +
458 class Fp8KVCacheMethod(QuantizeMethodBase):
459 """Supports loading kv-cache scaling factors from FP8 checkpoints.
460 """
@@ -321,3 +506,8 @@ def per_tensor_dequantize(
506 fake_qweight = tensor.to(torch.float16)
507 dq_weight = fake_qweight * inv_scale
508 return dq_weight
509 +
510 +
511 +def all_close_1d(x: torch.Tensor) -> bool:
512 + assert len(x.shape) == 1
513 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/mixtral.py
+41 -235
@@ -27,13 +27,10 @@ import torch
27 from torch import nn
28 from transformers import MixtralConfig
29
30 -from vllm import _custom_ops as ops
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig, LoRAConfig
33 -from vllm.distributed import (get_tensor_model_parallel_rank,
34 - get_tensor_model_parallel_world_size,
35 - tensor_model_parallel_all_reduce)
36 -from vllm.model_executor.layers.fused_moe import fused_moe
32 +from vllm.distributed import get_tensor_model_parallel_world_size
33 +from vllm.model_executor.layers.fused_moe import FusedMoE
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (QKVParallelLinear,
36 ReplicatedLinear,
@@ -41,16 +38,12 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
38 from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization.base_config import (
40 QuantizationConfig)
44 -from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,
45 - per_tensor_dequantize,
46 - per_tensor_quantize)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 -from vllm.model_executor.utils import set_weight_attrs
47 from vllm.sequence import IntermediateTensors, SamplerOutput
48 from vllm.utils import print_warning_once
49
@@ -66,227 +59,40 @@ class MixtralMoE(nn.Module):
59 across ranks.
60 """
61
69 - def __init__(
70 - self,
71 - num_experts: int,
72 - top_k: int,
73 - hidden_size: int,
74 - intermediate_size: int,
75 - params_dtype: Optional[torch.dtype] = None,
76 - tp_size: Optional[int] = None,
77 - quant_config: Optional[QuantizationConfig] = None,
78 - ):
62 + def __init__(self,
63 + num_experts: int,
64 + top_k: int,
65 + hidden_size: int,
66 + intermediate_size: int,
67 + params_dtype: Optional[torch.dtype] = None,
68 + quant_config: Optional[QuantizationConfig] = None,
69 + tp_size: Optional[int] = None):
70 super().__init__()
80 - self.tp_size = tp_size or get_tensor_model_parallel_world_size()
81 - self.num_total_experts = num_experts
82 - self.top_k = top_k
71 self.hidden_size = hidden_size
84 - self.intermediate_size = intermediate_size // self.tp_size
85 - self.quant_config = quant_config
86 -
87 - # FIXME(pcmoritz): Make this more general to support different
88 - # quantization schemes
89 - self.use_fp8 = isinstance(quant_config, Fp8Config)
90 -
91 - if params_dtype is None:
92 - params_dtype = torch.get_default_dtype()
93 - self.params_dtype = params_dtype
72
73 # Gate always runs at half / full precision for now.
96 - self.gate = ReplicatedLinear(self.hidden_size,
97 - self.num_total_experts,
74 + self.gate = ReplicatedLinear(hidden_size,
75 + num_experts,
76 bias=False,
99 - params_dtype=self.params_dtype,
77 + params_dtype=params_dtype,
78 quant_config=None)
79
102 - if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
103 - params_dtype = torch.float8_e4m3fn
104 -
105 - self.w13_weight = nn.Parameter(torch.empty(self.num_total_experts,
106 - 2 * self.intermediate_size,
107 - self.hidden_size,
108 - dtype=params_dtype),
109 - requires_grad=False)
110 - self.w2_weight = nn.Parameter(torch.empty(self.num_total_experts,
111 - self.hidden_size,
112 - self.intermediate_size,
113 - dtype=params_dtype),
114 - requires_grad=False)
115 -
116 - set_weight_attrs(self.w13_weight, {
117 - "weight_loader": self.weight_loader,
118 - })
119 - set_weight_attrs(self.w2_weight, {
120 - "weight_loader": self.weight_loader,
121 - })
122 -
123 - # Used for fp8.
124 - self.w13_scale = None
125 - self.w2_scale = None
126 - self.a13_scale = None
127 - self.a2_scale = None
128 -
129 - if self.use_fp8:
130 - # WEIGHT_SCALE (for fp8)
131 - # Allocate 2 scales for w1 and w3 respectively.
132 - # They will be combined to a single scale after weight loading.
133 - self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
134 - 2,
135 - dtype=torch.float32),
136 - requires_grad=False)
137 - self.w2_scale = nn.Parameter(torch.ones(self.num_total_experts,
138 - dtype=torch.float32),
139 - requires_grad=False)
140 -
141 - # If loading fp8 checkpoint, pass the weight loaders.
142 - # If loading an fp16 checkpoint, do not (we will quantize in
143 - # process_weights_after_loading()
144 - if quant_config.is_checkpoint_fp8_serialized:
145 - set_weight_attrs(self.w13_scale, {
146 - "weight_loader": self.weight_loader,
147 - })
148 - set_weight_attrs(self.w2_scale, {
149 - "weight_loader": self.weight_loader,
150 - })
151 -
152 - # INPUT_SCALE (for fp8)
153 - if quant_config.activation_scheme == "static":
154 - if not quant_config.is_checkpoint_fp8_serialized:
155 - raise ValueError(
156 - "Found static activation scheme for checkpoint that "
157 - "was not serialized fp8.")
158 - self.a13_scale = nn.Parameter(torch.ones(
159 - self.num_total_experts, dtype=torch.float32),
160 - requires_grad=False)
161 - self.a2_scale = nn.Parameter(torch.ones(self.num_total_experts,
162 - dtype=torch.float32),
163 - requires_grad=False)
164 -
165 - set_weight_attrs(self.a13_scale, {
166 - "weight_loader": self.weight_loader,
167 - })
168 - set_weight_attrs(self.a2_scale, {
169 - "weight_loader": self.weight_loader,
170 - })
171 -
172 - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
173 - weight_name: str, expert_id: int):
174 - tp_rank = get_tensor_model_parallel_rank()
175 - param_data = param.data
176 - shard_size = self.intermediate_size
177 - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
178 - if weight_name.endswith("w1.weight"):
179 - param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
180 - if weight_name.endswith("w3.weight"):
181 - param_data[expert_id,
182 - shard_size:2 * shard_size, :] = loaded_weight[shard, :]
183 - if weight_name.endswith("w2.weight"):
184 - param_data[expert_id, :, :] = loaded_weight[:, shard]
185 -
186 - # Loading scales
187 - if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
188 - if param_data[expert_id] != 1 and (param_data[expert_id] -
189 - loaded_weight).abs() > 1e-5:
190 - raise ValueError(
191 - "input_scales of w1 and w3 of a layer "
192 - f"must be equal. But got {param_data[expert_id]} "
193 - f"vs. {loaded_weight}")
194 - param_data[expert_id] = loaded_weight
195 - elif "weight_scale" in weight_name:
196 - # We have to keep the weight scales of w1 and w3 because
197 - # we need to re-quantize w1/w3 weights after weight loading.
198 - assert "w1" in weight_name or "w3" in weight_name
199 - shard_id = 0 if "w1" in weight_name else 1
200 - param_data[expert_id][shard_id] = loaded_weight
201 -
202 - def process_weights_after_loading(self):
203 - # Fp8 is the only case where we need to process after loading.
204 - if not self.use_fp8:
205 - return
206 -
207 - # If checkpoint is fp16, quantize here.
208 - if not self.quant_config.is_checkpoint_fp8_serialized:
209 - w13_weight = torch.empty_like(self.w13_weight.data,
210 - dtype=torch.float8_e4m3fn)
211 - w2_weight = torch.empty_like(self.w2_weight.data,
212 - dtype=torch.float8_e4m3fn)
213 -
214 - # Re-initialize w13_scale because we directly quantize
215 - # merged w13 weights and generate a single scaling factor.
216 - self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
217 - dtype=torch.float32),
218 - requires_grad=False)
219 - for expert in range(self.num_total_experts):
220 - w13_weight[expert, :, :], self.w13_scale[
221 - expert] = ops.scaled_fp8_quant(
222 - self.w13_weight.data[expert, :, :])
223 - w2_weight[expert, :, :], self.w2_scale[
224 - expert] = ops.scaled_fp8_quant(
225 - self.w2_weight.data[expert, :, :])
226 - self.w13_weight = nn.Parameter(w13_weight, requires_grad=False)
227 - self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
228 -
229 - else:
230 - # If checkpoint is fp8 + static, cleanup input_scales.
231 - # Since state_dict has an input_scale per expert but our kernels
232 - # are passed one input_scale shared across all experts.
233 - if self.quant_config.activation_scheme == "static":
234 - if self.a13_scale is None or self.a2_scale is None:
235 - raise ValueError(
236 - "QuantConfig has static quantization, but found "
237 - "activation scales are None.")
238 -
239 - if (not all_close_1d(self.a13_scale)
240 - or not all_close_1d(self.a2_scale)):
241 - print_warning_once(
242 - "Found input_scales that are not equal for "
243 - "fp8 MoE layer. Using the maximum across experts "
244 - "for each layer. ")
245 -
246 - self.a13_scale = nn.Parameter(self.a13_scale.max(),
247 - requires_grad=False)
248 - self.a2_scale = nn.Parameter(self.a2_scale.max(),
249 - requires_grad=False)
250 -
251 - assert self.w13_scale is not None
252 - shard_size = self.intermediate_size
253 - max_w13_scales = self.w13_scale.max(dim=1).values
254 - for expert_id in range(self.num_total_experts):
255 - start = 0
256 - for shard_id in range(2):
257 - dq_weight = per_tensor_dequantize(
258 - self.w13_weight[expert_id][start:start +
259 - shard_size, :],
260 - self.w13_scale[expert_id][shard_id])
261 - self.w13_weight[expert_id][
262 - start:start + shard_size, :] = per_tensor_quantize(
263 - dq_weight, max_w13_scales[expert_id])
264 - start += shard_size
265 -
266 - self.w13_scale = nn.Parameter(max_w13_scales, requires_grad=False)
80 + self.experts = FusedMoE(num_experts=num_experts,
81 + top_k=top_k,
82 + hidden_size=hidden_size,
83 + intermediate_size=intermediate_size,
84 + params_dtype=params_dtype,
85 + reduce_results=True,
86 + renormalize=True,
87 + quant_config=quant_config,
88 + tp_size=tp_size)
89
90 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
91 num_tokens, hidden_size = hidden_states.shape
92 hidden_states = hidden_states.view(-1, self.hidden_size)
93 # router_logits: (num_tokens, n_experts)
94 router_logits, _ = self.gate(hidden_states)
273 - final_hidden_states = fused_moe(hidden_states,
274 - self.w13_weight,
275 - self.w2_weight,
276 - router_logits,
277 - self.top_k,
278 - renormalize=True,
279 - inplace=True,
280 - use_fp8=self.use_fp8,
281 - w1_scale=self.w13_scale,
282 - w2_scale=self.w2_scale,
283 - a1_scale=self.a13_scale,
284 - a2_scale=self.a2_scale)
285 -
286 - if self.tp_size > 1:
287 - final_hidden_states = tensor_model_parallel_all_reduce(
288 - final_hidden_states)
289 -
95 + final_hidden_states = self.experts(hidden_states, router_logits)
96 return final_hidden_states.view(num_tokens, hidden_size)
97
98
@@ -566,25 +372,28 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
372
373 expert_params_mapping = [
374 # These are the weight scales for the experts
569 - # (param_name, weight_name, expert_id)
570 - ("w13_scale" if weight_name in ["w1", "w3"] else "w2_scale",
571 - f"experts.{expert_id}.{weight_name}.weight_scale", expert_id)
572 - for expert_id in range(self.config.num_local_experts)
573 - for weight_name in ["w1", "w2", "w3"]
375 + # (param_name, weight_name, expert_id, shard_id)
376 + ("experts.w13_scale"
377 + if weight_name in ["w1", "w3"] else "experts.w2_scale",
378 + f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
379 + shard_id) for expert_id in range(self.config.num_local_experts)
380 + for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
381 ] + [
382 # These are the weights for the experts
383 # (param_name, weight_name, expert_id)
577 - ("w13_weight" if weight_name in ["w1", "w3"] else "w2_weight",
578 - f"experts.{expert_id}.{weight_name}.weight", expert_id)
384 + ("experts.w13_weight"
385 + if weight_name in ["w1", "w3"] else "experts.w2_weight",
386 + f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
387 for expert_id in range(self.config.num_local_experts)
580 - for weight_name in ["w1", "w2", "w3"]
388 + for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
389 ] + [
390 # These are the activation scales for the experts
391 # (param_name, weight_name, expert_id)
584 - ("a13_scale" if weight_name in ["w1", "w3"] else "a2_scale",
585 - f"experts.{expert_id}.{weight_name}.input_scale", expert_id)
586 - for expert_id in range(self.config.num_local_experts)
587 - for weight_name in ["w1", "w2", "w3"]
392 + ("experts.a13_scale"
393 + if weight_name in ["w1", "w3"] else "experts.a2_scale",
394 + f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
395 + shard_id) for expert_id in range(self.config.num_local_experts)
396 + for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
397 ]
398
399 params_dict = dict(self.named_parameters())
@@ -604,7 +413,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
413 weight_loader(param, loaded_weight, shard_id)
414 break
415 else:
607 - for param_name, weight_name, expert_id in expert_params_mapping:
416 + for mapping in expert_params_mapping:
417 + param_name, weight_name, expert_id, shard_id = mapping
418 if weight_name not in name:
419 continue
420 name = name.replace(weight_name, param_name)
@@ -613,6 +423,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
423 weight_loader(param,
424 loaded_weight,
425 weight_name,
426 + shard_id=shard_id,
427 expert_id=expert_id)
428 break
429 else:
@@ -637,8 +448,3 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
448 weight_loader = getattr(param, "weight_loader",
449 default_weight_loader)
450 weight_loader(param, loaded_weight)
640 -
641 -
642 -def all_close_1d(x: torch.Tensor) -> bool:
643 - assert len(x.shape) == 1
644 - return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/qwen2_moe.py
+68 -68
@@ -31,11 +31,10 @@ from transformers import PretrainedConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 from vllm.config import CacheConfig
34 -from vllm.distributed import (get_tensor_model_parallel_rank,
35 - get_tensor_model_parallel_world_size,
34 +from vllm.distributed import (get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
36 from vllm.model_executor.layers.activation import SiluAndMul
38 -from vllm.model_executor.layers.fused_moe import fused_moe
37 +from vllm.model_executor.layers.fused_moe import FusedMoE
38 from vllm.model_executor.layers.layernorm import RMSNorm
39 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40 QKVParallelLinear,
@@ -93,28 +92,23 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
92 quant_config: Optional[QuantizationConfig] = None,
93 ):
94 super().__init__()
96 - self.config = config
97 - self.rank = get_tensor_model_parallel_rank()
95 self.tp_size = get_tensor_model_parallel_world_size()
99 - self.n_routed_experts = config.num_experts
100 - self.top_k = config.num_experts_per_tok
101 - if self.tp_size > self.n_routed_experts:
96 +
97 + if self.tp_size > config.num_experts:
98 raise ValueError(
99 f"Tensor parallel size {self.tp_size} is greater than "
104 - f"the number of experts {self.n_routed_experts}.")
105 -
106 - self.experts = nn.ModuleList([
107 - Qwen2MoeMLP(hidden_size=config.hidden_size,
108 - intermediate_size=config.moe_intermediate_size,
109 - hidden_act=config.hidden_act,
110 - quant_config=quant_config,
111 - reduce_results=False)
112 - for idx in range(self.n_routed_experts)
113 - ])
114 - self.pack_params()
100 + f"the number of experts {config.num_experts}.")
101 +
102 + self.experts = FusedMoE(num_experts=config.num_experts,
103 + top_k=config.num_experts_per_tok,
104 + hidden_size=config.hidden_size,
105 + intermediate_size=config.moe_intermediate_size,
106 + reduce_results=False,
107 + renormalize=config.norm_topk_prob,
108 + quant_config=quant_config)
109
110 self.gate = ReplicatedLinear(config.hidden_size,
117 - self.n_routed_experts,
111 + config.num_experts,
112 bias=False,
113 quant_config=None)
114 if config.shared_expert_intermediate_size > 0:
@@ -131,25 +125,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
125 1,
126 bias=False)
127
134 - def pack_params(self):
135 - w1 = []
136 - w2 = []
137 - for expert in self.experts:
138 - w1.append(expert.gate_up_proj.weight)
139 - w2.append(expert.down_proj.weight)
140 - self.w1 = torch._utils._flatten_dense_tensors(w1)
141 - w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
142 - for data, param in zip(w1s, w1):
143 - param.data = data
144 - self.w1 = self.w1.view(len(w1), *w1s[0].shape)
145 -
146 - self.w2 = torch._utils._flatten_dense_tensors(w2)
147 - w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
148 - for data, param in zip(w2s, w2):
149 - param.data = data
150 -
151 - self.w2 = self.w2.view(len(w2), *w2s[0].shape)
152 -
128 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129 num_tokens, hidden_dim = hidden_states.shape
130 hidden_states = hidden_states.view(-1, hidden_dim)
@@ -162,18 +137,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
137
138 # router_logits: (num_tokens, n_experts)
139 router_logits, _ = self.gate(hidden_states)
165 - final_hidden_states = fused_moe(hidden_states,
166 - self.w1,
167 - self.w2,
168 - router_logits,
169 - self.top_k,
170 - renormalize=self.config.norm_topk_prob,
171 - inplace=True)
172 -
140 + final_hidden_states = self.experts(hidden_states=hidden_states,
141 + router_logits=router_logits)
142 if shared_output is not None:
143 final_hidden_states = final_hidden_states + shared_output
175 - final_hidden_states = tensor_model_parallel_all_reduce(
176 - final_hidden_states)
144 + if self.tp_size > 1:
145 + final_hidden_states = tensor_model_parallel_all_reduce(
146 + final_hidden_states)
147
148 return final_hidden_states.view(num_tokens, hidden_dim)
149
@@ -284,7 +254,12 @@ class Qwen2MoeDecoderLayer(nn.Module):
254 cache_config=cache_config,
255 quant_config=quant_config,
256 )
287 - if (layer_idx not in config.mlp_only_layers) and (
257 +
258 + # Note: Qwen/Qwen2-57B-A14B-Instruct does not have
259 + # `mlp_only_layers` in the config.
260 + mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else
261 + config.mlp_only_layers)
262 + if (layer_idx not in mlp_only_layers) and (
263 config.num_experts > 0 and
264 (layer_idx + 1) % config.decoder_sparse_step == 0):
265 self.mlp = Qwen2MoeSparseMoeBlock(config=config,
@@ -427,21 +402,36 @@ class Qwen2MoeForCausalLM(nn.Module):
402 ("gate_up_proj", "up_proj", 1),
403 ]
404
405 + expert_params_mapping = [
406 + # These are the weights for the experts
407 + # (param_name, weight_name, expert_id, shard_id)
408 + ("experts.w13_weight" if weight_name in ["gate_proj", "up_proj"]
409 + else "experts.w2_weight",
410 + f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
411 + for expert_id in range(self.config.num_experts) for shard_id,
412 + weight_name in enumerate(["gate_proj", "down_proj", "up_proj"])
413 + ]
414 +
415 params_dict = dict(self.named_parameters())
416 for name, loaded_weight in weights:
417 if "rotary_emb.inv_freq" in name:
418 continue
419 for (param_name, weight_name, shard_id) in stacked_params_mapping:
420 + # Skip non-stacked layers and experts (experts handled below).
421 if weight_name not in name:
422 continue
423 + # We have mlp.experts[0].gate_proj in the checkpoint.
424 + # Since we handle the experts below in expert_params_mapping,
425 + # we need to skip here BEFORE we update the name, otherwise
426 + # name will be updated to mlp.experts[0].gate_up_proj, which
427 + # will then be updated below in expert_params_mapping
428 + # for mlp.experts[0].gate_gate_up_proj, which breaks load.
429 + if "mlp.experts" in name:
430 + continue
431 name = name.replace(weight_name, param_name)
432 # Skip loading extra bias for GPTQ models.
433 if name.endswith(".bias") and name not in params_dict:
434 continue
441 - # Skip experts that are not assigned to this worker.
442 - if (("mlp.experts." in name or "mlp.shared_expert." in name)
443 - and name not in params_dict):
444 - continue
435 if name not in params_dict:
436 continue
437
@@ -450,17 +440,27 @@ class Qwen2MoeForCausalLM(nn.Module):
440 weight_loader(param, loaded_weight, shard_id)
441 break
442 else:
453 - # Skip loading extra bias for GPTQ models.
454 - if name.endswith(".bias") and name not in params_dict:
455 - continue
456 - # Skip experts that are not assigned to this worker.
457 - if (("mlp.experts." in name or "mlp.shared_expert." in name)
458 - and name not in params_dict):
459 - continue
460 - if name not in params_dict:
461 - continue
462 -
463 - param = params_dict[name]
464 - weight_loader = getattr(param, "weight_loader",
465 - default_weight_loader)
466 - weight_loader(param, loaded_weight)
443 + for mapping in expert_params_mapping:
444 + param_name, weight_name, expert_id, shard_id = mapping
445 + if weight_name not in name:
446 + continue
447 + name = name.replace(weight_name, param_name)
448 + param = params_dict[name]
449 + weight_loader = param.weight_loader
450 + weight_loader(param,
451 + loaded_weight,
452 + weight_name,
453 + shard_id=shard_id,
454 + expert_id=expert_id)
455 + break
456 + else:
457 + # Skip loading extra bias for GPTQ models.
458 + if name.endswith(".bias") and name not in params_dict:
459 + continue
460 + if name not in params_dict:
461 + continue
462 +
463 + param = params_dict[name]
464 + weight_loader = getattr(param, "weight_loader",
465 + default_weight_loader)
466 + weight_loader(param, loaded_weight)