A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Refactor MoE to isolate Fp8 From Mixtral (#5970)
Co-authored-by: Robert Shaw <rshaw@neuralmagic> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Robert Shaw committed
Jul 2, 2024 at 17:54 UTC
7c008c51a9aa4c0d53d09ab3a1ba61ecec354565
10 files changed
+537
-306
.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic -b "auto" -l 250 -f 5 -t 8
2
+model_name: "neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.86
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.86
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 -b "auto" -l 250 -f 5 -t 4
2
+model_name: "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.624
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.624
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash ./run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4
2
+model_name: "Qwen/Qwen2-57B-A14B-Instruct"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.792
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.824
10
+limit: 250
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,2 +1,3 @@
1
Meta-Llama-3-70B-Instruct.yaml
2
Mixtral-8x7B-Instruct-v0.1.yaml
3
+Qwen2-57B-A14-Instruct.yaml
tests/kernels/test_moe.py
+2
-2
@@ -77,8 +77,8 @@ def test_mixtral_moe(dtype: torch.dtype):
77
for i in range(config.num_local_experts):
78
weights = (hf_moe.experts[i].w1.weight.data,
79
hf_moe.experts[i].w3.weight.data)
80
- vllm_moe.w13_weight[i][:] = torch.cat(weights, dim=0)
81
- vllm_moe.w2_weight[i][:] = hf_moe.experts[i].w2.weight.data
80
+ vllm_moe.experts.w13_weight[i][:] = torch.cat(weights, dim=0)
81
+ vllm_moe.experts.w2_weight[i][:] = hf_moe.experts[i].w2.weight.data
82
83
# Generate input batch of dimensions [batch_size, seq_len, hidden_dim]
84
hf_inputs = torch.randn((1, 64, config.hidden_size)).to(dtype).to("cuda")
vllm/model_executor/layers/fused_moe/__init__.py
+4
@@ -1,5 +1,7 @@
1
from vllm.model_executor.layers.fused_moe.fused_moe import (
2
fused_experts, fused_moe, fused_topk, get_config_file_name, grouped_topk)
3
+from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
4
+ FusedMoEMethodBase)
5
6
__all__ = [
7
"fused_moe",
@@ -7,4 +9,6 @@ __all__ = [
9
"fused_experts",
10
"get_config_file_name",
11
"grouped_topk",
12
+ "FusedMoE",
13
+ "FusedMoEMethodBase",
14
]
vllm/model_executor/layers/fused_moe/layer.py
new
+197
@@ -0,0 +1,197 @@
1
+from abc import abstractmethod
2
+from typing import Optional
3
+
4
+import torch
5
+
6
+from vllm.distributed import (get_tensor_model_parallel_rank,
7
+ get_tensor_model_parallel_world_size,
8
+ tensor_model_parallel_all_reduce)
9
+from vllm.logger import init_logger
10
+from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
11
+from vllm.model_executor.layers.quantization.base_config import (
12
+ QuantizationConfig, QuantizeMethodBase)
13
+from vllm.model_executor.utils import set_weight_attrs
14
+
15
+logger = init_logger(__name__)
16
+
17
+
18
+class FusedMoEMethodBase(QuantizeMethodBase):
19
+
20
+ @abstractmethod
21
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
22
+ hidden_size: int, intermediate_size: int,
23
+ params_dtype: torch.dtype, **extra_weight_attrs):
24
+ raise NotImplementedError
25
+
26
+ @abstractmethod
27
+ def apply(self,
28
+ layer: torch.nn.Module,
29
+ x: torch.Tensor,
30
+ router_logits: torch.Tensor,
31
+ top_k: int,
32
+ renormalize: bool = True) -> torch.Tensor:
33
+ raise NotImplementedError
34
+
35
+
36
+class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
37
+ """MoE method without quantization."""
38
+
39
+ def create_weights(self, layer: torch.nn.Module, num_experts: int,
40
+ hidden_size: int, intermediate_size: int,
41
+ params_dtype: torch.dtype, **extra_weight_attrs):
42
+
43
+ # Fused gate_up_proj (column parallel)
44
+ w13_weight = torch.nn.Parameter(torch.empty(num_experts,
45
+ 2 * intermediate_size,
46
+ hidden_size,
47
+ dtype=params_dtype),
48
+ requires_grad=False)
49
+ layer.register_parameter("w13_weight", w13_weight)
50
+ set_weight_attrs(w13_weight, extra_weight_attrs)
51
+
52
+ # down_proj (row parallel)
53
+ w2_weight = torch.nn.Parameter(torch.empty(num_experts,
54
+ hidden_size,
55
+ intermediate_size,
56
+ dtype=params_dtype),
57
+ requires_grad=False)
58
+ layer.register_parameter("w2_weight", w2_weight)
59
+ set_weight_attrs(w2_weight, extra_weight_attrs)
60
+
61
+ def apply(self,
62
+ layer: torch.nn.Module,
63
+ x: torch.Tensor,
64
+ router_logits: torch.Tensor,
65
+ top_k: int,
66
+ renormalize: bool = True) -> torch.Tensor:
67
+
68
+ return fused_moe(x,
69
+ layer.w13_weight,
70
+ layer.w2_weight,
71
+ router_logits,
72
+ top_k,
73
+ renormalize=renormalize,
74
+ inplace=True)
75
+
76
+
77
+class FusedMoE(torch.nn.Module):
78
+ """FusedMoE layer for MoE models.
79
+
80
+ This layer contains both MergedColumnParallel weights (gate_up_proj /
81
+ w13) and RowParallelLinear weights (down_proj/ w2).
82
+
83
+ Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We
84
+ copy that naming convention here and handle any remapping in the
85
+ load_weights function in each model implementation.
86
+
87
+ Args:
88
+ num_experts: Number of experts in the model
89
+ top_k: Number of experts selected for each token
90
+ hidden_size: Input hidden state size of the transformer
91
+ intermediate_size: Intermediate size of the experts
92
+ params_dtype: Data type for the parameters.
93
+ reduce_results: Whether to all all_reduce on the output of the layer
94
+ renomalize: Whether to renormalize the logits in the fused_moe kernel
95
+ quant_config: Quantization configure.
96
+ """
97
+
98
+ def __init__(
99
+ self,
100
+ num_experts: int,
101
+ top_k: int,
102
+ hidden_size: int,
103
+ intermediate_size: int,
104
+ params_dtype: Optional[torch.dtype] = None,
105
+ reduce_results: bool = False,
106
+ renormalize: bool = True,
107
+ quant_config: Optional[QuantizationConfig] = None,
108
+ tp_size: Optional[int] = None,
109
+ ):
110
+ super().__init__()
111
+
112
+ if params_dtype is None:
113
+ params_dtype = torch.get_default_dtype()
114
+
115
+ self.tp_size = (tp_size if tp_size is not None else
116
+ get_tensor_model_parallel_world_size())
117
+ self.top_k = top_k
118
+ self.num_experts = num_experts
119
+ self.intermediate_size_per_partition = intermediate_size // self.tp_size
120
+ self.reduce_results = reduce_results
121
+ self.renormalize = renormalize
122
+
123
+ if quant_config is None:
124
+ self.quant_method: Optional[QuantizeMethodBase] = (
125
+ UnquantizedFusedMoEMethod())
126
+ else:
127
+ self.quant_method = quant_config.get_quant_method(self)
128
+ assert self.quant_method is not None
129
+
130
+ self.quant_method.create_weights(
131
+ layer=self,
132
+ num_experts=num_experts,
133
+ hidden_size=hidden_size,
134
+ intermediate_size=self.intermediate_size_per_partition,
135
+ params_dtype=params_dtype,
136
+ weight_loader=self.weight_loader)
137
+
138
+ def weight_loader(self, param: torch.nn.Parameter,
139
+ loaded_weight: torch.Tensor, weight_name: str,
140
+ shard_id: int, expert_id: int):
141
+ param_data = param.data
142
+
143
+ # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
144
+ # Follow up PR to enable fp8 for other MoE models.
145
+ if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
146
+ if param_data[expert_id] != 1 and (param_data[expert_id] -
147
+ loaded_weight).abs() > 1e-5:
148
+ raise ValueError(
149
+ "input_scales of w1 and w3 of a layer "
150
+ f"must be equal. But got {param_data[expert_id]} "
151
+ f"vs. {loaded_weight}")
152
+ param_data[expert_id] = loaded_weight
153
+ # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
154
+ # Follow up PR to enable fp8 for other MoE models.
155
+ elif "weight_scale" in weight_name:
156
+ # We have to keep the weight scales of w1 and w3 because
157
+ # we need to re-quantize w1/w3 weights after weight loading.
158
+ assert "w1" in weight_name or "w3" in weight_name
159
+ shard_id = 0 if "w1" in weight_name else 1
160
+ param_data[expert_id][shard_id] = loaded_weight
161
+ else:
162
+ tp_rank = get_tensor_model_parallel_rank()
163
+ shard_size = self.intermediate_size_per_partition
164
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
165
+
166
+ # w1, gate_proj case: Load into first shard of w13.
167
+ if shard_id == 0:
168
+ param_data[expert_id,
169
+ 0:shard_size, :] = loaded_weight[shard, :]
170
+ # w3, up_proj case: Load into second shard of w13.
171
+ elif shard_id == 2:
172
+ param_data[expert_id, shard_size:2 *
173
+ shard_size, :] = loaded_weight[shard, :]
174
+ # w2, down_proj case: Load into only shard of w2.
175
+ elif shard_id == 1:
176
+ param_data[expert_id, :, :] = loaded_weight[:, shard]
177
+ else:
178
+ raise ValueError(
179
+ f"Shard id must be in [0,1,2] but got {shard_id}")
180
+
181
+ def forward(self, hidden_states: torch.Tensor,
182
+ router_logits: torch.Tensor):
183
+ assert self.quant_method is not None
184
+
185
+ # Matrix multiply.
186
+ final_hidden_states = self.quant_method.apply(
187
+ self,
188
+ x=hidden_states,
189
+ router_logits=router_logits,
190
+ top_k=self.top_k,
191
+ renormalize=self.renormalize)
192
+
193
+ if self.reduce_results and self.tp_size > 1:
194
+ final_hidden_states = tensor_model_parallel_all_reduce(
195
+ final_hidden_states)
196
+
197
+ return final_hidden_states
vllm/model_executor/layers/quantization/fp8.py
+191
-1
@@ -6,6 +6,8 @@ from torch.nn.parameter import Parameter
6
7
from vllm import _custom_ops as ops
8
from vllm.logger import init_logger
9
+from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
10
+ fused_moe)
11
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
@@ -71,7 +73,9 @@ class Fp8Config(QuantizationConfig):
73
74
if isinstance(layer, LinearBase):
75
return Fp8LinearMethod(self)
74
- if isinstance(layer, Attention):
76
+ elif isinstance(layer, FusedMoE):
77
+ return Fp8MoEMethod(self)
78
+ elif isinstance(layer, Attention):
79
return Fp8KVCacheMethod(self)
80
return None
81
@@ -270,6 +274,187 @@ class Fp8LinearMethod(LinearMethodBase):
274
return torch.narrow(output, 0, 0, x.shape[0])
275
276
277
+class Fp8MoEMethod(FusedMoEMethodBase):
278
+ """MoE method for FP8.
279
+ Supports loading FP8 checkpoints with static weight scale and
280
+ dynamic/static activation scale.
281
+
282
+ Also supports loading quantized FP16/BF16 model checkpoints with dynamic
283
+ activation scaling. The weight scaling factor will be initialized after
284
+ the model weights are loaded.
285
+
286
+ Args:
287
+ quant_config: The quantization config.
288
+ """
289
+
290
+ def __init__(self, quant_config: Fp8Config):
291
+ self.quant_config = quant_config
292
+
293
+ def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
294
+ intermediate_size: int, params_dtype: torch.dtype,
295
+ **extra_weight_attrs):
296
+
297
+ layer.process_after_load = True
298
+
299
+ if self.quant_config.is_checkpoint_fp8_serialized:
300
+ params_dtype = torch.float8_e4m3fn
301
+
302
+ # WEIGHTS
303
+ w13_weight = torch.nn.Parameter(torch.empty(num_experts,
304
+ 2 * intermediate_size,
305
+ hidden_size,
306
+ dtype=params_dtype),
307
+ requires_grad=False)
308
+ layer.register_parameter("w13_weight", w13_weight)
309
+ set_weight_attrs(w13_weight, extra_weight_attrs)
310
+
311
+ w2_weight = torch.nn.Parameter(torch.empty(num_experts,
312
+ hidden_size,
313
+ intermediate_size,
314
+ dtype=params_dtype),
315
+ requires_grad=False)
316
+ layer.register_parameter("w2_weight", w2_weight)
317
+ set_weight_attrs(w2_weight, extra_weight_attrs)
318
+
319
+ # WEIGHT_SCALES
320
+ # Allocate 2 scales for w1 and w3 respectively.
321
+ # They will be combined to a single scale after weight loading.
322
+ w13_scale = torch.nn.Parameter(torch.ones(num_experts,
323
+ 2,
324
+ dtype=torch.float32),
325
+ requires_grad=False)
326
+ layer.register_parameter("w13_scale", w13_scale)
327
+
328
+ w2_scale = torch.nn.Parameter(torch.ones(num_experts,
329
+ dtype=torch.float32),
330
+ requires_grad=False)
331
+ layer.register_parameter("w2_scale", w2_scale)
332
+
333
+ # If loading fp8 checkpoint, pass the weight loaders.
334
+ # If loading an fp16 checkpoint, do not (we will quantize in
335
+ # process_weights_after_loading()
336
+ if self.quant_config.is_checkpoint_fp8_serialized:
337
+ set_weight_attrs(w13_scale, extra_weight_attrs)
338
+ set_weight_attrs(w2_scale, extra_weight_attrs)
339
+
340
+ # INPUT_SCALES
341
+ if self.quant_config.activation_scheme == "static":
342
+ if not self.quant_config.is_checkpoint_fp8_serialized:
343
+ raise ValueError(
344
+ "Found static activation scheme for checkpoint that "
345
+ "was not serialized fp8.")
346
+
347
+ a13_scale = torch.nn.Parameter(torch.ones(num_experts,
348
+ dtype=torch.float32),
349
+ requires_grad=False)
350
+ layer.register_parameter("a13_scale", a13_scale)
351
+ set_weight_attrs(a13_scale, extra_weight_attrs)
352
+
353
+ a2_scale = torch.nn.Parameter(torch.ones(num_experts,
354
+ dtype=torch.float32),
355
+ requires_grad=False)
356
+ layer.register_parameter("a2_scale", a2_scale)
357
+ set_weight_attrs(a2_scale, extra_weight_attrs)
358
+ else:
359
+ layer.a13_scale = None
360
+ layer.a2_scale = None
361
+
362
+ def process_weights_after_loading(self, layer: Module) -> None:
363
+ if (not hasattr(layer, "process_after_load")
364
+ or not layer.process_after_load):
365
+ return
366
+
367
+ # If checkpoint is fp16, quantize in place.
368
+ if not self.quant_config.is_checkpoint_fp8_serialized:
369
+ w13_weight = torch.empty_like(layer.w13_weight.data,
370
+ dtype=torch.float8_e4m3fn)
371
+ w2_weight = torch.empty_like(layer.w2_weight.data,
372
+ dtype=torch.float8_e4m3fn)
373
+
374
+ # Re-initialize w13_scale because we directly quantize
375
+ # merged w13 weights and generate a single scaling factor.
376
+ layer.w13_scale = torch.nn.Parameter(torch.ones(
377
+ layer.num_experts,
378
+ dtype=torch.float32,
379
+ device=w13_weight.device),
380
+ requires_grad=False)
381
+ for expert in range(layer.num_experts):
382
+ w13_weight[expert, :, :], layer.w13_scale[
383
+ expert] = ops.scaled_fp8_quant(
384
+ layer.w13_weight.data[expert, :, :])
385
+ w2_weight[expert, :, :], layer.w2_scale[
386
+ expert] = ops.scaled_fp8_quant(
387
+ layer.w2_weight.data[expert, :, :])
388
+ layer.w13_weight = torch.nn.Parameter(w13_weight,
389
+ requires_grad=False)
390
+ layer.w2_weight = torch.nn.Parameter(w2_weight,
391
+ requires_grad=False)
392
+ return
393
+
394
+ # If checkpoint is fp8, we need to handle that the
395
+ # MoE kernels require single activation scale and single weight
396
+ # scale for w13 per expert.
397
+ else:
398
+ # Fp8 moe kernels require a single activation scale.
399
+ # We take the max of all the scales in case they differ.
400
+ if self.quant_config.activation_scheme == "static":
401
+ if layer.a13_scale is None or layer.a2_scale is None:
402
+ raise ValueError(
403
+ "QuantConfig has static quantization, but found "
404
+ "activation scales are None.")
405
+ if (not all_close_1d(layer.a13_scale)
406
+ or not all_close_1d(layer.a2_scale)):
407
+ print_warning_once(
408
+ "Found input_scales that are not equal for "
409
+ "fp8 MoE layer. Using the maximum across experts "
410
+ "for each layer. ")
411
+ layer.a13_scale = torch.nn.Parameter(layer.a13_scale.max(),
412
+ requires_grad=False)
413
+ layer.a2_scale = torch.nn.Parameter(layer.a2_scale.max(),
414
+ requires_grad=False)
415
+
416
+ # Fp8 moe kernel needs single weight scale for w13 per expert.
417
+ # We take the max then dequant and requant each expert.
418
+ assert layer.w13_scale is not None
419
+ shard_size = layer.intermediate_size_per_partition
420
+ max_w13_scales = layer.w13_scale.max(dim=1).values
421
+ for expert_id in range(layer.num_experts):
422
+ start = 0
423
+ for shard_id in range(2):
424
+ dq_weight = per_tensor_dequantize(
425
+ layer.w13_weight[expert_id][start:start +
426
+ shard_size, :],
427
+ layer.w13_scale[expert_id][shard_id])
428
+ layer.w13_weight[expert_id][
429
+ start:start + shard_size, :] = per_tensor_quantize(
430
+ dq_weight, max_w13_scales[expert_id])
431
+ start += shard_size
432
+
433
+ layer.w13_scale = torch.nn.Parameter(max_w13_scales,
434
+ requires_grad=False)
435
+ return
436
+
437
+ def apply(self,
438
+ layer: torch.nn.Module,
439
+ x: torch.Tensor,
440
+ router_logits: torch.Tensor,
441
+ top_k: int,
442
+ renormalize: bool = True) -> torch.Tensor:
443
+
444
+ return fused_moe(x,
445
+ layer.w13_weight,
446
+ layer.w2_weight,
447
+ router_logits,
448
+ top_k,
449
+ renormalize=renormalize,
450
+ inplace=True,
451
+ use_fp8=True,
452
+ w1_scale=layer.w13_scale,
453
+ w2_scale=layer.w2_scale,
454
+ a1_scale=layer.a13_scale,
455
+ a2_scale=layer.a2_scale)
456
+
457
+
458
class Fp8KVCacheMethod(QuantizeMethodBase):
459
"""Supports loading kv-cache scaling factors from FP8 checkpoints.
460
"""
@@ -321,3 +506,8 @@ def per_tensor_dequantize(
506
fake_qweight = tensor.to(torch.float16)
507
dq_weight = fake_qweight * inv_scale
508
return dq_weight
509
+
510
+
511
+def all_close_1d(x: torch.Tensor) -> bool:
512
+ assert len(x.shape) == 1
513
+ return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/mixtral.py
+41
-235
@@ -27,13 +27,10 @@ import torch
27
from torch import nn
28
from transformers import MixtralConfig
29
30
-from vllm import _custom_ops as ops
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig, LoRAConfig
33
-from vllm.distributed import (get_tensor_model_parallel_rank,
34
- get_tensor_model_parallel_world_size,
35
- tensor_model_parallel_all_reduce)
36
-from vllm.model_executor.layers.fused_moe import fused_moe
32
+from vllm.distributed import get_tensor_model_parallel_world_size
33
+from vllm.model_executor.layers.fused_moe import FusedMoE
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (QKVParallelLinear,
36
ReplicatedLinear,
@@ -41,16 +38,12 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
38
from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization.base_config import (
40
QuantizationConfig)
44
-from vllm.model_executor.layers.quantization.fp8 import (Fp8Config,
45
- per_tensor_dequantize,
46
- per_tensor_quantize)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
-from vllm.model_executor.utils import set_weight_attrs
47
from vllm.sequence import IntermediateTensors, SamplerOutput
48
from vllm.utils import print_warning_once
49
@@ -66,227 +59,40 @@ class MixtralMoE(nn.Module):
59
across ranks.
60
"""
61
69
- def __init__(
70
- self,
71
- num_experts: int,
72
- top_k: int,
73
- hidden_size: int,
74
- intermediate_size: int,
75
- params_dtype: Optional[torch.dtype] = None,
76
- tp_size: Optional[int] = None,
77
- quant_config: Optional[QuantizationConfig] = None,
78
- ):
62
+ def __init__(self,
63
+ num_experts: int,
64
+ top_k: int,
65
+ hidden_size: int,
66
+ intermediate_size: int,
67
+ params_dtype: Optional[torch.dtype] = None,
68
+ quant_config: Optional[QuantizationConfig] = None,
69
+ tp_size: Optional[int] = None):
70
super().__init__()
80
- self.tp_size = tp_size or get_tensor_model_parallel_world_size()
81
- self.num_total_experts = num_experts
82
- self.top_k = top_k
71
self.hidden_size = hidden_size
84
- self.intermediate_size = intermediate_size // self.tp_size
85
- self.quant_config = quant_config
86
-
87
- # FIXME(pcmoritz): Make this more general to support different
88
- # quantization schemes
89
- self.use_fp8 = isinstance(quant_config, Fp8Config)
90
-
91
- if params_dtype is None:
92
- params_dtype = torch.get_default_dtype()
93
- self.params_dtype = params_dtype
72
73
# Gate always runs at half / full precision for now.
96
- self.gate = ReplicatedLinear(self.hidden_size,
97
- self.num_total_experts,
74
+ self.gate = ReplicatedLinear(hidden_size,
75
+ num_experts,
76
bias=False,
99
- params_dtype=self.params_dtype,
77
+ params_dtype=params_dtype,
78
quant_config=None)
79
102
- if self.use_fp8 and self.quant_config.is_checkpoint_fp8_serialized:
103
- params_dtype = torch.float8_e4m3fn
104
-
105
- self.w13_weight = nn.Parameter(torch.empty(self.num_total_experts,
106
- 2 * self.intermediate_size,
107
- self.hidden_size,
108
- dtype=params_dtype),
109
- requires_grad=False)
110
- self.w2_weight = nn.Parameter(torch.empty(self.num_total_experts,
111
- self.hidden_size,
112
- self.intermediate_size,
113
- dtype=params_dtype),
114
- requires_grad=False)
115
-
116
- set_weight_attrs(self.w13_weight, {
117
- "weight_loader": self.weight_loader,
118
- })
119
- set_weight_attrs(self.w2_weight, {
120
- "weight_loader": self.weight_loader,
121
- })
122
-
123
- # Used for fp8.
124
- self.w13_scale = None
125
- self.w2_scale = None
126
- self.a13_scale = None
127
- self.a2_scale = None
128
-
129
- if self.use_fp8:
130
- # WEIGHT_SCALE (for fp8)
131
- # Allocate 2 scales for w1 and w3 respectively.
132
- # They will be combined to a single scale after weight loading.
133
- self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
134
- 2,
135
- dtype=torch.float32),
136
- requires_grad=False)
137
- self.w2_scale = nn.Parameter(torch.ones(self.num_total_experts,
138
- dtype=torch.float32),
139
- requires_grad=False)
140
-
141
- # If loading fp8 checkpoint, pass the weight loaders.
142
- # If loading an fp16 checkpoint, do not (we will quantize in
143
- # process_weights_after_loading()
144
- if quant_config.is_checkpoint_fp8_serialized:
145
- set_weight_attrs(self.w13_scale, {
146
- "weight_loader": self.weight_loader,
147
- })
148
- set_weight_attrs(self.w2_scale, {
149
- "weight_loader": self.weight_loader,
150
- })
151
-
152
- # INPUT_SCALE (for fp8)
153
- if quant_config.activation_scheme == "static":
154
- if not quant_config.is_checkpoint_fp8_serialized:
155
- raise ValueError(
156
- "Found static activation scheme for checkpoint that "
157
- "was not serialized fp8.")
158
- self.a13_scale = nn.Parameter(torch.ones(
159
- self.num_total_experts, dtype=torch.float32),
160
- requires_grad=False)
161
- self.a2_scale = nn.Parameter(torch.ones(self.num_total_experts,
162
- dtype=torch.float32),
163
- requires_grad=False)
164
-
165
- set_weight_attrs(self.a13_scale, {
166
- "weight_loader": self.weight_loader,
167
- })
168
- set_weight_attrs(self.a2_scale, {
169
- "weight_loader": self.weight_loader,
170
- })
171
-
172
- def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
173
- weight_name: str, expert_id: int):
174
- tp_rank = get_tensor_model_parallel_rank()
175
- param_data = param.data
176
- shard_size = self.intermediate_size
177
- shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
178
- if weight_name.endswith("w1.weight"):
179
- param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
180
- if weight_name.endswith("w3.weight"):
181
- param_data[expert_id,
182
- shard_size:2 * shard_size, :] = loaded_weight[shard, :]
183
- if weight_name.endswith("w2.weight"):
184
- param_data[expert_id, :, :] = loaded_weight[:, shard]
185
-
186
- # Loading scales
187
- if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
188
- if param_data[expert_id] != 1 and (param_data[expert_id] -
189
- loaded_weight).abs() > 1e-5:
190
- raise ValueError(
191
- "input_scales of w1 and w3 of a layer "
192
- f"must be equal. But got {param_data[expert_id]} "
193
- f"vs. {loaded_weight}")
194
- param_data[expert_id] = loaded_weight
195
- elif "weight_scale" in weight_name:
196
- # We have to keep the weight scales of w1 and w3 because
197
- # we need to re-quantize w1/w3 weights after weight loading.
198
- assert "w1" in weight_name or "w3" in weight_name
199
- shard_id = 0 if "w1" in weight_name else 1
200
- param_data[expert_id][shard_id] = loaded_weight
201
-
202
- def process_weights_after_loading(self):
203
- # Fp8 is the only case where we need to process after loading.
204
- if not self.use_fp8:
205
- return
206
-
207
- # If checkpoint is fp16, quantize here.
208
- if not self.quant_config.is_checkpoint_fp8_serialized:
209
- w13_weight = torch.empty_like(self.w13_weight.data,
210
- dtype=torch.float8_e4m3fn)
211
- w2_weight = torch.empty_like(self.w2_weight.data,
212
- dtype=torch.float8_e4m3fn)
213
-
214
- # Re-initialize w13_scale because we directly quantize
215
- # merged w13 weights and generate a single scaling factor.
216
- self.w13_scale = nn.Parameter(torch.ones(self.num_total_experts,
217
- dtype=torch.float32),
218
- requires_grad=False)
219
- for expert in range(self.num_total_experts):
220
- w13_weight[expert, :, :], self.w13_scale[
221
- expert] = ops.scaled_fp8_quant(
222
- self.w13_weight.data[expert, :, :])
223
- w2_weight[expert, :, :], self.w2_scale[
224
- expert] = ops.scaled_fp8_quant(
225
- self.w2_weight.data[expert, :, :])
226
- self.w13_weight = nn.Parameter(w13_weight, requires_grad=False)
227
- self.w2_weight = nn.Parameter(w2_weight, requires_grad=False)
228
-
229
- else:
230
- # If checkpoint is fp8 + static, cleanup input_scales.
231
- # Since state_dict has an input_scale per expert but our kernels
232
- # are passed one input_scale shared across all experts.
233
- if self.quant_config.activation_scheme == "static":
234
- if self.a13_scale is None or self.a2_scale is None:
235
- raise ValueError(
236
- "QuantConfig has static quantization, but found "
237
- "activation scales are None.")
238
-
239
- if (not all_close_1d(self.a13_scale)
240
- or not all_close_1d(self.a2_scale)):
241
- print_warning_once(
242
- "Found input_scales that are not equal for "
243
- "fp8 MoE layer. Using the maximum across experts "
244
- "for each layer. ")
245
-
246
- self.a13_scale = nn.Parameter(self.a13_scale.max(),
247
- requires_grad=False)
248
- self.a2_scale = nn.Parameter(self.a2_scale.max(),
249
- requires_grad=False)
250
-
251
- assert self.w13_scale is not None
252
- shard_size = self.intermediate_size
253
- max_w13_scales = self.w13_scale.max(dim=1).values
254
- for expert_id in range(self.num_total_experts):
255
- start = 0
256
- for shard_id in range(2):
257
- dq_weight = per_tensor_dequantize(
258
- self.w13_weight[expert_id][start:start +
259
- shard_size, :],
260
- self.w13_scale[expert_id][shard_id])
261
- self.w13_weight[expert_id][
262
- start:start + shard_size, :] = per_tensor_quantize(
263
- dq_weight, max_w13_scales[expert_id])
264
- start += shard_size
265
-
266
- self.w13_scale = nn.Parameter(max_w13_scales, requires_grad=False)
80
+ self.experts = FusedMoE(num_experts=num_experts,
81
+ top_k=top_k,
82
+ hidden_size=hidden_size,
83
+ intermediate_size=intermediate_size,
84
+ params_dtype=params_dtype,
85
+ reduce_results=True,
86
+ renormalize=True,
87
+ quant_config=quant_config,
88
+ tp_size=tp_size)
89
90
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
91
num_tokens, hidden_size = hidden_states.shape
92
hidden_states = hidden_states.view(-1, self.hidden_size)
93
# router_logits: (num_tokens, n_experts)
94
router_logits, _ = self.gate(hidden_states)
273
- final_hidden_states = fused_moe(hidden_states,
274
- self.w13_weight,
275
- self.w2_weight,
276
- router_logits,
277
- self.top_k,
278
- renormalize=True,
279
- inplace=True,
280
- use_fp8=self.use_fp8,
281
- w1_scale=self.w13_scale,
282
- w2_scale=self.w2_scale,
283
- a1_scale=self.a13_scale,
284
- a2_scale=self.a2_scale)
285
-
286
- if self.tp_size > 1:
287
- final_hidden_states = tensor_model_parallel_all_reduce(
288
- final_hidden_states)
289
-
95
+ final_hidden_states = self.experts(hidden_states, router_logits)
96
return final_hidden_states.view(num_tokens, hidden_size)
97
98
@@ -566,25 +372,28 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
372
373
expert_params_mapping = [
374
# These are the weight scales for the experts
569
- # (param_name, weight_name, expert_id)
570
- ("w13_scale" if weight_name in ["w1", "w3"] else "w2_scale",
571
- f"experts.{expert_id}.{weight_name}.weight_scale", expert_id)
572
- for expert_id in range(self.config.num_local_experts)
573
- for weight_name in ["w1", "w2", "w3"]
375
+ # (param_name, weight_name, expert_id, shard_id)
376
+ ("experts.w13_scale"
377
+ if weight_name in ["w1", "w3"] else "experts.w2_scale",
378
+ f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
379
+ shard_id) for expert_id in range(self.config.num_local_experts)
380
+ for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
381
] + [
382
# These are the weights for the experts
383
# (param_name, weight_name, expert_id)
577
- ("w13_weight" if weight_name in ["w1", "w3"] else "w2_weight",
578
- f"experts.{expert_id}.{weight_name}.weight", expert_id)
384
+ ("experts.w13_weight"
385
+ if weight_name in ["w1", "w3"] else "experts.w2_weight",
386
+ f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
387
for expert_id in range(self.config.num_local_experts)
580
- for weight_name in ["w1", "w2", "w3"]
388
+ for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
389
] + [
390
# These are the activation scales for the experts
391
# (param_name, weight_name, expert_id)
584
- ("a13_scale" if weight_name in ["w1", "w3"] else "a2_scale",
585
- f"experts.{expert_id}.{weight_name}.input_scale", expert_id)
586
- for expert_id in range(self.config.num_local_experts)
587
- for weight_name in ["w1", "w2", "w3"]
392
+ ("experts.a13_scale"
393
+ if weight_name in ["w1", "w3"] else "experts.a2_scale",
394
+ f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
395
+ shard_id) for expert_id in range(self.config.num_local_experts)
396
+ for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
397
]
398
399
params_dict = dict(self.named_parameters())
@@ -604,7 +413,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
413
weight_loader(param, loaded_weight, shard_id)
414
break
415
else:
607
- for param_name, weight_name, expert_id in expert_params_mapping:
416
+ for mapping in expert_params_mapping:
417
+ param_name, weight_name, expert_id, shard_id = mapping
418
if weight_name not in name:
419
continue
420
name = name.replace(weight_name, param_name)
@@ -613,6 +423,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
423
weight_loader(param,
424
loaded_weight,
425
weight_name,
426
+ shard_id=shard_id,
427
expert_id=expert_id)
428
break
429
else:
@@ -637,8 +448,3 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
448
weight_loader = getattr(param, "weight_loader",
449
default_weight_loader)
450
weight_loader(param, loaded_weight)
640
-
641
-
642
-def all_close_1d(x: torch.Tensor) -> bool:
643
- assert len(x.shape) == 1
644
- return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0]))
vllm/model_executor/models/qwen2_moe.py
+68
-68
@@ -31,11 +31,10 @@ from transformers import PretrainedConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
from vllm.config import CacheConfig
34
-from vllm.distributed import (get_tensor_model_parallel_rank,
35
- get_tensor_model_parallel_world_size,
34
+from vllm.distributed import (get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
36
from vllm.model_executor.layers.activation import SiluAndMul
38
-from vllm.model_executor.layers.fused_moe import fused_moe
37
+from vllm.model_executor.layers.fused_moe import FusedMoE
38
from vllm.model_executor.layers.layernorm import RMSNorm
39
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
40
QKVParallelLinear,
@@ -93,28 +92,23 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
92
quant_config: Optional[QuantizationConfig] = None,
93
):
94
super().__init__()
96
- self.config = config
97
- self.rank = get_tensor_model_parallel_rank()
95
self.tp_size = get_tensor_model_parallel_world_size()
99
- self.n_routed_experts = config.num_experts
100
- self.top_k = config.num_experts_per_tok
101
- if self.tp_size > self.n_routed_experts:
96
+
97
+ if self.tp_size > config.num_experts:
98
raise ValueError(
99
f"Tensor parallel size {self.tp_size} is greater than "
104
- f"the number of experts {self.n_routed_experts}.")
105
-
106
- self.experts = nn.ModuleList([
107
- Qwen2MoeMLP(hidden_size=config.hidden_size,
108
- intermediate_size=config.moe_intermediate_size,
109
- hidden_act=config.hidden_act,
110
- quant_config=quant_config,
111
- reduce_results=False)
112
- for idx in range(self.n_routed_experts)
113
- ])
114
- self.pack_params()
100
+ f"the number of experts {config.num_experts}.")
101
+
102
+ self.experts = FusedMoE(num_experts=config.num_experts,
103
+ top_k=config.num_experts_per_tok,
104
+ hidden_size=config.hidden_size,
105
+ intermediate_size=config.moe_intermediate_size,
106
+ reduce_results=False,
107
+ renormalize=config.norm_topk_prob,
108
+ quant_config=quant_config)
109
110
self.gate = ReplicatedLinear(config.hidden_size,
117
- self.n_routed_experts,
111
+ config.num_experts,
112
bias=False,
113
quant_config=None)
114
if config.shared_expert_intermediate_size > 0:
@@ -131,25 +125,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
125
1,
126
bias=False)
127
134
- def pack_params(self):
135
- w1 = []
136
- w2 = []
137
- for expert in self.experts:
138
- w1.append(expert.gate_up_proj.weight)
139
- w2.append(expert.down_proj.weight)
140
- self.w1 = torch._utils._flatten_dense_tensors(w1)
141
- w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
142
- for data, param in zip(w1s, w1):
143
- param.data = data
144
- self.w1 = self.w1.view(len(w1), *w1s[0].shape)
145
-
146
- self.w2 = torch._utils._flatten_dense_tensors(w2)
147
- w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
148
- for data, param in zip(w2s, w2):
149
- param.data = data
150
-
151
- self.w2 = self.w2.view(len(w2), *w2s[0].shape)
152
-
128
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129
num_tokens, hidden_dim = hidden_states.shape
130
hidden_states = hidden_states.view(-1, hidden_dim)
@@ -162,18 +137,13 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
137
138
# router_logits: (num_tokens, n_experts)
139
router_logits, _ = self.gate(hidden_states)
165
- final_hidden_states = fused_moe(hidden_states,
166
- self.w1,
167
- self.w2,
168
- router_logits,
169
- self.top_k,
170
- renormalize=self.config.norm_topk_prob,
171
- inplace=True)
172
-
140
+ final_hidden_states = self.experts(hidden_states=hidden_states,
141
+ router_logits=router_logits)
142
if shared_output is not None:
143
final_hidden_states = final_hidden_states + shared_output
175
- final_hidden_states = tensor_model_parallel_all_reduce(
176
- final_hidden_states)
144
+ if self.tp_size > 1:
145
+ final_hidden_states = tensor_model_parallel_all_reduce(
146
+ final_hidden_states)
147
148
return final_hidden_states.view(num_tokens, hidden_dim)
149
@@ -284,7 +254,12 @@ class Qwen2MoeDecoderLayer(nn.Module):
254
cache_config=cache_config,
255
quant_config=quant_config,
256
)
287
- if (layer_idx not in config.mlp_only_layers) and (
257
+
258
+ # Note: Qwen/Qwen2-57B-A14B-Instruct does not have
259
+ # `mlp_only_layers` in the config.
260
+ mlp_only_layers = ([] if not hasattr(config, "mlp_only_layers") else
261
+ config.mlp_only_layers)
262
+ if (layer_idx not in mlp_only_layers) and (
263
config.num_experts > 0 and
264
(layer_idx + 1) % config.decoder_sparse_step == 0):
265
self.mlp = Qwen2MoeSparseMoeBlock(config=config,
@@ -427,21 +402,36 @@ class Qwen2MoeForCausalLM(nn.Module):
402
("gate_up_proj", "up_proj", 1),
403
]
404
405
+ expert_params_mapping = [
406
+ # These are the weights for the experts
407
+ # (param_name, weight_name, expert_id, shard_id)
408
+ ("experts.w13_weight" if weight_name in ["gate_proj", "up_proj"]
409
+ else "experts.w2_weight",
410
+ f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
411
+ for expert_id in range(self.config.num_experts) for shard_id,
412
+ weight_name in enumerate(["gate_proj", "down_proj", "up_proj"])
413
+ ]
414
+
415
params_dict = dict(self.named_parameters())
416
for name, loaded_weight in weights:
417
if "rotary_emb.inv_freq" in name:
418
continue
419
for (param_name, weight_name, shard_id) in stacked_params_mapping:
420
+ # Skip non-stacked layers and experts (experts handled below).
421
if weight_name not in name:
422
continue
423
+ # We have mlp.experts[0].gate_proj in the checkpoint.
424
+ # Since we handle the experts below in expert_params_mapping,
425
+ # we need to skip here BEFORE we update the name, otherwise
426
+ # name will be updated to mlp.experts[0].gate_up_proj, which
427
+ # will then be updated below in expert_params_mapping
428
+ # for mlp.experts[0].gate_gate_up_proj, which breaks load.
429
+ if "mlp.experts" in name:
430
+ continue
431
name = name.replace(weight_name, param_name)
432
# Skip loading extra bias for GPTQ models.
433
if name.endswith(".bias") and name not in params_dict:
434
continue
441
- # Skip experts that are not assigned to this worker.
442
- if (("mlp.experts." in name or "mlp.shared_expert." in name)
443
- and name not in params_dict):
444
- continue
435
if name not in params_dict:
436
continue
437
@@ -450,17 +440,27 @@ class Qwen2MoeForCausalLM(nn.Module):
440
weight_loader(param, loaded_weight, shard_id)
441
break
442
else:
453
- # Skip loading extra bias for GPTQ models.
454
- if name.endswith(".bias") and name not in params_dict:
455
- continue
456
- # Skip experts that are not assigned to this worker.
457
- if (("mlp.experts." in name or "mlp.shared_expert." in name)
458
- and name not in params_dict):
459
- continue
460
- if name not in params_dict:
461
- continue
462
-
463
- param = params_dict[name]
464
- weight_loader = getattr(param, "weight_loader",
465
- default_weight_loader)
466
- weight_loader(param, loaded_weight)
443
+ for mapping in expert_params_mapping:
444
+ param_name, weight_name, expert_id, shard_id = mapping
445
+ if weight_name not in name:
446
+ continue
447
+ name = name.replace(weight_name, param_name)
448
+ param = params_dict[name]
449
+ weight_loader = param.weight_loader
450
+ weight_loader(param,
451
+ loaded_weight,
452
+ weight_name,
453
+ shard_id=shard_id,
454
+ expert_id=expert_id)
455
+ break
456
+ else:
457
+ # Skip loading extra bias for GPTQ models.
458
+ if name.endswith(".bias") and name not in params_dict:
459
+ continue
460
+ if name not in params_dict:
461
+ continue
462
+
463
+ param = params_dict[name]
464
+ weight_loader = getattr(param, "weight_loader",
465
+ default_weight_loader)
466
+ weight_loader(param, loaded_weight)