A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[ Misc ] Apply MoE Refactor to Deepseekv2 To Support Fp8 (#6417)
Robert Shaw committed
Jul 13, 2024 at 23:03 UTC
fb6af8bc086328ca6659e72d11ffd4309ce4de22
9 files changed
+223
-137
.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash ./run-lm-eval-gsm-vllm-baseline.sh -m deepseek-ai/DeepSeek-V2-Lite-Chat -b "auto" -l 1000 -f 5 -t 2
2
+model_name: "deepseek-ai/DeepSeek-V2-Lite-Chat"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.671
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.664
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,3 +1,4 @@
1
Meta-Llama-3-70B-Instruct.yaml
2
Mixtral-8x7B-Instruct-v0.1.yaml
3
Qwen2-57B-A14-Instruct.yaml
4
+DeepSeek-V2-Lite-Chat.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray" \
49
+ --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true \
50
--tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51
--batch_size $BATCH_SIZE
vllm/model_executor/layers/fused_moe/fused_moe.py
+26
-10
@@ -394,14 +394,16 @@ def fused_topk(
394
395
396
# This is used by the Deepseek-V2 model
397
-def grouped_topk(
398
- hidden_states: torch.Tensor,
399
- gating_output: torch.Tensor,
400
- topk: int,
401
- renormalize: bool,
402
- num_expert_group: int = 0,
403
- topk_group: int = 0,
404
-):
397
+def grouped_topk(hidden_states: torch.Tensor,
398
+ gating_output: torch.Tensor,
399
+ topk: int,
400
+ renormalize: bool,
401
+ num_expert_group: int = 0,
402
+ topk_group: int = 0):
403
+
404
+ assert hidden_states.shape[0] == gating_output.shape[0], (
405
+ "Number of tokens mismatch")
406
+
407
scores = torch.softmax(gating_output, dim=-1)
408
num_token = scores.shape[0]
409
group_scores = scores.view(num_token, num_expert_group,
@@ -557,6 +559,9 @@ def fused_moe(
559
renormalize: bool,
560
inplace: bool = False,
561
override_config: Optional[Dict[str, Any]] = None,
562
+ use_grouped_topk: bool = False,
563
+ num_expert_group: Optional[int] = None,
564
+ topk_group: Optional[int] = None,
565
use_fp8: bool = False,
566
w1_scale: Optional[torch.Tensor] = None,
567
w2_scale: Optional[torch.Tensor] = None,
@@ -579,6 +584,10 @@ def fused_moe(
584
Defaults to False.
585
- override_config (Optional[Dict[str, Any]]): Optional override
586
for the kernel configuration.
587
+ - num_expert_group: Optional[int]: additional parameter for grouped_topk
588
+ - topk_group: Optional[int]: additional parameter for grouped_topk
589
+ - use_grouped_topk: If True, use grouped_topk instead of fused_topk
590
+ note: Deepseekv2 model uses grouped_topk
591
- use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
592
products for w1 and w2. Defaults to False.
593
- w1_scale (Optional[torch.Tensor]): Optional scale to be used for
@@ -592,8 +601,15 @@ def fused_moe(
601
# Check constraints.
602
assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
603
595
- topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
596
- renormalize)
604
+ if use_grouped_topk:
605
+ assert num_expert_group is not None and topk_group is not None
606
+ topk_weights, topk_ids = grouped_topk(hidden_states, gating_output,
607
+ topk, renormalize,
608
+ num_expert_group, topk_group)
609
+ else:
610
+ topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
611
+ renormalize)
612
+
613
return fused_experts(hidden_states,
614
w1,
615
w2,
vllm/model_executor/layers/fused_moe/layer.py
+78
-15
@@ -1,5 +1,5 @@
1
from abc import abstractmethod
2
-from typing import Optional
2
+from typing import List, Optional, Tuple
3
4
import torch
5
@@ -29,7 +29,10 @@ class FusedMoEMethodBase(QuantizeMethodBase):
29
x: torch.Tensor,
30
router_logits: torch.Tensor,
31
top_k: int,
32
- renormalize: bool = True) -> torch.Tensor:
32
+ renormalize: bool = True,
33
+ use_grouped_topk: bool = False,
34
+ num_expert_group: Optional[int] = None,
35
+ topk_group: Optional[int] = None) -> torch.Tensor:
36
raise NotImplementedError
37
38
@@ -63,7 +66,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
66
x: torch.Tensor,
67
router_logits: torch.Tensor,
68
top_k: int,
66
- renormalize: bool = True) -> torch.Tensor:
69
+ renormalize: bool = True,
70
+ use_grouped_topk: bool = False,
71
+ num_expert_group: Optional[int] = None,
72
+ topk_group: Optional[int] = None) -> torch.Tensor:
73
74
return fused_moe(x,
75
layer.w13_weight,
@@ -71,7 +77,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
77
router_logits,
78
top_k,
79
renormalize=renormalize,
74
- inplace=True)
80
+ inplace=True,
81
+ use_grouped_topk=use_grouped_topk,
82
+ num_expert_group=num_expert_group,
83
+ topk_group=topk_group)
84
85
86
class FusedMoE(torch.nn.Module):
@@ -104,6 +113,9 @@ class FusedMoE(torch.nn.Module):
113
params_dtype: Optional[torch.dtype] = None,
114
reduce_results: bool = False,
115
renormalize: bool = True,
116
+ use_grouped_topk: bool = False,
117
+ num_expert_group: Optional[int] = None,
118
+ topk_group: Optional[int] = None,
119
quant_config: Optional[QuantizationConfig] = None,
120
tp_size: Optional[int] = None,
121
):
@@ -119,6 +131,11 @@ class FusedMoE(torch.nn.Module):
131
self.intermediate_size_per_partition = intermediate_size // self.tp_size
132
self.reduce_results = reduce_results
133
self.renormalize = renormalize
134
+ self.use_grouped_topk = use_grouped_topk
135
+ if self.use_grouped_topk:
136
+ assert num_expert_group is not None and topk_group is not None
137
+ self.num_expert_group = num_expert_group
138
+ self.topk_group = topk_group
139
140
if quant_config is None:
141
self.quant_method: Optional[QuantizeMethodBase] = (
@@ -140,9 +157,8 @@ class FusedMoE(torch.nn.Module):
157
shard_id: int, expert_id: int):
158
param_data = param.data
159
143
- # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
144
- # Follow up PR to enable fp8 for other MoE models.
145
- if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
160
+ # Input scales can be loaded directly and should be equal.
161
+ if "input_scale" in weight_name:
162
if param_data[expert_id] != 1 and (param_data[expert_id] -
163
loaded_weight).abs() > 1e-5:
164
raise ValueError(
@@ -150,14 +166,21 @@ class FusedMoE(torch.nn.Module):
166
f"must be equal. But got {param_data[expert_id]} "
167
f"vs. {loaded_weight}")
168
param_data[expert_id] = loaded_weight
153
- # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
154
- # Follow up PR to enable fp8 for other MoE models.
169
+ # Weight scales
170
elif "weight_scale" in weight_name:
156
- # We have to keep the weight scales of w1 and w3 because
157
- # we need to re-quantize w1/w3 weights after weight loading.
158
- assert "w1" in weight_name or "w3" in weight_name
159
- shard_id = 0 if "w1" in weight_name else 1
160
- param_data[expert_id][shard_id] = loaded_weight
171
+ # If we are in merged column case (gate_up_proj)
172
+ # shard_id 0 == gate_proj / w1
173
+ # shard_id 2 == up_proj / w3
174
+ if shard_id == 0 or shard_id == 2:
175
+ # We have to keep the weight scales of w1 and w3 because
176
+ # we need to re-quantize w1/w3 weights after weight loading.
177
+ idx = 0 if shard_id == 0 else 1
178
+ param_data[expert_id][idx] = loaded_weight
179
+ # If we are in the row parallel case (down_proj)
180
+ # shard_id 1 == down_proj / w2
181
+ else:
182
+ param_data[expert_id] = loaded_weight
183
+ # Weights
184
else:
185
tp_rank = get_tensor_model_parallel_rank()
186
shard_size = self.intermediate_size_per_partition
@@ -188,10 +211,50 @@ class FusedMoE(torch.nn.Module):
211
x=hidden_states,
212
router_logits=router_logits,
213
top_k=self.top_k,
191
- renormalize=self.renormalize)
214
+ renormalize=self.renormalize,
215
+ use_grouped_topk=self.use_grouped_topk,
216
+ num_expert_group=self.num_expert_group,
217
+ topk_group=self.topk_group)
218
219
if self.reduce_results and self.tp_size > 1:
220
final_hidden_states = tensor_model_parallel_all_reduce(
221
final_hidden_states)
222
223
return final_hidden_states
224
+
225
+ @classmethod
226
+ def make_expert_params_mapping(
227
+ cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str,
228
+ ckpt_up_proj_name: str,
229
+ num_experts: int) -> List[Tuple[str, str, int, int]]:
230
+
231
+ gate_up = [ckpt_gate_proj_name, ckpt_up_proj_name]
232
+ gate_down_up = [
233
+ ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name
234
+ ]
235
+
236
+ return [
237
+ # These are the weight scales for the experts
238
+ # (param_name, weight_name, expert_id, shard_id)
239
+ ("experts.w13_scale"
240
+ if weight_name in gate_up else "experts.w2_scale",
241
+ f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
242
+ shard_id) for expert_id in range(num_experts)
243
+ for shard_id, weight_name in enumerate(gate_down_up)
244
+ ] + [
245
+ # These are the weights for the experts
246
+ # (param_name, weight_name, expert_id, shard_id)
247
+ ("experts.w13_weight"
248
+ if weight_name in gate_up else "experts.w2_weight",
249
+ f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
250
+ for expert_id in range(num_experts)
251
+ for shard_id, weight_name in enumerate(gate_down_up)
252
+ ] + [
253
+ # These are the weight scales for the experts
254
+ # (param_name, weight_name, expert_id, shard_id)
255
+ ("experts.a13_scale"
256
+ if weight_name in gate_up else "experts.a2_scale",
257
+ f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
258
+ shard_id) for expert_id in range(num_experts)
259
+ for shard_id, weight_name in enumerate(gate_down_up)
260
+ ]
vllm/model_executor/layers/quantization/fp8.py
+8
-2
@@ -377,7 +377,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
377
x: torch.Tensor,
378
router_logits: torch.Tensor,
379
top_k: int,
380
- renormalize: bool = True) -> torch.Tensor:
380
+ renormalize: bool = True,
381
+ use_grouped_topk: bool = False,
382
+ num_expert_group: Optional[int] = None,
383
+ topk_group: Optional[int] = None) -> torch.Tensor:
384
385
return fused_moe(x,
386
layer.w13_weight,
@@ -390,7 +393,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
393
w1_scale=layer.w13_scale,
394
w2_scale=layer.w2_scale,
395
a1_scale=layer.a13_scale,
393
- a2_scale=layer.a2_scale)
396
+ a2_scale=layer.a2_scale,
397
+ use_grouped_topk=use_grouped_topk,
398
+ num_expert_group=num_expert_group,
399
+ topk_group=topk_group)
400
401
402
class Fp8KVCacheMethod(QuantizeMethodBase):
vllm/model_executor/models/deepseek_v2.py
+69
-73
@@ -29,11 +29,10 @@ from transformers import PretrainedConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import CacheConfig
32
-from vllm.distributed import (get_tensor_model_parallel_rank,
33
- get_tensor_model_parallel_world_size,
32
+from vllm.distributed import (get_tensor_model_parallel_world_size,
33
tensor_model_parallel_all_reduce)
34
from vllm.model_executor.layers.activation import SiluAndMul
36
-from vllm.model_executor.layers.fused_moe import fused_experts, grouped_topk
35
+from vllm.model_executor.layers.fused_moe import FusedMoE
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38
MergedColumnParallelLinear,
@@ -91,32 +90,34 @@ class DeepseekV2MoE(nn.Module):
90
quant_config: Optional[QuantizationConfig] = None,
91
):
92
super().__init__()
94
- self.config = config
95
- self.rank = get_tensor_model_parallel_rank()
93
self.tp_size = get_tensor_model_parallel_world_size()
97
- self.n_routed_experts = config.n_routed_experts
98
- self.top_k = config.num_experts_per_tok
94
self.routed_scaling_factor = config.routed_scaling_factor
100
- if self.tp_size > self.n_routed_experts:
95
+ self.n_shared_experts = config.n_shared_experts
96
+ self.routed_scaling_factor = config.routed_scaling_factor
97
+ if self.tp_size > config.n_routed_experts:
98
raise ValueError(
99
f"Tensor parallel size {self.tp_size} is greater than "
103
- f"the number of experts {self.n_routed_experts}.")
104
-
105
- self.experts = nn.ModuleList([
106
- DeepseekV2MLP(hidden_size=config.hidden_size,
107
- intermediate_size=config.moe_intermediate_size,
108
- hidden_act=config.hidden_act,
109
- quant_config=quant_config,
110
- reduce_results=False)
111
- for idx in range(self.n_routed_experts)
112
- ])
113
- self.pack_params()
100
+ f"the number of experts {config.n_routed_experts}.")
101
+
102
+ if config.hidden_act != "silu":
103
+ raise ValueError(f"Unsupported activation: {config.hidden_act}. "
104
+ "Only silu is supported for now.")
105
+
106
+ self.experts = FusedMoE(num_experts=config.n_routed_experts,
107
+ top_k=config.num_experts_per_tok,
108
+ hidden_size=config.hidden_size,
109
+ intermediate_size=config.moe_intermediate_size,
110
+ reduce_results=False,
111
+ renormalize=config.norm_topk_prob,
112
+ quant_config=quant_config,
113
+ use_grouped_topk=True,
114
+ num_expert_group=config.n_group,
115
+ topk_group=config.topk_group)
116
117
self.gate = ReplicatedLinear(config.hidden_size,
116
- self.n_routed_experts,
118
+ config.n_routed_experts,
119
bias=False,
120
quant_config=None)
119
-
121
if config.n_shared_experts is not None:
122
intermediate_size = (config.moe_intermediate_size *
123
config.n_shared_experts)
@@ -128,50 +129,21 @@ class DeepseekV2MoE(nn.Module):
129
reduce_results=False,
130
)
131
131
- def pack_params(self):
132
- w1 = []
133
- w2 = []
134
- for expert in self.experts:
135
- w1.append(expert.gate_up_proj.weight)
136
- w2.append(expert.down_proj.weight)
137
- self.w1 = torch._utils._flatten_dense_tensors(w1)
138
- w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
139
- for data, param in zip(w1s, w1):
140
- param.data = data
141
- self.w1 = self.w1.view(len(w1), *w1s[0].shape)
142
-
143
- self.w2 = torch._utils._flatten_dense_tensors(w2)
144
- w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
145
- for data, param in zip(w2s, w2):
146
- param.data = data
147
-
148
- self.w2 = self.w2.view(len(w2), *w2s[0].shape)
149
-
132
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
133
num_tokens, hidden_dim = hidden_states.shape
134
hidden_states = hidden_states.view(-1, hidden_dim)
153
- if self.config.n_shared_experts is not None:
135
+ if self.n_shared_experts is not None:
136
shared_output = self.shared_experts(hidden_states)
137
# router_logits: (num_tokens, n_experts)
138
router_logits, _ = self.gate(hidden_states)
157
- topk_weights, topk_ids = grouped_topk(
158
- hidden_states,
159
- router_logits,
160
- self.top_k,
161
- renormalize=self.config.norm_topk_prob,
162
- num_expert_group=self.config.n_group,
163
- topk_group=self.config.topk_group)
164
- final_hidden_states = fused_experts(
165
- hidden_states,
166
- self.w1,
167
- self.w2,
168
- topk_weights,
169
- topk_ids,
170
- inplace=True) * self.routed_scaling_factor
171
- if self.config.n_shared_experts is not None:
139
+ final_hidden_states = self.experts(
140
+ hidden_states=hidden_states,
141
+ router_logits=router_logits) * self.routed_scaling_factor
142
+ if shared_output is not None:
143
final_hidden_states = final_hidden_states + shared_output
173
- final_hidden_states = tensor_model_parallel_all_reduce(
174
- final_hidden_states)
144
+ if self.tp_size > 1:
145
+ final_hidden_states = tensor_model_parallel_all_reduce(
146
+ final_hidden_states)
147
148
return final_hidden_states.view(num_tokens, hidden_dim)
149
@@ -504,34 +476,58 @@ class DeepseekV2ForCausalLM(nn.Module):
476
("gate_up_proj", "up_proj", 1),
477
]
478
479
+ # Params for weights, fp8 weight scales, fp8 activation scales
480
+ # (param_name, weight_name, expert_id, shard_id)
481
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
482
+ ckpt_gate_proj_name="gate_proj",
483
+ ckpt_down_proj_name="down_proj",
484
+ ckpt_up_proj_name="up_proj",
485
+ num_experts=self.config.n_routed_experts)
486
+
487
params_dict = dict(self.named_parameters())
488
for name, loaded_weight in weights:
489
if "rotary_emb.inv_freq" in name:
490
continue
491
for (param_name, weight_name, shard_id) in stacked_params_mapping:
492
+ # Skip non-stacked layers and experts (experts handled below).
493
if weight_name not in name:
494
continue
495
+ # We have mlp.experts[0].gate_proj in the checkpoint.
496
+ # Since we handle the experts below in expert_params_mapping,
497
+ # we need to skip here BEFORE we update the name, otherwise
498
+ # name will be updated to mlp.experts[0].gate_up_proj, which
499
+ # will then be updated below in expert_params_mapping
500
+ # for mlp.experts[0].gate_gate_up_proj, which breaks load.
501
+ if (("mlp.experts." in name) and name not in params_dict):
502
+ continue
503
name = name.replace(weight_name, param_name)
504
# Skip loading extra bias for GPTQ models.
505
if name.endswith(".bias") and name not in params_dict:
506
continue
518
- # Skip experts that are not assigned to this worker.
519
- if (("mlp.experts." in name or "mlp.shared_experts." in name)
520
- and name not in params_dict):
521
- continue
507
param = params_dict[name]
508
weight_loader = param.weight_loader
509
weight_loader(param, loaded_weight, shard_id)
510
break
511
else:
527
- # Skip loading extra bias for GPTQ models.
528
- if name.endswith(".bias") and name not in params_dict:
529
- continue
530
- # Skip experts that are not assigned to this worker.
531
- if (("mlp.experts." in name or "mlp.shared_experts." in name)
532
- and name not in params_dict):
533
- continue
534
- param = params_dict[name]
535
- weight_loader = getattr(param, "weight_loader",
536
- default_weight_loader)
537
- weight_loader(param, loaded_weight)
512
+ for mapping in expert_params_mapping:
513
+ param_name, weight_name, expert_id, shard_id = mapping
514
+ if weight_name not in name:
515
+ continue
516
+ name = name.replace(weight_name, param_name)
517
+ param = params_dict[name]
518
+ weight_loader = param.weight_loader
519
+ weight_loader(param,
520
+ loaded_weight,
521
+ weight_name,
522
+ shard_id=shard_id,
523
+ expert_id=expert_id)
524
+ break
525
+ else:
526
+ # Skip loading extra bias for GPTQ models.
527
+ if name.endswith(".bias") and name not in params_dict:
528
+ continue
529
+
530
+ param = params_dict[name]
531
+ weight_loader = getattr(param, "weight_loader",
532
+ default_weight_loader)
533
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/mixtral.py
+7
-25
@@ -372,31 +372,13 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
372
("qkv_proj", "v_proj", "v"),
373
]
374
375
- expert_params_mapping = [
376
- # These are the weight scales for the experts
377
- # (param_name, weight_name, expert_id, shard_id)
378
- ("experts.w13_scale"
379
- if weight_name in ["w1", "w3"] else "experts.w2_scale",
380
- f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
381
- shard_id) for expert_id in range(self.config.num_local_experts)
382
- for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
383
- ] + [
384
- # These are the weights for the experts
385
- # (param_name, weight_name, expert_id)
386
- ("experts.w13_weight"
387
- if weight_name in ["w1", "w3"] else "experts.w2_weight",
388
- f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
389
- for expert_id in range(self.config.num_local_experts)
390
- for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
391
- ] + [
392
- # These are the activation scales for the experts
393
- # (param_name, weight_name, expert_id)
394
- ("experts.a13_scale"
395
- if weight_name in ["w1", "w3"] else "experts.a2_scale",
396
- f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
397
- shard_id) for expert_id in range(self.config.num_local_experts)
398
- for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
399
- ]
375
+ # Params for weights, fp8 weight scales, fp8 activation scales
376
+ # (param_name, weight_name, expert_id, shard_id)
377
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
378
+ ckpt_gate_proj_name="w1",
379
+ ckpt_down_proj_name="w2",
380
+ ckpt_up_proj_name="w3",
381
+ num_experts=self.config.num_local_experts)
382
383
params_dict = dict(self.named_parameters())
384
for name, loaded_weight in weights:
vllm/model_executor/models/qwen2_moe.py
+22
-11
@@ -50,6 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.sequence import IntermediateTensors, SamplerOutput
53
+from vllm.utils import print_warning_once
54
55
56
class Qwen2MoeMLP(nn.Module):
@@ -406,15 +407,13 @@ class Qwen2MoeForCausalLM(nn.Module):
407
("gate_up_proj", "up_proj", 1),
408
]
409
409
- expert_params_mapping = [
410
- # These are the weights for the experts
411
- # (param_name, weight_name, expert_id, shard_id)
412
- ("experts.w13_weight" if weight_name in ["gate_proj", "up_proj"]
413
- else "experts.w2_weight",
414
- f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
415
- for expert_id in range(self.config.num_experts) for shard_id,
416
- weight_name in enumerate(["gate_proj", "down_proj", "up_proj"])
417
- ]
410
+ # Params for weights, fp8 weight scales, fp8 activation scales
411
+ # (param_name, weight_name, expert_id, shard_id)
412
+ expert_params_mapping = FusedMoE.make_expert_params_mapping(
413
+ ckpt_gate_proj_name="gate_proj",
414
+ ckpt_down_proj_name="down_proj",
415
+ ckpt_up_proj_name="up_proj",
416
+ num_experts=self.config.num_experts)
417
418
params_dict = dict(self.named_parameters())
419
for name, loaded_weight in weights:
@@ -461,8 +460,20 @@ class Qwen2MoeForCausalLM(nn.Module):
460
# Skip loading extra bias for GPTQ models.
461
if name.endswith(".bias") and name not in params_dict:
462
continue
464
- if name not in params_dict:
465
- continue
463
+ # Remapping the name of FP8 kv-scale.
464
+ if name.endswith("kv_scale"):
465
+ remapped_kv_scale_name = name.replace(
466
+ ".kv_scale", ".attn.kv_scale")
467
+ if remapped_kv_scale_name not in params_dict:
468
+ print_warning_once(
469
+ "Found kv scale in the checkpoint "
470
+ f"(e.g. {name}), but not found the expected "
471
+ f"name in the model "
472
+ f"(e.g. {remapped_kv_scale_name}). "
473
+ "kv-scale is not loaded.")
474
+ continue
475
+ else:
476
+ name = remapped_kv_scale_name
477
478
param = params_dict[name]
479
weight_loader = getattr(param, "weight_loader",