@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fb6af8bc08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ Misc ] Apply MoE Refactor to Deepseekv2 To Support Fp8 (#6417)

Robert Shaw committed Jul 13, 2024 at 23:03 UTC fb6af8bc086328ca6659e72d11ffd4309ce4de22
9 files changed +223 -137
.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash ./run-lm-eval-gsm-vllm-baseline.sh -m deepseek-ai/DeepSeek-V2-Lite-Chat -b "auto" -l 1000 -f 5 -t 2
2 +model_name: "deepseek-ai/DeepSeek-V2-Lite-Chat"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.671
8 + - name: "exact_match,flexible-extract"
9 + value: 0.664
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -1,3 +1,4 @@
1 Meta-Llama-3-70B-Instruct.yaml
2 Mixtral-8x7B-Instruct-v0.1.yaml
3 Qwen2-57B-A14-Instruct.yaml
4 +DeepSeek-V2-Lite-Chat.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray" \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true \
50 --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 --batch_size $BATCH_SIZE
vllm/model_executor/layers/fused_moe/fused_moe.py
+26 -10
@@ -394,14 +394,16 @@ def fused_topk(
394
395
396 # This is used by the Deepseek-V2 model
397 -def grouped_topk(
398 - hidden_states: torch.Tensor,
399 - gating_output: torch.Tensor,
400 - topk: int,
401 - renormalize: bool,
402 - num_expert_group: int = 0,
403 - topk_group: int = 0,
404 -):
397 +def grouped_topk(hidden_states: torch.Tensor,
398 + gating_output: torch.Tensor,
399 + topk: int,
400 + renormalize: bool,
401 + num_expert_group: int = 0,
402 + topk_group: int = 0):
403 +
404 + assert hidden_states.shape[0] == gating_output.shape[0], (
405 + "Number of tokens mismatch")
406 +
407 scores = torch.softmax(gating_output, dim=-1)
408 num_token = scores.shape[0]
409 group_scores = scores.view(num_token, num_expert_group,
@@ -557,6 +559,9 @@ def fused_moe(
559 renormalize: bool,
560 inplace: bool = False,
561 override_config: Optional[Dict[str, Any]] = None,
562 + use_grouped_topk: bool = False,
563 + num_expert_group: Optional[int] = None,
564 + topk_group: Optional[int] = None,
565 use_fp8: bool = False,
566 w1_scale: Optional[torch.Tensor] = None,
567 w2_scale: Optional[torch.Tensor] = None,
@@ -579,6 +584,10 @@ def fused_moe(
584 Defaults to False.
585 - override_config (Optional[Dict[str, Any]]): Optional override
586 for the kernel configuration.
587 + - num_expert_group: Optional[int]: additional parameter for grouped_topk
588 + - topk_group: Optional[int]: additional parameter for grouped_topk
589 + - use_grouped_topk: If True, use grouped_topk instead of fused_topk
590 + note: Deepseekv2 model uses grouped_topk
591 - use_fp8 (bool): If True, use fp8 arithmetic to compute the inner
592 products for w1 and w2. Defaults to False.
593 - w1_scale (Optional[torch.Tensor]): Optional scale to be used for
@@ -592,8 +601,15 @@ def fused_moe(
601 # Check constraints.
602 assert gating_output.shape[1] == w1.shape[0], "Number of experts mismatch"
603
595 - topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
596 - renormalize)
604 + if use_grouped_topk:
605 + assert num_expert_group is not None and topk_group is not None
606 + topk_weights, topk_ids = grouped_topk(hidden_states, gating_output,
607 + topk, renormalize,
608 + num_expert_group, topk_group)
609 + else:
610 + topk_weights, topk_ids = fused_topk(hidden_states, gating_output, topk,
611 + renormalize)
612 +
613 return fused_experts(hidden_states,
614 w1,
615 w2,
vllm/model_executor/layers/fused_moe/layer.py
+78 -15
@@ -1,5 +1,5 @@
1 from abc import abstractmethod
2 -from typing import Optional
2 +from typing import List, Optional, Tuple
3
4 import torch
5
@@ -29,7 +29,10 @@ class FusedMoEMethodBase(QuantizeMethodBase):
29 x: torch.Tensor,
30 router_logits: torch.Tensor,
31 top_k: int,
32 - renormalize: bool = True) -> torch.Tensor:
32 + renormalize: bool = True,
33 + use_grouped_topk: bool = False,
34 + num_expert_group: Optional[int] = None,
35 + topk_group: Optional[int] = None) -> torch.Tensor:
36 raise NotImplementedError
37
38
@@ -63,7 +66,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
66 x: torch.Tensor,
67 router_logits: torch.Tensor,
68 top_k: int,
66 - renormalize: bool = True) -> torch.Tensor:
69 + renormalize: bool = True,
70 + use_grouped_topk: bool = False,
71 + num_expert_group: Optional[int] = None,
72 + topk_group: Optional[int] = None) -> torch.Tensor:
73
74 return fused_moe(x,
75 layer.w13_weight,
@@ -71,7 +77,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
77 router_logits,
78 top_k,
79 renormalize=renormalize,
74 - inplace=True)
80 + inplace=True,
81 + use_grouped_topk=use_grouped_topk,
82 + num_expert_group=num_expert_group,
83 + topk_group=topk_group)
84
85
86 class FusedMoE(torch.nn.Module):
@@ -104,6 +113,9 @@ class FusedMoE(torch.nn.Module):
113 params_dtype: Optional[torch.dtype] = None,
114 reduce_results: bool = False,
115 renormalize: bool = True,
116 + use_grouped_topk: bool = False,
117 + num_expert_group: Optional[int] = None,
118 + topk_group: Optional[int] = None,
119 quant_config: Optional[QuantizationConfig] = None,
120 tp_size: Optional[int] = None,
121 ):
@@ -119,6 +131,11 @@ class FusedMoE(torch.nn.Module):
131 self.intermediate_size_per_partition = intermediate_size // self.tp_size
132 self.reduce_results = reduce_results
133 self.renormalize = renormalize
134 + self.use_grouped_topk = use_grouped_topk
135 + if self.use_grouped_topk:
136 + assert num_expert_group is not None and topk_group is not None
137 + self.num_expert_group = num_expert_group
138 + self.topk_group = topk_group
139
140 if quant_config is None:
141 self.quant_method: Optional[QuantizeMethodBase] = (
@@ -140,9 +157,8 @@ class FusedMoE(torch.nn.Module):
157 shard_id: int, expert_id: int):
158 param_data = param.data
159
143 - # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
144 - # Follow up PR to enable fp8 for other MoE models.
145 - if "input_scale" in weight_name or "w2.weight_scale" in weight_name:
160 + # Input scales can be loaded directly and should be equal.
161 + if "input_scale" in weight_name:
162 if param_data[expert_id] != 1 and (param_data[expert_id] -
163 loaded_weight).abs() > 1e-5:
164 raise ValueError(
@@ -150,14 +166,21 @@ class FusedMoE(torch.nn.Module):
166 f"must be equal. But got {param_data[expert_id]} "
167 f"vs. {loaded_weight}")
168 param_data[expert_id] = loaded_weight
153 - # FIXME(robertgshaw2-neuralmagic): Overfit to Mixtral.
154 - # Follow up PR to enable fp8 for other MoE models.
169 + # Weight scales
170 elif "weight_scale" in weight_name:
156 - # We have to keep the weight scales of w1 and w3 because
157 - # we need to re-quantize w1/w3 weights after weight loading.
158 - assert "w1" in weight_name or "w3" in weight_name
159 - shard_id = 0 if "w1" in weight_name else 1
160 - param_data[expert_id][shard_id] = loaded_weight
171 + # If we are in merged column case (gate_up_proj)
172 + # shard_id 0 == gate_proj / w1
173 + # shard_id 2 == up_proj / w3
174 + if shard_id == 0 or shard_id == 2:
175 + # We have to keep the weight scales of w1 and w3 because
176 + # we need to re-quantize w1/w3 weights after weight loading.
177 + idx = 0 if shard_id == 0 else 1
178 + param_data[expert_id][idx] = loaded_weight
179 + # If we are in the row parallel case (down_proj)
180 + # shard_id 1 == down_proj / w2
181 + else:
182 + param_data[expert_id] = loaded_weight
183 + # Weights
184 else:
185 tp_rank = get_tensor_model_parallel_rank()
186 shard_size = self.intermediate_size_per_partition
@@ -188,10 +211,50 @@ class FusedMoE(torch.nn.Module):
211 x=hidden_states,
212 router_logits=router_logits,
213 top_k=self.top_k,
191 - renormalize=self.renormalize)
214 + renormalize=self.renormalize,
215 + use_grouped_topk=self.use_grouped_topk,
216 + num_expert_group=self.num_expert_group,
217 + topk_group=self.topk_group)
218
219 if self.reduce_results and self.tp_size > 1:
220 final_hidden_states = tensor_model_parallel_all_reduce(
221 final_hidden_states)
222
223 return final_hidden_states
224 +
225 + @classmethod
226 + def make_expert_params_mapping(
227 + cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str,
228 + ckpt_up_proj_name: str,
229 + num_experts: int) -> List[Tuple[str, str, int, int]]:
230 +
231 + gate_up = [ckpt_gate_proj_name, ckpt_up_proj_name]
232 + gate_down_up = [
233 + ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name
234 + ]
235 +
236 + return [
237 + # These are the weight scales for the experts
238 + # (param_name, weight_name, expert_id, shard_id)
239 + ("experts.w13_scale"
240 + if weight_name in gate_up else "experts.w2_scale",
241 + f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
242 + shard_id) for expert_id in range(num_experts)
243 + for shard_id, weight_name in enumerate(gate_down_up)
244 + ] + [
245 + # These are the weights for the experts
246 + # (param_name, weight_name, expert_id, shard_id)
247 + ("experts.w13_weight"
248 + if weight_name in gate_up else "experts.w2_weight",
249 + f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
250 + for expert_id in range(num_experts)
251 + for shard_id, weight_name in enumerate(gate_down_up)
252 + ] + [
253 + # These are the weight scales for the experts
254 + # (param_name, weight_name, expert_id, shard_id)
255 + ("experts.a13_scale"
256 + if weight_name in gate_up else "experts.a2_scale",
257 + f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
258 + shard_id) for expert_id in range(num_experts)
259 + for shard_id, weight_name in enumerate(gate_down_up)
260 + ]
vllm/model_executor/layers/quantization/fp8.py
+8 -2
@@ -377,7 +377,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
377 x: torch.Tensor,
378 router_logits: torch.Tensor,
379 top_k: int,
380 - renormalize: bool = True) -> torch.Tensor:
380 + renormalize: bool = True,
381 + use_grouped_topk: bool = False,
382 + num_expert_group: Optional[int] = None,
383 + topk_group: Optional[int] = None) -> torch.Tensor:
384
385 return fused_moe(x,
386 layer.w13_weight,
@@ -390,7 +393,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
393 w1_scale=layer.w13_scale,
394 w2_scale=layer.w2_scale,
395 a1_scale=layer.a13_scale,
393 - a2_scale=layer.a2_scale)
396 + a2_scale=layer.a2_scale,
397 + use_grouped_topk=use_grouped_topk,
398 + num_expert_group=num_expert_group,
399 + topk_group=topk_group)
400
401
402 class Fp8KVCacheMethod(QuantizeMethodBase):
vllm/model_executor/models/deepseek_v2.py
+69 -73
@@ -29,11 +29,10 @@ from transformers import PretrainedConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.config import CacheConfig
32 -from vllm.distributed import (get_tensor_model_parallel_rank,
33 - get_tensor_model_parallel_world_size,
32 +from vllm.distributed import (get_tensor_model_parallel_world_size,
33 tensor_model_parallel_all_reduce)
34 from vllm.model_executor.layers.activation import SiluAndMul
36 -from vllm.model_executor.layers.fused_moe import fused_experts, grouped_topk
35 +from vllm.model_executor.layers.fused_moe import FusedMoE
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
38 MergedColumnParallelLinear,
@@ -91,32 +90,34 @@ class DeepseekV2MoE(nn.Module):
90 quant_config: Optional[QuantizationConfig] = None,
91 ):
92 super().__init__()
94 - self.config = config
95 - self.rank = get_tensor_model_parallel_rank()
93 self.tp_size = get_tensor_model_parallel_world_size()
97 - self.n_routed_experts = config.n_routed_experts
98 - self.top_k = config.num_experts_per_tok
94 self.routed_scaling_factor = config.routed_scaling_factor
100 - if self.tp_size > self.n_routed_experts:
95 + self.n_shared_experts = config.n_shared_experts
96 + self.routed_scaling_factor = config.routed_scaling_factor
97 + if self.tp_size > config.n_routed_experts:
98 raise ValueError(
99 f"Tensor parallel size {self.tp_size} is greater than "
103 - f"the number of experts {self.n_routed_experts}.")
104 -
105 - self.experts = nn.ModuleList([
106 - DeepseekV2MLP(hidden_size=config.hidden_size,
107 - intermediate_size=config.moe_intermediate_size,
108 - hidden_act=config.hidden_act,
109 - quant_config=quant_config,
110 - reduce_results=False)
111 - for idx in range(self.n_routed_experts)
112 - ])
113 - self.pack_params()
100 + f"the number of experts {config.n_routed_experts}.")
101 +
102 + if config.hidden_act != "silu":
103 + raise ValueError(f"Unsupported activation: {config.hidden_act}. "
104 + "Only silu is supported for now.")
105 +
106 + self.experts = FusedMoE(num_experts=config.n_routed_experts,
107 + top_k=config.num_experts_per_tok,
108 + hidden_size=config.hidden_size,
109 + intermediate_size=config.moe_intermediate_size,
110 + reduce_results=False,
111 + renormalize=config.norm_topk_prob,
112 + quant_config=quant_config,
113 + use_grouped_topk=True,
114 + num_expert_group=config.n_group,
115 + topk_group=config.topk_group)
116
117 self.gate = ReplicatedLinear(config.hidden_size,
116 - self.n_routed_experts,
118 + config.n_routed_experts,
119 bias=False,
120 quant_config=None)
119 -
121 if config.n_shared_experts is not None:
122 intermediate_size = (config.moe_intermediate_size *
123 config.n_shared_experts)
@@ -128,50 +129,21 @@ class DeepseekV2MoE(nn.Module):
129 reduce_results=False,
130 )
131
131 - def pack_params(self):
132 - w1 = []
133 - w2 = []
134 - for expert in self.experts:
135 - w1.append(expert.gate_up_proj.weight)
136 - w2.append(expert.down_proj.weight)
137 - self.w1 = torch._utils._flatten_dense_tensors(w1)
138 - w1s = torch._utils._unflatten_dense_tensors(self.w1, w1)
139 - for data, param in zip(w1s, w1):
140 - param.data = data
141 - self.w1 = self.w1.view(len(w1), *w1s[0].shape)
142 -
143 - self.w2 = torch._utils._flatten_dense_tensors(w2)
144 - w2s = torch._utils._unflatten_dense_tensors(self.w2, w2)
145 - for data, param in zip(w2s, w2):
146 - param.data = data
147 -
148 - self.w2 = self.w2.view(len(w2), *w2s[0].shape)
149 -
132 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
133 num_tokens, hidden_dim = hidden_states.shape
134 hidden_states = hidden_states.view(-1, hidden_dim)
153 - if self.config.n_shared_experts is not None:
135 + if self.n_shared_experts is not None:
136 shared_output = self.shared_experts(hidden_states)
137 # router_logits: (num_tokens, n_experts)
138 router_logits, _ = self.gate(hidden_states)
157 - topk_weights, topk_ids = grouped_topk(
158 - hidden_states,
159 - router_logits,
160 - self.top_k,
161 - renormalize=self.config.norm_topk_prob,
162 - num_expert_group=self.config.n_group,
163 - topk_group=self.config.topk_group)
164 - final_hidden_states = fused_experts(
165 - hidden_states,
166 - self.w1,
167 - self.w2,
168 - topk_weights,
169 - topk_ids,
170 - inplace=True) * self.routed_scaling_factor
171 - if self.config.n_shared_experts is not None:
139 + final_hidden_states = self.experts(
140 + hidden_states=hidden_states,
141 + router_logits=router_logits) * self.routed_scaling_factor
142 + if shared_output is not None:
143 final_hidden_states = final_hidden_states + shared_output
173 - final_hidden_states = tensor_model_parallel_all_reduce(
174 - final_hidden_states)
144 + if self.tp_size > 1:
145 + final_hidden_states = tensor_model_parallel_all_reduce(
146 + final_hidden_states)
147
148 return final_hidden_states.view(num_tokens, hidden_dim)
149
@@ -504,34 +476,58 @@ class DeepseekV2ForCausalLM(nn.Module):
476 ("gate_up_proj", "up_proj", 1),
477 ]
478
479 + # Params for weights, fp8 weight scales, fp8 activation scales
480 + # (param_name, weight_name, expert_id, shard_id)
481 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
482 + ckpt_gate_proj_name="gate_proj",
483 + ckpt_down_proj_name="down_proj",
484 + ckpt_up_proj_name="up_proj",
485 + num_experts=self.config.n_routed_experts)
486 +
487 params_dict = dict(self.named_parameters())
488 for name, loaded_weight in weights:
489 if "rotary_emb.inv_freq" in name:
490 continue
491 for (param_name, weight_name, shard_id) in stacked_params_mapping:
492 + # Skip non-stacked layers and experts (experts handled below).
493 if weight_name not in name:
494 continue
495 + # We have mlp.experts[0].gate_proj in the checkpoint.
496 + # Since we handle the experts below in expert_params_mapping,
497 + # we need to skip here BEFORE we update the name, otherwise
498 + # name will be updated to mlp.experts[0].gate_up_proj, which
499 + # will then be updated below in expert_params_mapping
500 + # for mlp.experts[0].gate_gate_up_proj, which breaks load.
501 + if (("mlp.experts." in name) and name not in params_dict):
502 + continue
503 name = name.replace(weight_name, param_name)
504 # Skip loading extra bias for GPTQ models.
505 if name.endswith(".bias") and name not in params_dict:
506 continue
518 - # Skip experts that are not assigned to this worker.
519 - if (("mlp.experts." in name or "mlp.shared_experts." in name)
520 - and name not in params_dict):
521 - continue
507 param = params_dict[name]
508 weight_loader = param.weight_loader
509 weight_loader(param, loaded_weight, shard_id)
510 break
511 else:
527 - # Skip loading extra bias for GPTQ models.
528 - if name.endswith(".bias") and name not in params_dict:
529 - continue
530 - # Skip experts that are not assigned to this worker.
531 - if (("mlp.experts." in name or "mlp.shared_experts." in name)
532 - and name not in params_dict):
533 - continue
534 - param = params_dict[name]
535 - weight_loader = getattr(param, "weight_loader",
536 - default_weight_loader)
537 - weight_loader(param, loaded_weight)
512 + for mapping in expert_params_mapping:
513 + param_name, weight_name, expert_id, shard_id = mapping
514 + if weight_name not in name:
515 + continue
516 + name = name.replace(weight_name, param_name)
517 + param = params_dict[name]
518 + weight_loader = param.weight_loader
519 + weight_loader(param,
520 + loaded_weight,
521 + weight_name,
522 + shard_id=shard_id,
523 + expert_id=expert_id)
524 + break
525 + else:
526 + # Skip loading extra bias for GPTQ models.
527 + if name.endswith(".bias") and name not in params_dict:
528 + continue
529 +
530 + param = params_dict[name]
531 + weight_loader = getattr(param, "weight_loader",
532 + default_weight_loader)
533 + weight_loader(param, loaded_weight)
vllm/model_executor/models/mixtral.py
+7 -25
@@ -372,31 +372,13 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
372 ("qkv_proj", "v_proj", "v"),
373 ]
374
375 - expert_params_mapping = [
376 - # These are the weight scales for the experts
377 - # (param_name, weight_name, expert_id, shard_id)
378 - ("experts.w13_scale"
379 - if weight_name in ["w1", "w3"] else "experts.w2_scale",
380 - f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
381 - shard_id) for expert_id in range(self.config.num_local_experts)
382 - for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
383 - ] + [
384 - # These are the weights for the experts
385 - # (param_name, weight_name, expert_id)
386 - ("experts.w13_weight"
387 - if weight_name in ["w1", "w3"] else "experts.w2_weight",
388 - f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
389 - for expert_id in range(self.config.num_local_experts)
390 - for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
391 - ] + [
392 - # These are the activation scales for the experts
393 - # (param_name, weight_name, expert_id)
394 - ("experts.a13_scale"
395 - if weight_name in ["w1", "w3"] else "experts.a2_scale",
396 - f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
397 - shard_id) for expert_id in range(self.config.num_local_experts)
398 - for shard_id, weight_name in enumerate(["w1", "w2", "w3"])
399 - ]
375 + # Params for weights, fp8 weight scales, fp8 activation scales
376 + # (param_name, weight_name, expert_id, shard_id)
377 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
378 + ckpt_gate_proj_name="w1",
379 + ckpt_down_proj_name="w2",
380 + ckpt_up_proj_name="w3",
381 + num_experts=self.config.num_local_experts)
382
383 params_dict = dict(self.named_parameters())
384 for name, loaded_weight in weights:
vllm/model_executor/models/qwen2_moe.py
+22 -11
@@ -50,6 +50,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.sequence import IntermediateTensors, SamplerOutput
53 +from vllm.utils import print_warning_once
54
55
56 class Qwen2MoeMLP(nn.Module):
@@ -406,15 +407,13 @@ class Qwen2MoeForCausalLM(nn.Module):
407 ("gate_up_proj", "up_proj", 1),
408 ]
409
409 - expert_params_mapping = [
410 - # These are the weights for the experts
411 - # (param_name, weight_name, expert_id, shard_id)
412 - ("experts.w13_weight" if weight_name in ["gate_proj", "up_proj"]
413 - else "experts.w2_weight",
414 - f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
415 - for expert_id in range(self.config.num_experts) for shard_id,
416 - weight_name in enumerate(["gate_proj", "down_proj", "up_proj"])
417 - ]
410 + # Params for weights, fp8 weight scales, fp8 activation scales
411 + # (param_name, weight_name, expert_id, shard_id)
412 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
413 + ckpt_gate_proj_name="gate_proj",
414 + ckpt_down_proj_name="down_proj",
415 + ckpt_up_proj_name="up_proj",
416 + num_experts=self.config.num_experts)
417
418 params_dict = dict(self.named_parameters())
419 for name, loaded_weight in weights:
@@ -461,8 +460,20 @@ class Qwen2MoeForCausalLM(nn.Module):
460 # Skip loading extra bias for GPTQ models.
461 if name.endswith(".bias") and name not in params_dict:
462 continue
464 - if name not in params_dict:
465 - continue
463 + # Remapping the name of FP8 kv-scale.
464 + if name.endswith("kv_scale"):
465 + remapped_kv_scale_name = name.replace(
466 + ".kv_scale", ".attn.kv_scale")
467 + if remapped_kv_scale_name not in params_dict:
468 + print_warning_once(
469 + "Found kv scale in the checkpoint "
470 + f"(e.g. {name}), but not found the expected "
471 + f"name in the model "
472 + f"(e.g. {remapped_kv_scale_name}). "
473 + "kv-scale is not loaded.")
474 + continue
475 + else:
476 + name = remapped_kv_scale_name
477
478 param = params_dict[name]
479 weight_loader = getattr(param, "weight_loader",