@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d3bdfd3ab9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Misc] Update Fused MoE weight loading (#7334)

Dipika Sikka committed Aug 13, 2024 at 14:57 UTC d3bdfd3ab9bac6bf1a88f717869bf9c06683d4b4
6 files changed +264 -201
vllm/model_executor/layers/fused_moe/layer.py
+180 -136
@@ -24,15 +24,9 @@ class FusedMoEMethodBase(QuantizeMethodBase):
24 raise NotImplementedError
25
26 @abstractmethod
27 - def apply(self,
28 - layer: torch.nn.Module,
29 - x: torch.Tensor,
30 - router_logits: torch.Tensor,
31 - top_k: int,
32 - renormalize: bool = True,
33 - use_grouped_topk: bool = False,
34 - num_expert_group: Optional[int] = None,
35 - topk_group: Optional[int] = None) -> torch.Tensor:
27 + def apply(self, layer: torch.nn.Module, x: torch.Tensor,
28 + router_logits: torch.Tensor, top_k: int, renormalize: bool,
29 + use_grouped_topk: bool) -> torch.Tensor:
30 raise NotImplementedError
31
32
@@ -61,66 +55,78 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
55 layer.register_parameter("w2_weight", w2_weight)
56 set_weight_attrs(w2_weight, extra_weight_attrs)
57
64 - def apply(
65 - self,
66 - layer: torch.nn.Module,
67 - x: torch.Tensor,
68 - router_logits: torch.Tensor,
69 - top_k: int,
70 - renormalize: bool = True,
71 - use_grouped_topk: bool = False,
72 - num_expert_group: Optional[int] = None,
73 - topk_group: Optional[int] = None,
74 - ) -> torch.Tensor:
75 - return self.forward(x, layer.w13_weight, layer.w2_weight,
76 - router_logits, top_k, renormalize,
77 - use_grouped_topk, num_expert_group, topk_group)
78 -
79 - def forward_cuda(
80 - self,
81 - x: torch.Tensor,
82 - w1: torch.Tensor,
83 - w2: torch.Tensor,
84 - router_logits: torch.Tensor,
85 - top_k: int,
86 - renormalize: bool,
87 - use_grouped_topk: bool,
88 - num_expert_group: Optional[int],
89 - topk_group: Optional[int],
90 - ) -> torch.Tensor:
91 - from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
92 - return fused_moe(x,
93 - w1,
94 - w2,
95 - router_logits,
96 - top_k,
97 - renormalize=renormalize,
98 - inplace=True,
99 - use_grouped_topk=use_grouped_topk,
100 - num_expert_group=num_expert_group,
101 - topk_group=topk_group)
58 + def apply(self,
59 + layer: torch.nn.Module,
60 + x: torch.Tensor,
61 + router_logits: torch.Tensor,
62 + top_k: int,
63 + renormalize: bool,
64 + use_grouped_topk: bool,
65 + topk_group: Optional[int] = None,
66 + num_expert_group: Optional[int] = None) -> torch.Tensor:
67 +
68 + return self.forward(x=x,
69 + layer=layer,
70 + router_logits=router_logits,
71 + top_k=top_k,
72 + renormalize=renormalize,
73 + use_grouped_topk=use_grouped_topk,
74 + topk_group=topk_group,
75 + num_expert_group=num_expert_group)
76 +
77 + def forward_cuda(self,
78 + layer: torch.nn.Module,
79 + x: torch.Tensor,
80 + use_grouped_topk: bool,
81 + top_k: int,
82 + router_logits: torch.Tensor,
83 + renormalize: bool,
84 + topk_group: Optional[int] = None,
85 + num_expert_group: Optional[int] = None) -> torch.Tensor:
86 +
87 + from vllm.model_executor.layers.fused_moe.fused_moe import (
88 + fused_experts)
89 +
90 + topk_weights, topk_ids = FusedMoE.select_experts(
91 + hidden_states=x,
92 + router_logits=router_logits,
93 + use_grouped_topk=use_grouped_topk,
94 + top_k=top_k,
95 + renormalize=renormalize,
96 + topk_group=topk_group,
97 + num_expert_group=num_expert_group)
98 +
99 + return fused_experts(hidden_states=x,
100 + w1=layer.w13_weight,
101 + w2=layer.w2_weight,
102 + topk_weights=topk_weights,
103 + topk_ids=topk_ids,
104 + inplace=True)
105
106 def forward_cpu(self, *args, **kwargs):
107 raise NotImplementedError(
108 "The CPU backend currently does not support MoE.")
109
107 - def forward_tpu(
108 - self,
109 - x: torch.Tensor,
110 - w1: torch.Tensor,
111 - w2: torch.Tensor,
112 - router_logits: torch.Tensor,
113 - top_k: int,
114 - renormalize: bool,
115 - use_grouped_topk: bool,
116 - num_expert_group: Optional[int],
117 - topk_group: Optional[int],
118 - ) -> torch.Tensor:
110 + def forward_tpu(self,
111 + layer: torch.nn.Module,
112 + x: torch.Tensor,
113 + use_grouped_topk: bool,
114 + top_k: int,
115 + router_logits: torch.Tensor,
116 + renormalize: bool,
117 + topk_group: Optional[int] = None,
118 + num_expert_group: Optional[int] = None) -> torch.Tensor:
119 +
120 from vllm.model_executor.layers.fused_moe.moe_pallas import fused_moe
121 assert not use_grouped_topk
122 assert num_expert_group is None
123 assert topk_group is None
123 - return fused_moe(x, w1, w2, router_logits, top_k, renormalize)
124 + return fused_moe(hidden_states=x,
125 + w1=layer.w13_weight,
126 + w2=layer.w2_weight,
127 + topk=top_k,
128 + gating_output=router_logits,
129 + renormalize=renormalize)
130
131
132 class FusedMoE(torch.nn.Module):
@@ -195,52 +201,83 @@ class FusedMoE(torch.nn.Module):
201
202 def weight_loader(self, param: torch.nn.Parameter,
203 loaded_weight: torch.Tensor, weight_name: str,
198 - shard_id: int, expert_id: int):
199 - param_data = param.data
200 -
201 - # Input scales can be loaded directly and should be equal.
202 - if "input_scale" in weight_name:
203 - if param_data[expert_id] != 1 and (param_data[expert_id] -
204 - loaded_weight).abs() > 1e-5:
205 - raise ValueError(
206 - "input_scales of w1 and w3 of a layer "
207 - f"must be equal. But got {param_data[expert_id]} "
208 - f"vs. {loaded_weight}")
209 - param_data[expert_id] = loaded_weight
210 - # Weight scales
211 - elif "weight_scale" in weight_name:
212 - # If we are in merged column case (gate_up_proj)
213 - # shard_id 0 == gate_proj / w1
214 - # shard_id 2 == up_proj / w3
215 - if shard_id == 0 or shard_id == 2:
216 - # We have to keep the weight scales of w1 and w3 because
217 - # we need to re-quantize w1/w3 weights after weight loading.
218 - idx = 0 if shard_id == 0 else 1
219 - param_data[expert_id][idx] = loaded_weight
220 - # If we are in the row parallel case (down_proj)
221 - # shard_id 1 == down_proj / w2
222 - else:
223 - param_data[expert_id] = loaded_weight
224 - # Weights
204 + shard_id: str, expert_id: int) -> None:
205 + if shard_id not in ("w1", "w2", "w3"):
206 + raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
207 + f"got {shard_id}.")
208 +
209 + # Special case for fp8 scales.
210 + if getattr(param, "is_fp8_scale", False):
211 + self._load_fp8_scale(param.data, loaded_weight, weight_name,
212 + shard_id, expert_id)
213 + return
214 +
215 + expert_data = param.data[expert_id]
216 + tp_rank = get_tensor_model_parallel_rank()
217 +
218 + # If transposed, weight is saved as [input_dim, output_dim]
219 + # Otherwise, weight is saved as [output_dim, input_dim]
220 + # Default is not transposed/input dim is dim 1
221 + input_dim = getattr(param, "input_dim", 1)
222 + output_dim = getattr(param, "output_dim", 0)
223 +
224 + # Index the loaded weight for tp sharding.
225 + # down_proj: "RowParallel" so tp sharding on input_dim
226 + if shard_id == "w2":
227 + shard_dim = input_dim
228 + shard_size = expert_data.shape[shard_dim]
229 + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim
230 + elif shard_id in ("w1", "w3"):
231 + shard_dim = output_dim
232 + shard_size = expert_data.shape[output_dim] // 2
233 + offset = shard_size * tp_rank
234 + loaded_weight = loaded_weight.narrow(shard_dim, offset, shard_size)
235 +
236 + # Narrow parameter and load.
237 + # w1, gate_proj: Load into first logical weight of w13.
238 + if shard_id == "w1":
239 + expert_data = expert_data.narrow(shard_dim, 0, shard_size)
240 + expert_data.copy_(loaded_weight)
241 + # w3, up_proj: Load into second logical weight of w13.
242 + elif shard_id == "w3":
243 + expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
244 + expert_data.copy_(loaded_weight)
245 + # w2, down_proj: Load into only logical weight of w2.
246 + elif shard_id == "w2":
247 + expert_data.copy_(loaded_weight)
248 else:
226 - tp_rank = get_tensor_model_parallel_rank()
227 - shard_size = self.intermediate_size_per_partition
228 - shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
229 -
230 - # w1, gate_proj case: Load into first shard of w13.
231 - if shard_id == 0:
232 - param_data[expert_id,
233 - 0:shard_size, :] = loaded_weight[shard, :]
234 - # w3, up_proj case: Load into second shard of w13.
235 - elif shard_id == 2:
236 - param_data[expert_id, shard_size:2 *
237 - shard_size, :] = loaded_weight[shard, :]
238 - # w2, down_proj case: Load into only shard of w2.
239 - elif shard_id == 1:
240 - param_data[expert_id, :, :] = loaded_weight[:, shard]
241 - else:
242 - raise ValueError(
243 - f"Shard id must be in [0,1,2] but got {shard_id}")
249 + raise ValueError(
250 + f"Expected shard_id w1,w2 or w3 but got {shard_id}")
251 +
252 + @staticmethod
253 + def select_experts(hidden_states: torch.Tensor,
254 + router_logits: torch.Tensor,
255 + top_k: int,
256 + use_grouped_topk: bool,
257 + renormalize: bool,
258 + topk_group: Optional[int] = None,
259 + num_expert_group: Optional[int] = None):
260 + from vllm.model_executor.layers.fused_moe.fused_moe import (
261 + fused_topk, grouped_topk)
262 +
263 + # DeekSeekv2 uses grouped_top_k
264 + if use_grouped_topk:
265 + assert topk_group is not None
266 + assert num_expert_group is not None
267 + topk_weights, topk_ids = grouped_topk(
268 + hidden_states=hidden_states,
269 + gating_output=router_logits,
270 + topk=top_k,
271 + renormalize=renormalize,
272 + num_expert_group=num_expert_group,
273 + topk_group=topk_group)
274 + else:
275 + topk_weights, topk_ids = fused_topk(hidden_states=hidden_states,
276 + gating_output=router_logits,
277 + topk=top_k,
278 + renormalize=renormalize)
279 +
280 + return topk_weights, topk_ids
281
282 def forward(self, hidden_states: torch.Tensor,
283 router_logits: torch.Tensor):
@@ -248,14 +285,14 @@ class FusedMoE(torch.nn.Module):
285
286 # Matrix multiply.
287 final_hidden_states = self.quant_method.apply(
251 - self,
288 + layer=self,
289 x=hidden_states,
290 router_logits=router_logits,
291 top_k=self.top_k,
292 renormalize=self.renormalize,
293 use_grouped_topk=self.use_grouped_topk,
257 - num_expert_group=self.num_expert_group,
258 - topk_group=self.topk_group)
294 + topk_group=self.topk_group,
295 + num_expert_group=self.num_expert_group)
296
297 if self.reduce_results and self.tp_size > 1:
298 final_hidden_states = tensor_model_parallel_all_reduce(
@@ -267,35 +304,42 @@ class FusedMoE(torch.nn.Module):
304 def make_expert_params_mapping(
305 cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str,
306 ckpt_up_proj_name: str,
270 - num_experts: int) -> List[Tuple[str, str, int, int]]:
271 -
272 - gate_up = [ckpt_gate_proj_name, ckpt_up_proj_name]
273 - gate_down_up = [
274 - ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name
275 - ]
307 + num_experts: int) -> List[Tuple[str, str, int, str]]:
308
309 return [
278 - # These are the weight scales for the experts
279 - # (param_name, weight_name, expert_id, shard_id)
280 - ("experts.w13_scale"
281 - if weight_name in gate_up else "experts.w2_scale",
282 - f"experts.{expert_id}.{weight_name}.weight_scale", expert_id,
283 - shard_id) for expert_id in range(num_experts)
284 - for shard_id, weight_name in enumerate(gate_down_up)
285 - ] + [
286 - # These are the weights for the experts
310 # (param_name, weight_name, expert_id, shard_id)
288 - ("experts.w13_weight"
289 - if weight_name in gate_up else "experts.w2_weight",
290 - f"experts.{expert_id}.{weight_name}.weight", expert_id, shard_id)
291 - for expert_id in range(num_experts)
292 - for shard_id, weight_name in enumerate(gate_down_up)
293 - ] + [
294 - # These are the weight scales for the experts
295 - # (param_name, weight_name, expert_id, shard_id)
296 - ("experts.a13_scale"
297 - if weight_name in gate_up else "experts.a2_scale",
298 - f"experts.{expert_id}.{weight_name}.input_scale", expert_id,
299 - shard_id) for expert_id in range(num_experts)
300 - for shard_id, weight_name in enumerate(gate_down_up)
311 + ("experts.w13_" if weight_name
312 + in [ckpt_gate_proj_name, ckpt_up_proj_name] else "experts.w2_",
313 + f"experts.{expert_id}.{weight_name}.", expert_id, shard_id)
314 + for expert_id in range(num_experts) for shard_id, weight_name in [
315 + ("w1", ckpt_gate_proj_name),
316 + ("w2", ckpt_down_proj_name),
317 + ("w3", ckpt_up_proj_name),
318 + ]
319 ]
320 +
321 + def _load_fp8_scale(self, param: torch.nn.Parameter,
322 + loaded_weight: torch.Tensor, weight_name: str,
323 + shard_id: str, expert_id: int) -> None:
324 + param_data = param.data
325 +
326 + # Input scales can be loaded directly and should be equal.
327 + if "input_scale" in weight_name:
328 + if param_data[expert_id] != 1 and (param_data[expert_id] -
329 + loaded_weight).abs() > 1e-5:
330 + raise ValueError(
331 + "input_scales of w1 and w3 of a layer "
332 + f"must be equal. But got {param_data[expert_id]} "
333 + f"vs. {loaded_weight}")
334 + param_data[expert_id] = loaded_weight
335 + # Weight scales
336 + elif "weight_scale" in weight_name:
337 + # If we are in merged column case (gate_up_proj)
338 + if shard_id in ("w1", "w3"):
339 + # We have to keep the weight scales of w1 and w3 because
340 + # we need to re-quantize w1/w3 weights after weight loading.
341 + idx = 0 if shard_id == "w1" else 1
342 + param_data[expert_id][idx] = loaded_weight
343 + # If we are in the row parallel case (down_proj)
344 + else:
345 + param_data[expert_id] = loaded_weight
vllm/model_executor/layers/quantization/fp8.py
+80 -61
@@ -290,23 +290,29 @@ class Fp8MoEMethod(FusedMoEMethodBase):
290 # WEIGHT_SCALES
291 # Allocate 2 scales for w1 and w3 respectively.
292 # They will be combined to a single scale after weight loading.
293 - w13_scale = torch.nn.Parameter(torch.ones(num_experts,
294 - 2,
295 - dtype=torch.float32),
296 - requires_grad=False)
297 - layer.register_parameter("w13_scale", w13_scale)
293 + w13_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
294 + 2,
295 + dtype=torch.float32),
296 + requires_grad=False)
297 + layer.register_parameter("w13_weight_scale", w13_weight_scale)
298
299 - w2_scale = torch.nn.Parameter(torch.ones(num_experts,
300 - dtype=torch.float32),
301 - requires_grad=False)
302 - layer.register_parameter("w2_scale", w2_scale)
299 + w2_weight_scale = torch.nn.Parameter(torch.ones(num_experts,
300 + dtype=torch.float32),
301 + requires_grad=False)
302 + layer.register_parameter("w2_weight_scale", w2_weight_scale)
303
304 # If loading fp8 checkpoint, pass the weight loaders.
305 # If loading an fp16 checkpoint, do not (we will quantize in
306 # process_weights_after_loading()
307 if self.quant_config.is_checkpoint_fp8_serialized:
308 - set_weight_attrs(w13_scale, extra_weight_attrs)
309 - set_weight_attrs(w2_scale, extra_weight_attrs)
308 + set_weight_attrs(w13_weight_scale, {
309 + "is_fp8_scale": True,
310 + **extra_weight_attrs
311 + })
312 + set_weight_attrs(w2_weight_scale, {
313 + "is_fp8_scale": True,
314 + **extra_weight_attrs
315 + })
316
317 # INPUT_SCALES
318 if self.quant_config.activation_scheme == "static":
@@ -315,20 +321,26 @@ class Fp8MoEMethod(FusedMoEMethodBase):
321 "Found static activation scheme for checkpoint that "
322 "was not serialized fp8.")
323
318 - a13_scale = torch.nn.Parameter(torch.ones(num_experts,
319 - dtype=torch.float32),
320 - requires_grad=False)
321 - layer.register_parameter("a13_scale", a13_scale)
322 - set_weight_attrs(a13_scale, extra_weight_attrs)
323 -
324 - a2_scale = torch.nn.Parameter(torch.ones(num_experts,
325 - dtype=torch.float32),
326 - requires_grad=False)
327 - layer.register_parameter("a2_scale", a2_scale)
328 - set_weight_attrs(a2_scale, extra_weight_attrs)
324 + w13_input_scale = torch.nn.Parameter(torch.ones(
325 + num_experts, dtype=torch.float32),
326 + requires_grad=False)
327 + layer.register_parameter("w13_input_scale", w13_input_scale)
328 + set_weight_attrs(w13_input_scale, {
329 + "is_fp8_scale": True,
330 + **extra_weight_attrs
331 + })
332 +
333 + w2_input_scale = torch.nn.Parameter(torch.ones(
334 + num_experts, dtype=torch.float32),
335 + requires_grad=False)
336 + layer.register_parameter("w2_input_scale", w2_input_scale)
337 + set_weight_attrs(w2_input_scale, {
338 + "is_fp8_scale": True,
339 + **extra_weight_attrs
340 + })
341 else:
330 - layer.a13_scale = None
331 - layer.a2_scale = None
342 + layer.w13_input_scale = None
343 + layer.w2_input_scale = None
344
345 def process_weights_after_loading(self, layer: Module) -> None:
346
@@ -341,16 +353,16 @@ class Fp8MoEMethod(FusedMoEMethodBase):
353
354 # Re-initialize w13_scale because we directly quantize
355 # merged w13 weights and generate a single scaling factor.
344 - layer.w13_scale = torch.nn.Parameter(torch.ones(
356 + layer.w13_weight_scale = torch.nn.Parameter(torch.ones(
357 layer.num_experts,
358 dtype=torch.float32,
359 device=w13_weight.device),
348 - requires_grad=False)
360 + requires_grad=False)
361 for expert in range(layer.num_experts):
350 - w13_weight[expert, :, :], layer.w13_scale[
362 + w13_weight[expert, :, :], layer.w13_weight_scale[
363 expert] = ops.scaled_fp8_quant(
364 layer.w13_weight.data[expert, :, :])
353 - w2_weight[expert, :, :], layer.w2_scale[
365 + w2_weight[expert, :, :], layer.w2_weight_scale[
366 expert] = ops.scaled_fp8_quant(
367 layer.w2_weight.data[expert, :, :])
368 layer.w13_weight = torch.nn.Parameter(w13_weight,
@@ -366,40 +378,41 @@ class Fp8MoEMethod(FusedMoEMethodBase):
378 # Fp8 moe kernels require a single activation scale.
379 # We take the max of all the scales in case they differ.
380 if self.quant_config.activation_scheme == "static":
369 - if layer.a13_scale is None or layer.a2_scale is None:
381 + if (layer.w13_input_scale is None
382 + or layer.w2_input_scale is None):
383 raise ValueError(
384 "QuantConfig has static quantization, but found "
385 "activation scales are None.")
373 - if (not all_close_1d(layer.a13_scale)
374 - or not all_close_1d(layer.a2_scale)):
386 + if (not all_close_1d(layer.w13_input_scale)
387 + or not all_close_1d(layer.w2_input_scale)):
388 print_warning_once(
389 "Found input_scales that are not equal for "
390 "fp8 MoE layer. Using the maximum across experts "
391 "for each layer. ")
379 - layer.a13_scale = torch.nn.Parameter(layer.a13_scale.max(),
380 - requires_grad=False)
381 - layer.a2_scale = torch.nn.Parameter(layer.a2_scale.max(),
382 - requires_grad=False)
392 + layer.w13_input_scale = torch.nn.Parameter(
393 + layer.w13_input_scale.max(), requires_grad=False)
394 + layer.w2_input_scale = torch.nn.Parameter(
395 + layer.w2_input_scale.max(), requires_grad=False)
396
397 # Fp8 moe kernel needs single weight scale for w13 per expert.
398 # We take the max then dequant and requant each expert.
386 - assert layer.w13_scale is not None
399 + assert layer.w13_weight_scale is not None
400 shard_size = layer.intermediate_size_per_partition
388 - max_w13_scales = layer.w13_scale.max(dim=1).values
401 + max_w13_scales = layer.w13_weight_scale.max(dim=1).values
402 for expert_id in range(layer.num_experts):
403 start = 0
404 for shard_id in range(2):
405 dq_weight = per_tensor_dequantize(
406 layer.w13_weight[expert_id][start:start +
407 shard_size, :],
395 - layer.w13_scale[expert_id][shard_id])
408 + layer.w13_weight_scale[expert_id][shard_id])
409 layer.w13_weight[expert_id][
410 start:start + shard_size, :], _ = ops.scaled_fp8_quant(
411 dq_weight, max_w13_scales[expert_id])
412 start += shard_size
413
401 - layer.w13_scale = torch.nn.Parameter(max_w13_scales,
402 - requires_grad=False)
414 + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales,
415 + requires_grad=False)
416 return
417
418 def apply(self,
@@ -407,27 +420,33 @@ class Fp8MoEMethod(FusedMoEMethodBase):
420 x: torch.Tensor,
421 router_logits: torch.Tensor,
422 top_k: int,
410 - renormalize: bool = True,
411 - use_grouped_topk: bool = False,
412 - num_expert_group: Optional[int] = None,
413 - topk_group: Optional[int] = None) -> torch.Tensor:
414 -
415 - from vllm.model_executor.layers.fused_moe import fused_moe
416 - return fused_moe(x,
417 - layer.w13_weight,
418 - layer.w2_weight,
419 - router_logits,
420 - top_k,
421 - renormalize=renormalize,
422 - inplace=True,
423 - use_fp8=True,
424 - w1_scale=layer.w13_scale,
425 - w2_scale=layer.w2_scale,
426 - a1_scale=layer.a13_scale,
427 - a2_scale=layer.a2_scale,
428 - use_grouped_topk=use_grouped_topk,
429 - num_expert_group=num_expert_group,
430 - topk_group=topk_group)
423 + renormalize: bool,
424 + use_grouped_topk: bool,
425 + topk_group: Optional[int] = None,
426 + num_expert_group: Optional[int] = None) -> torch.Tensor:
427 +
428 + from vllm.model_executor.layers.fused_moe import fused_experts
429 +
430 + topk_weights, topk_ids = FusedMoE.select_experts(
431 + hidden_states=x,
432 + router_logits=router_logits,
433 + use_grouped_topk=use_grouped_topk,
434 + top_k=top_k,
435 + renormalize=renormalize,
436 + topk_group=topk_group,
437 + num_expert_group=num_expert_group)
438 +
439 + return fused_experts(x,
440 + layer.w13_weight,
441 + layer.w2_weight,
442 + topk_weights=topk_weights,
443 + topk_ids=topk_ids,
444 + inplace=True,
445 + use_fp8=True,
446 + w1_scale=layer.w13_weight_scale,
447 + w2_scale=layer.w2_weight_scale,
448 + a1_scale=layer.w13_input_scale,
449 + a2_scale=layer.w2_input_scale)
450
451
452 class Fp8KVCacheMethod(BaseKVCacheMethod):
vllm/model_executor/models/deepseek_v2.py
+1 -1
@@ -593,7 +593,7 @@ class DeepseekV2ForCausalLM(nn.Module):
593 weight_loader = param.weight_loader
594 weight_loader(param,
595 loaded_weight,
596 - weight_name,
596 + name,
597 shard_id=shard_id,
598 expert_id=expert_id)
599 break
vllm/model_executor/models/jamba.py
+1 -1
@@ -930,7 +930,7 @@ class JambaForCausalLM(nn.Module, HasInnerState):
930 weight_loader = param.weight_loader
931 weight_loader(param,
932 loaded_weight,
933 - weight_name,
933 + name,
934 shard_id=shard_id,
935 expert_id=expert_id)
936 break
vllm/model_executor/models/mixtral.py
+1 -1
@@ -455,7 +455,7 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA):
455 weight_loader = param.weight_loader
456 weight_loader(param,
457 loaded_weight,
458 - weight_name,
458 + name,
459 shard_id=shard_id,
460 expert_id=expert_id)
461 break
vllm/model_executor/models/qwen2_moe.py
+1 -1
@@ -492,7 +492,7 @@ class Qwen2MoeForCausalLM(nn.Module):
492 weight_loader = param.weight_loader
493 weight_loader(param,
494 loaded_weight,
495 - weight_name,
495 + name,
496 shard_id=shard_id,
497 expert_id=expert_id)
498 break