@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb5cb5e528
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[BugFix] Fix parameter names and `process_after_weight_loading` for W4A16 MoE Group Act Order (#11528)

Signed-off-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Dipika Sikka committed Jan 23, 2025 at 16:40 UTC eb5cb5e5280c83226408c3d1e30928364c6258db
8 files changed +243 -148
vllm/model_executor/layers/fused_moe/layer.py
+60 -34
@@ -38,7 +38,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
38
39 @abstractmethod
40 def create_weights(self, layer: torch.nn.Module, num_experts: int,
41 - hidden_size: int, intermediate_size: int,
41 + hidden_size: int, intermediate_size_per_partition: int,
42 params_dtype: torch.dtype, **extra_weight_attrs):
43 raise NotImplementedError
44
@@ -65,22 +65,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
65 """MoE method without quantization."""
66
67 def create_weights(self, layer: torch.nn.Module, num_experts: int,
68 - hidden_size: int, intermediate_size: int,
68 + hidden_size: int, intermediate_size_per_partition: int,
69 params_dtype: torch.dtype, **extra_weight_attrs):
70 # Fused gate_up_proj (column parallel)
71 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
72 - 2 * intermediate_size,
73 - hidden_size,
74 - dtype=params_dtype),
71 + w13_weight = torch.nn.Parameter(torch.empty(
72 + num_experts,
73 + 2 * intermediate_size_per_partition,
74 + hidden_size,
75 + dtype=params_dtype),
76 requires_grad=False)
77 layer.register_parameter("w13_weight", w13_weight)
78 set_weight_attrs(w13_weight, extra_weight_attrs)
79
80 # down_proj (row parallel)
80 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
81 - hidden_size,
82 - intermediate_size,
83 - dtype=params_dtype),
81 + w2_weight = torch.nn.Parameter(torch.empty(
82 + num_experts,
83 + hidden_size,
84 + intermediate_size_per_partition,
85 + dtype=params_dtype),
86 requires_grad=False)
87 layer.register_parameter("w2_weight", w2_weight)
88 set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -289,13 +291,20 @@ class FusedMoE(torch.nn.Module):
291 self.quant_method = quant_config.get_quant_method(self, prefix)
292 assert self.quant_method is not None
293
292 - self.quant_method.create_weights(
293 - layer=self,
294 - num_experts=num_experts,
295 - hidden_size=hidden_size,
296 - intermediate_size=self.intermediate_size_per_partition,
297 - params_dtype=params_dtype,
298 - weight_loader=self.weight_loader)
294 + moe_quant_params = {
295 + "num_experts": num_experts,
296 + "hidden_size": hidden_size,
297 + "intermediate_size_per_partition":
298 + self.intermediate_size_per_partition,
299 + "params_dtype": params_dtype,
300 + "weight_loader": self.weight_loader,
301 + }
302 + # need full intermediate size pre-sharding for WNA16 act order
303 + if (self.quant_method.__class__.__name__ ==
304 + "CompressedTensorsWNA16MoEMethod"):
305 + moe_quant_params["intermediate_size_full"] = intermediate_size
306 +
307 + self.quant_method.create_weights(layer=self, **moe_quant_params)
308
309 def _load_per_tensor_weight_scale(self, shard_id: str,
310 param: torch.nn.Parameter,
@@ -312,19 +321,30 @@ class FusedMoE(torch.nn.Module):
321 elif shard_id == "w2":
322 param_data[expert_id] = loaded_weight
323
315 - def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
324 + def _load_model_weight_or_group_weight_scale(self,
325 + shard_dim: int,
326 expert_data: torch.Tensor,
327 shard_id: str,
328 loaded_weight: torch.Tensor,
319 - tp_rank: int):
320 - # Load grouped weight scales for group quantization
321 - # or model weights
329 + tp_rank: int,
330 + load_full_w2: bool = False):
331 + """
332 + Load grouped weight scales for group quantization or model weights
333 + :param shard_dim: dimension to shard
334 + :param expert_data: parameter for a particular expert
335 + :param shard_id: either w1, w2, or w3
336 + :param loaded_weight: checkpoint weight to load into the param
337 + :param tp_rank: tensor parallel rank
338 + :param load_full_w2: whether or not the w2 loaded should be sharded.
339 + """
340 if shard_id == "w2":
323 - self._load_w2(shard_id=shard_id,
324 - shard_dim=shard_dim,
341 + # In the case where we have actorder/g_idx, we do not partition the
342 + # w2 scales, as indicated by `load_full` argument, for all tp cases
343 + self._load_w2(shard_dim=shard_dim,
344 loaded_weight=loaded_weight,
345 expert_data=expert_data,
327 - tp_rank=tp_rank)
346 + tp_rank=tp_rank,
347 + load_full=load_full_w2)
348 elif shard_id in ("w1", "w3"):
349 self._load_w13(shard_id=shard_id,
350 shard_dim=shard_dim,
@@ -364,15 +384,21 @@ class FusedMoE(torch.nn.Module):
384 expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
385 expert_data.copy_(loaded_weight)
386
367 - def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
368 - shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
387 + def _load_w2(self,
388 + expert_data: torch.Tensor,
389 + shard_dim: int,
390 + loaded_weight: torch.Tensor,
391 + tp_rank: int,
392 + load_full: bool = False):
393
394 # Index the loaded weight for tp sharding.
395 # down_proj: "RowParallel" so tp sharding on input_dim
396 # Narrow parameter and load.
397 shard_size = expert_data.shape[shard_dim]
374 - loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
375 - shard_size)
398 + if not load_full:
399 + loaded_weight = loaded_weight.narrow(shard_dim,
400 + shard_size * tp_rank,
401 + shard_size)
402 # w2, down_proj: Load into only logical weight of w2.
403 expert_data.copy_(loaded_weight)
404
@@ -387,8 +413,7 @@ class FusedMoE(torch.nn.Module):
413 shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int):
414
415 if shard_id == "w2":
390 - self._load_w2(shard_id=shard_id,
391 - shard_dim=shard_dim,
416 + self._load_w2(shard_dim=shard_dim,
417 loaded_weight=loaded_weight,
418 expert_data=expert_data,
419 tp_rank=tp_rank)
@@ -416,7 +441,7 @@ class FusedMoE(torch.nn.Module):
441 ]
442 # Fetch the dim to shard the parameter/loaded weight
443 # based on the shard id. This will be whatever
419 - # dimension intermediate_size is used.
444 + # dimension intermediate_size_per_partition is used.
445 SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
446
447 expert_data = param.data[expert_id]
@@ -424,11 +449,11 @@ class FusedMoE(torch.nn.Module):
449
450 # is_transposed: if the dim to shard the weight
451 # should be flipped. Required by GPTQ, compressed-tensors
427 - # should be whatever dimension intermediate_size is
452 + # should be whatever dimension intermediate_size_per_partition is
453 is_transposed = getattr(param, "is_transposed", False)
454 shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
455 if is_transposed:
431 - shard_dim = ~shard_dim
456 + shard_dim = int(not shard_dim)
457
458 # Case input scale: input_scale loading is only supported for fp8
459 if "input_scale" in weight_name:
@@ -480,7 +505,8 @@ class FusedMoE(torch.nn.Module):
505 shard_dim=shard_dim,
506 loaded_weight=loaded_weight,
507 expert_data=expert_data,
483 - tp_rank=tp_rank)
508 + tp_rank=tp_rank,
509 + load_full_w2=getattr(param, "load_full_w2", False))
510 elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
511 self._load_per_tensor_weight_scale(shard_id=shard_id,
512 param=param,
vllm/model_executor/layers/quantization/awq_marlin.py
+19 -16
@@ -303,7 +303,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
303 self.quant_config = quant_config
304
305 def create_weights(self, layer: torch.nn.Module, num_experts: int,
306 - hidden_size: int, intermediate_size: int,
306 + hidden_size: int, intermediate_size_per_partition: int,
307 params_dtype: torch.dtype, **extra_weight_attrs):
308 extra_weight_attrs.update({
309 "is_transposed":
@@ -312,17 +312,18 @@ class AWQMoEMethod(FusedMoEMethodBase):
312 FusedMoeWeightScaleSupported.GROUP.value,
313 })
314
315 - w13_qweight = Parameter(torch.empty(num_experts,
316 - hidden_size,
317 - 2 * intermediate_size //
318 - self.quant_config.pack_factor,
319 - dtype=torch.int32),
320 - requires_grad=False)
315 + w13_qweight = Parameter(
316 + torch.empty(num_experts,
317 + hidden_size,
318 + 2 * intermediate_size_per_partition //
319 + self.quant_config.pack_factor,
320 + dtype=torch.int32),
321 + requires_grad=False)
322 layer.register_parameter("w13_qweight", w13_qweight)
323 set_weight_attrs(w13_qweight, extra_weight_attrs)
324
325 w2_qweight = Parameter(torch.empty(num_experts,
325 - intermediate_size,
326 + intermediate_size_per_partition,
327 hidden_size //
328 self.quant_config.pack_factor,
329 dtype=torch.int32),
@@ -331,13 +332,14 @@ class AWQMoEMethod(FusedMoEMethodBase):
332 set_weight_attrs(w2_qweight, extra_weight_attrs)
333
334 num_groups_w13 = hidden_size // self.quant_config.group_size
334 - num_groups_w2 = intermediate_size // self.quant_config.group_size
335 + num_groups_w2 = (intermediate_size_per_partition //
336 + self.quant_config.group_size)
337
338 # WEIGHT_SCALES
339 # Allocate 2 scales for w1 and w3 respectively.
340 w13_scales = Parameter(torch.empty(num_experts,
341 num_groups_w13,
340 - intermediate_size * 2,
342 + intermediate_size_per_partition * 2,
343 dtype=params_dtype),
344 requires_grad=False)
345 layer.register_parameter("w13_scales", w13_scales)
@@ -353,12 +355,13 @@ class AWQMoEMethod(FusedMoEMethodBase):
355
356 # WEIGHT_ZERO_POINT
357 # Allocate 2 zero points for w1 and w3 respectively.
356 - w13_qzeros = Parameter(torch.empty(num_experts,
357 - num_groups_w13,
358 - 2 * intermediate_size //
359 - self.quant_config.pack_factor,
360 - dtype=torch.int32),
361 - requires_grad=False)
358 + w13_qzeros = Parameter(
359 + torch.empty(num_experts,
360 + num_groups_w13,
361 + 2 * intermediate_size_per_partition //
362 + self.quant_config.pack_factor,
363 + dtype=torch.int32),
364 + requires_grad=False)
365 layer.register_parameter("w13_qzeros", w13_qzeros)
366 set_weight_attrs(w13_qzeros, extra_weight_attrs)
367
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+103 -50
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
13 FusedMoeWeightScaleSupported)
14 from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
15 WNA16_SUPPORTED_BITS)
16 +from vllm.model_executor.layers.quantization.utils import replace_parameter
17 from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18 all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
19 from vllm.model_executor.utils import set_weight_attrs
@@ -75,24 +76,26 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
76 self.static_input_scales = not self.input_quant.dynamic
77
78 def create_weights(self, layer: torch.nn.Module, num_experts: int,
78 - hidden_size: int, intermediate_size: int,
79 + hidden_size: int, intermediate_size_per_partition: int,
80 params_dtype: torch.dtype, **extra_weight_attrs):
81
82 params_dtype = torch.float8_e4m3fn
83
84 # WEIGHTS
84 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
85 - 2 * intermediate_size,
86 - hidden_size,
87 - dtype=params_dtype),
85 + w13_weight = torch.nn.Parameter(torch.empty(
86 + num_experts,
87 + 2 * intermediate_size_per_partition,
88 + hidden_size,
89 + dtype=params_dtype),
90 requires_grad=False)
91 layer.register_parameter("w13_weight", w13_weight)
92 set_weight_attrs(w13_weight, extra_weight_attrs)
93
92 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
93 - hidden_size,
94 - intermediate_size,
95 - dtype=params_dtype),
94 + w2_weight = torch.nn.Parameter(torch.empty(
95 + num_experts,
96 + hidden_size,
97 + intermediate_size_per_partition,
98 + dtype=params_dtype),
99 requires_grad=False)
100 layer.register_parameter("w2_weight", w2_weight)
101 set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -254,6 +257,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
257 self.packed_factor = 32 // config.num_bits
258 self.strategy = config.strategy
259 self.group_size = config.group_size
260 + self.actorder = config.actorder
261 assert config.symmetric, (
262 "Only symmetric quantization is supported for MoE")
263
@@ -266,9 +270,16 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
270 f"{WNA16_SUPPORTED_BITS}")
271
272 def create_weights(self, layer: torch.nn.Module, num_experts: int,
269 - hidden_size: int, intermediate_size: int,
273 + hidden_size: int, intermediate_size_per_partition: int,
274 params_dtype: torch.dtype, **extra_weight_attrs):
275
276 + assert params_dtype == torch.float16, (
277 + "float16 is required for MoE compressed models. Set dtype=torch.float16" # noqa: E501
278 + )
279 +
280 + intermediate_size_full = extra_weight_attrs.pop(
281 + "intermediate_size_full")
282 +
283 # Will transpose the loaded weight along the
284 # intermediate and hidden dim sizes. Will
285 # shard for TP along the transposed dims
@@ -276,35 +287,45 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
287 "is_transposed": True,
288 "quant_method": self.strategy
289 })
279 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
280 - hidden_size //
281 - self.packed_factor,
282 - 2 * intermediate_size,
283 - dtype=torch.int32),
290 + w13_weight = torch.nn.Parameter(torch.empty(
291 + num_experts,
292 + hidden_size // self.packed_factor,
293 + 2 * intermediate_size_per_partition,
294 + dtype=torch.int32),
295 requires_grad=False)
296 layer.register_parameter("w13_weight_packed", w13_weight)
297 set_weight_attrs(w13_weight, extra_weight_attrs)
298
288 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
289 - intermediate_size //
290 - self.packed_factor,
291 - hidden_size,
292 - dtype=torch.int32),
299 + w2_weight = torch.nn.Parameter(torch.empty(
300 + num_experts,
301 + intermediate_size_per_partition // self.packed_factor,
302 + hidden_size,
303 + dtype=torch.int32),
304 requires_grad=False)
305 layer.register_parameter("w2_weight_packed", w2_weight)
306 set_weight_attrs(w2_weight, extra_weight_attrs)
307
308 + # In the case where we have actorder/g_idx,
309 + # we do not partition the w2 scales
310 + load_full_w2 = self.actorder and self.group_size != -1
311 + w2_scales_size = (intermediate_size_full
312 + if load_full_w2 else intermediate_size_per_partition)
313 +
314 + self.is_k_full = (not self.actorder) or (
315 + intermediate_size_per_partition == intermediate_size_full)
316 +
317 if self.strategy == "channel":
318 num_groups_w2 = num_groups_w13 = 1
319 self.group_size = -1
320 else:
301 - num_groups_w2 = intermediate_size // self.group_size
321 + num_groups_w2 = w2_scales_size // self.group_size
322 num_groups_w13 = hidden_size // self.group_size
323
304 - w13_scale = torch.nn.Parameter(torch.ones(num_experts,
305 - num_groups_w13,
306 - 2 * intermediate_size,
307 - dtype=params_dtype),
324 + w13_scale = torch.nn.Parameter(torch.ones(
325 + num_experts,
326 + num_groups_w13,
327 + 2 * intermediate_size_per_partition,
328 + dtype=params_dtype),
329 requires_grad=False)
330 layer.register_parameter("w13_weight_scale", w13_scale)
331 set_weight_attrs(w13_scale, extra_weight_attrs)
@@ -316,6 +337,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
337 requires_grad=False)
338 layer.register_parameter("w2_weight_scale", w2_scale)
339 set_weight_attrs(w2_scale, extra_weight_attrs)
340 + set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
341
342 w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
343 requires_grad=False)
@@ -335,18 +357,18 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
357 ),
358 requires_grad=False,
359 )
338 - layer.register_parameter("w13_g_idx", w13_g_idx)
360 + layer.register_parameter("w13_weight_g_idx", w13_g_idx)
361 set_weight_attrs(w13_g_idx, extra_weight_attrs)
362
363 w2_g_idx = torch.nn.Parameter(
364 torch.empty(
365 num_experts,
344 - intermediate_size,
366 + intermediate_size_per_partition,
367 dtype=torch.int32,
368 ),
369 requires_grad=False,
370 )
349 - layer.register_parameter("w2_g_idx", w2_g_idx)
371 + layer.register_parameter("w2_weight_g_idx", w2_g_idx)
372 set_weight_attrs(w2_g_idx, extra_weight_attrs)
373
374 w13_g_idx_sort_indices = torch.nn.Parameter(
@@ -364,7 +386,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
386 w2_g_idx_sort_indices = torch.nn.Parameter(
387 torch.empty(
388 num_experts,
367 - intermediate_size,
389 + intermediate_size_per_partition,
390 dtype=torch.int32,
391 ),
392 requires_grad=False,
@@ -422,24 +444,55 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
444 size_k2 = layer.w2_weight_packed.shape[2]
445 size_k13 = layer.w13_weight_packed.shape[2]
446
425 - num_experts = layer.w13_g_idx.shape[0]
426 - device = layer.w13_g_idx.device
427 - layer.w13_g_idx = torch.nn.Parameter(
428 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
429 - requires_grad=False,
430 - )
431 - layer.w2_g_idx = torch.nn.Parameter(
432 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
433 - requires_grad=False,
434 - )
435 - layer.w13_g_idx_sort_indices = torch.nn.Parameter(
436 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
437 - requires_grad=False,
438 - )
439 - layer.w2_g_idx_sort_indices = torch.nn.Parameter(
440 - torch.empty((num_experts, 0), dtype=torch.int32, device=device),
441 - requires_grad=False,
442 - )
447 + num_experts = layer.w13_weight_g_idx.shape[0]
448 + device = layer.w13_weight_g_idx.device
449 +
450 + # when running models with grouped act order,
451 + # resort to g_idx values provided in checkpoint
452 + if self.actorder == "group":
453 + w13_g_idx_sort_indices = torch.empty_like(layer.w13_weight_g_idx)
454 + w2_g_idx_sort_indices = torch.empty_like(layer.w2_weight_g_idx)
455 + w13_sorted_g_idx = torch.empty_like(layer.w13_weight_g_idx)
456 + w2_sorted_g_idx = torch.empty_like(layer.w2_weight_g_idx)
457 +
458 + for e in range(num_experts):
459 + w13_g_idx_sort_indices[e] = torch.argsort(
460 + layer.w13_weight_g_idx[e]).to(torch.int32)
461 + w2_g_idx_sort_indices[e] = torch.argsort(
462 + layer.w2_weight_g_idx[e]).to(torch.int32)
463 + w13_sorted_g_idx[e] = layer.w13_weight_g_idx[e][
464 + w13_g_idx_sort_indices[e]]
465 + w2_sorted_g_idx[e] = layer.w2_weight_g_idx[e][
466 + w2_g_idx_sort_indices[e]]
467 +
468 + replace_parameter(layer, "w13_weight_g_idx", w13_sorted_g_idx)
469 + replace_parameter(layer, "w2_weight_g_idx", w2_sorted_g_idx)
470 + replace_parameter(layer, "w13_g_idx_sort_indices",
471 + w13_g_idx_sort_indices)
472 + replace_parameter(layer, "w2_g_idx_sort_indices",
473 + w2_g_idx_sort_indices)
474 +
475 + else:
476 + layer.w13_weight_g_idx = torch.nn.Parameter(
477 + torch.empty((num_experts, 0), dtype=torch.int32,
478 + device=device),
479 + requires_grad=False,
480 + )
481 + layer.w2_weight_g_idx = torch.nn.Parameter(
482 + torch.empty((num_experts, 0), dtype=torch.int32,
483 + device=device),
484 + requires_grad=False,
485 + )
486 + layer.w13_g_idx_sort_indices = torch.nn.Parameter(
487 + torch.empty((num_experts, 0), dtype=torch.int32,
488 + device=device),
489 + requires_grad=False,
490 + )
491 + layer.w2_g_idx_sort_indices = torch.nn.Parameter(
492 + torch.empty((num_experts, 0), dtype=torch.int32,
493 + device=device),
494 + requires_grad=False,
495 + )
496
497 marlin_w13_qweight = ops.gptq_marlin_moe_repack(
498 layer.w13_weight_packed,
@@ -511,9 +564,9 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
564 router_logits,
565 topk_weights,
566 topk_ids,
514 - g_idx1=layer.w13_g_idx,
515 - g_idx2=layer.w2_g_idx,
567 + g_idx1=layer.w13_weight_g_idx,
568 + g_idx2=layer.w2_weight_g_idx,
569 sort_indices1=layer.w13_g_idx_sort_indices,
570 sort_indices2=layer.w2_g_idx_sort_indices,
571 num_bits=self.num_bits,
519 - )
572 + is_k_full=self.is_k_full)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+1 -1
@@ -62,7 +62,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
62 **kwargs):
63
64 assert params_dtype == torch.float16, (
65 - "float16 is required for marlin24 compressd models. Set dtype=torch.float16" # noqa: E501
65 + "float16 is required for marlin24 compressed models. Set dtype=torch.float16" # noqa: E501
66 )
67
68 pack_factor = 32 // self.quant_type.size_bits
vllm/model_executor/layers/quantization/experts_int8.py
+15 -12
@@ -52,7 +52,7 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
52 self.quant_config = quant_config
53
54 def create_weights(self, layer: torch.nn.Module, num_experts: int,
55 - hidden_size: int, intermediate_size: int,
55 + hidden_size: int, intermediate_size_per_partition: int,
56 params_dtype: torch.dtype, **extra_weight_attrs):
57
58 int8_dtype = torch.int8
@@ -64,26 +64,29 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
64 extra_weight_attrs['weight_loader'] = wrapped_weight_loader
65
66 # Fused gate_up_proj (column parallel)
67 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
68 - 2 * intermediate_size,
69 - hidden_size,
70 - dtype=int8_dtype),
67 + w13_weight = torch.nn.Parameter(torch.empty(
68 + num_experts,
69 + 2 * intermediate_size_per_partition,
70 + hidden_size,
71 + dtype=int8_dtype),
72 requires_grad=False)
73 layer.register_parameter("w13_weight", w13_weight)
74 set_weight_attrs(w13_weight, extra_weight_attrs)
75
76 # down_proj (row parallel)
76 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
77 - hidden_size,
78 - intermediate_size,
79 - dtype=int8_dtype),
77 + w2_weight = torch.nn.Parameter(torch.empty(
78 + num_experts,
79 + hidden_size,
80 + intermediate_size_per_partition,
81 + dtype=int8_dtype),
82 requires_grad=False)
83 layer.register_parameter("w2_weight", w2_weight)
84 set_weight_attrs(w2_weight, extra_weight_attrs)
85
84 - w13_scale = torch.nn.Parameter(torch.zeros(num_experts,
85 - 2 * intermediate_size,
86 - dtype=torch.float32),
86 + w13_scale = torch.nn.Parameter(torch.zeros(
87 + num_experts,
88 + 2 * intermediate_size_per_partition,
89 + dtype=torch.float32),
90 requires_grad=False)
91 layer.register_parameter("w13_scale", w13_scale)
92
vllm/model_executor/layers/quantization/fp8.py
+23 -18
@@ -386,8 +386,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
386 self.block_quant = self.quant_config.weight_block_size is not None
387
388 def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
389 - intermediate_size: int, params_dtype: torch.dtype,
390 - **extra_weight_attrs):
389 + intermediate_size_per_partition: int,
390 + params_dtype: torch.dtype, **extra_weight_attrs):
391
392 if self.quant_config.is_checkpoint_fp8_serialized:
393 params_dtype = torch.float8_e4m3fn
@@ -402,30 +402,34 @@ class Fp8MoEMethod(FusedMoEMethodBase):
402 # scales, the output_size of the weights for both the gate and up
403 # layers must be divisible by block_n.
404 # Required by column parallel or enabling merged weights
405 - if intermediate_size % block_n != 0:
405 + if intermediate_size_per_partition % block_n != 0:
406 raise ValueError(
407 f"The output_size of gate's and up's weight = "
408 - f"{intermediate_size} is not divisible by "
408 + f"{intermediate_size_per_partition} is not divisible by "
409 f"weight quantization block_n = {block_n}.")
410 - if (tp_size > 1 and intermediate_size % block_k != 0):
410 + if (tp_size > 1
411 + and intermediate_size_per_partition % block_k != 0):
412 # Required by row parallel
412 - raise ValueError(f"The input_size of down's weight = "
413 - f"{intermediate_size} is not divisible by "
414 - f"weight quantization block_k = {block_k}.")
413 + raise ValueError(
414 + f"The input_size of down's weight = "
415 + f"{intermediate_size_per_partition} is not divisible by "
416 + f"weight quantization block_k = {block_k}.")
417
418 # WEIGHTS
417 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
418 - 2 * intermediate_size,
419 - hidden_size,
420 - dtype=params_dtype),
419 + w13_weight = torch.nn.Parameter(torch.empty(
420 + num_experts,
421 + 2 * intermediate_size_per_partition,
422 + hidden_size,
423 + dtype=params_dtype),
424 requires_grad=False)
425 layer.register_parameter("w13_weight", w13_weight)
426 set_weight_attrs(w13_weight, extra_weight_attrs)
427
425 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
426 - hidden_size,
427 - intermediate_size,
428 - dtype=params_dtype),
428 + w2_weight = torch.nn.Parameter(torch.empty(
429 + num_experts,
430 + hidden_size,
431 + intermediate_size_per_partition,
432 + dtype=params_dtype),
433 requires_grad=False)
434 layer.register_parameter("w2_weight", w2_weight)
435 set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -446,7 +450,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
450 w13_weight_scale = torch.nn.Parameter(
451 torch.ones(
452 num_experts,
449 - 2 * ((intermediate_size + block_n - 1) // block_n),
453 + 2 * ((intermediate_size_per_partition + block_n - 1) //
454 + block_n),
455 (hidden_size + block_k - 1) // block_k,
456 dtype=torch.float32,
457 ),
@@ -456,7 +461,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
461 torch.ones(
462 num_experts,
463 (hidden_size + block_n - 1) // block_n,
459 - (intermediate_size + block_k - 1) // block_k,
464 + (intermediate_size_per_partition + block_k - 1) // block_k,
465 dtype=torch.float32,
466 ),
467 requires_grad=False,
vllm/model_executor/layers/quantization/gptq_marlin.py
+11 -8
@@ -317,7 +317,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
317 layer: torch.nn.Module,
318 num_experts: int,
319 hidden_size: int,
320 - intermediate_size: int,
320 + intermediate_size_per_partition: int,
321 params_dtype: torch.dtype,
322 **extra_weight_attrs,
323 ):
@@ -326,7 +326,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
326 # Supports only sym for now (no zp)
327 if self.quant_config.group_size != -1:
328 scales_size13 = hidden_size // self.quant_config.group_size
329 - scales_size2 = intermediate_size // self.quant_config.group_size
329 + scales_size2 = (intermediate_size_per_partition //
330 + self.quant_config.group_size)
331 strategy = FusedMoeWeightScaleSupported.GROUP.value
332 else:
333 scales_size13 = 1
@@ -342,7 +343,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
343 torch.empty(
344 num_experts,
345 hidden_size // self.quant_config.pack_factor,
345 - 2 * intermediate_size,
346 + 2 * intermediate_size_per_partition,
347 dtype=torch.int32,
348 ),
349 requires_grad=False,
@@ -353,7 +354,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
354 w2_qweight = torch.nn.Parameter(
355 torch.empty(
356 num_experts,
356 - intermediate_size // self.quant_config.pack_factor,
357 + intermediate_size_per_partition //
358 + self.quant_config.pack_factor,
359 hidden_size,
360 dtype=torch.int32,
361 ),
@@ -365,7 +367,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
367 w13_scales = torch.nn.Parameter(
368 torch.empty(num_experts,
369 scales_size13,
368 - 2 * intermediate_size,
370 + 2 * intermediate_size_per_partition,
371 dtype=torch.half),
372 requires_grad=False,
373 )
@@ -385,7 +387,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
387 w13_qzeros = torch.nn.Parameter(
388 torch.empty(num_experts,
389 scales_size13,
388 - 2 * intermediate_size // self.quant_config.pack_factor,
390 + 2 * intermediate_size_per_partition //
391 + self.quant_config.pack_factor,
392 dtype=params_dtype),
393 requires_grad=False,
394 )
@@ -414,7 +417,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
417 w2_g_idx = torch.nn.Parameter(
418 torch.empty(
419 num_experts,
417 - intermediate_size,
420 + intermediate_size_per_partition,
421 dtype=torch.int32,
422 ),
423 requires_grad=False,
@@ -435,7 +438,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
438 w2_g_idx_sort_indices = torch.nn.Parameter(
439 torch.empty(
440 num_experts,
438 - intermediate_size,
441 + intermediate_size_per_partition,
442 dtype=torch.int32,
443 ),
444 requires_grad=False,
vllm/model_executor/layers/quantization/quark/quark_moe.py
+11 -9
@@ -60,24 +60,26 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
60 self.static_input_scales = not self.input_quant.get("is_dynamic")
61
62 def create_weights(self, layer: torch.nn.Module, num_experts: int,
63 - hidden_size: int, intermediate_size: int,
63 + hidden_size: int, intermediate_size_per_partition: int,
64 params_dtype: torch.dtype, **extra_weight_attrs):
65
66 params_dtype = torch.float8_e4m3fn
67
68 # WEIGHTS
69 - w13_weight = torch.nn.Parameter(torch.empty(num_experts,
70 - 2 * intermediate_size,
71 - hidden_size,
72 - dtype=params_dtype),
69 + w13_weight = torch.nn.Parameter(torch.empty(
70 + num_experts,
71 + 2 * intermediate_size_per_partition,
72 + hidden_size,
73 + dtype=params_dtype),
74 requires_grad=False)
75 layer.register_parameter("w13_weight", w13_weight)
76 set_weight_attrs(w13_weight, extra_weight_attrs)
77
77 - w2_weight = torch.nn.Parameter(torch.empty(num_experts,
78 - hidden_size,
79 - intermediate_size,
80 - dtype=params_dtype),
78 + w2_weight = torch.nn.Parameter(torch.empty(
79 + num_experts,
80 + hidden_size,
81 + intermediate_size_per_partition,
82 + dtype=params_dtype),
83 requires_grad=False)
84 layer.register_parameter("w2_weight", w2_weight)
85 set_weight_attrs(w2_weight, extra_weight_attrs)