A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[BugFix] Fix parameter names and `process_after_weight_loading` for W4A16 MoE Group Act Order (#11528)
Signed-off-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: ElizaWszola <eliza@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Dipika Sikka committed
Jan 23, 2025 at 16:40 UTC
eb5cb5e5280c83226408c3d1e30928364c6258db
8 files changed
+243
-148
vllm/model_executor/layers/fused_moe/layer.py
+60
-34
@@ -38,7 +38,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
38
39
@abstractmethod
40
def create_weights(self, layer: torch.nn.Module, num_experts: int,
41
- hidden_size: int, intermediate_size: int,
41
+ hidden_size: int, intermediate_size_per_partition: int,
42
params_dtype: torch.dtype, **extra_weight_attrs):
43
raise NotImplementedError
44
@@ -65,22 +65,24 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
65
"""MoE method without quantization."""
66
67
def create_weights(self, layer: torch.nn.Module, num_experts: int,
68
- hidden_size: int, intermediate_size: int,
68
+ hidden_size: int, intermediate_size_per_partition: int,
69
params_dtype: torch.dtype, **extra_weight_attrs):
70
# Fused gate_up_proj (column parallel)
71
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
72
- 2 * intermediate_size,
73
- hidden_size,
74
- dtype=params_dtype),
71
+ w13_weight = torch.nn.Parameter(torch.empty(
72
+ num_experts,
73
+ 2 * intermediate_size_per_partition,
74
+ hidden_size,
75
+ dtype=params_dtype),
76
requires_grad=False)
77
layer.register_parameter("w13_weight", w13_weight)
78
set_weight_attrs(w13_weight, extra_weight_attrs)
79
80
# down_proj (row parallel)
80
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
81
- hidden_size,
82
- intermediate_size,
83
- dtype=params_dtype),
81
+ w2_weight = torch.nn.Parameter(torch.empty(
82
+ num_experts,
83
+ hidden_size,
84
+ intermediate_size_per_partition,
85
+ dtype=params_dtype),
86
requires_grad=False)
87
layer.register_parameter("w2_weight", w2_weight)
88
set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -289,13 +291,20 @@ class FusedMoE(torch.nn.Module):
291
self.quant_method = quant_config.get_quant_method(self, prefix)
292
assert self.quant_method is not None
293
292
- self.quant_method.create_weights(
293
- layer=self,
294
- num_experts=num_experts,
295
- hidden_size=hidden_size,
296
- intermediate_size=self.intermediate_size_per_partition,
297
- params_dtype=params_dtype,
298
- weight_loader=self.weight_loader)
294
+ moe_quant_params = {
295
+ "num_experts": num_experts,
296
+ "hidden_size": hidden_size,
297
+ "intermediate_size_per_partition":
298
+ self.intermediate_size_per_partition,
299
+ "params_dtype": params_dtype,
300
+ "weight_loader": self.weight_loader,
301
+ }
302
+ # need full intermediate size pre-sharding for WNA16 act order
303
+ if (self.quant_method.__class__.__name__ ==
304
+ "CompressedTensorsWNA16MoEMethod"):
305
+ moe_quant_params["intermediate_size_full"] = intermediate_size
306
+
307
+ self.quant_method.create_weights(layer=self, **moe_quant_params)
308
309
def _load_per_tensor_weight_scale(self, shard_id: str,
310
param: torch.nn.Parameter,
@@ -312,19 +321,30 @@ class FusedMoE(torch.nn.Module):
321
elif shard_id == "w2":
322
param_data[expert_id] = loaded_weight
323
315
- def _load_model_weight_or_group_weight_scale(self, shard_dim: int,
324
+ def _load_model_weight_or_group_weight_scale(self,
325
+ shard_dim: int,
326
expert_data: torch.Tensor,
327
shard_id: str,
328
loaded_weight: torch.Tensor,
319
- tp_rank: int):
320
- # Load grouped weight scales for group quantization
321
- # or model weights
329
+ tp_rank: int,
330
+ load_full_w2: bool = False):
331
+ """
332
+ Load grouped weight scales for group quantization or model weights
333
+ :param shard_dim: dimension to shard
334
+ :param expert_data: parameter for a particular expert
335
+ :param shard_id: either w1, w2, or w3
336
+ :param loaded_weight: checkpoint weight to load into the param
337
+ :param tp_rank: tensor parallel rank
338
+ :param load_full_w2: whether or not the w2 loaded should be sharded.
339
+ """
340
if shard_id == "w2":
323
- self._load_w2(shard_id=shard_id,
324
- shard_dim=shard_dim,
341
+ # In the case where we have actorder/g_idx, we do not partition the
342
+ # w2 scales, as indicated by `load_full` argument, for all tp cases
343
+ self._load_w2(shard_dim=shard_dim,
344
loaded_weight=loaded_weight,
345
expert_data=expert_data,
327
- tp_rank=tp_rank)
346
+ tp_rank=tp_rank,
347
+ load_full=load_full_w2)
348
elif shard_id in ("w1", "w3"):
349
self._load_w13(shard_id=shard_id,
350
shard_dim=shard_dim,
@@ -364,15 +384,21 @@ class FusedMoE(torch.nn.Module):
384
expert_data = expert_data.narrow(shard_dim, shard_size, shard_size)
385
expert_data.copy_(loaded_weight)
386
367
- def _load_w2(self, expert_data: torch.Tensor, shard_dim: int,
368
- shard_id: str, loaded_weight: torch.Tensor, tp_rank: int):
387
+ def _load_w2(self,
388
+ expert_data: torch.Tensor,
389
+ shard_dim: int,
390
+ loaded_weight: torch.Tensor,
391
+ tp_rank: int,
392
+ load_full: bool = False):
393
394
# Index the loaded weight for tp sharding.
395
# down_proj: "RowParallel" so tp sharding on input_dim
396
# Narrow parameter and load.
397
shard_size = expert_data.shape[shard_dim]
374
- loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank,
375
- shard_size)
398
+ if not load_full:
399
+ loaded_weight = loaded_weight.narrow(shard_dim,
400
+ shard_size * tp_rank,
401
+ shard_size)
402
# w2, down_proj: Load into only logical weight of w2.
403
expert_data.copy_(loaded_weight)
404
@@ -387,8 +413,7 @@ class FusedMoE(torch.nn.Module):
413
shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int):
414
415
if shard_id == "w2":
390
- self._load_w2(shard_id=shard_id,
391
- shard_dim=shard_dim,
416
+ self._load_w2(shard_dim=shard_dim,
417
loaded_weight=loaded_weight,
418
expert_data=expert_data,
419
tp_rank=tp_rank)
@@ -416,7 +441,7 @@ class FusedMoE(torch.nn.Module):
441
]
442
# Fetch the dim to shard the parameter/loaded weight
443
# based on the shard id. This will be whatever
419
- # dimension intermediate_size is used.
444
+ # dimension intermediate_size_per_partition is used.
445
SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0}
446
447
expert_data = param.data[expert_id]
@@ -424,11 +449,11 @@ class FusedMoE(torch.nn.Module):
449
450
# is_transposed: if the dim to shard the weight
451
# should be flipped. Required by GPTQ, compressed-tensors
427
- # should be whatever dimension intermediate_size is
452
+ # should be whatever dimension intermediate_size_per_partition is
453
is_transposed = getattr(param, "is_transposed", False)
454
shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id]
455
if is_transposed:
431
- shard_dim = ~shard_dim
456
+ shard_dim = int(not shard_dim)
457
458
# Case input scale: input_scale loading is only supported for fp8
459
if "input_scale" in weight_name:
@@ -480,7 +505,8 @@ class FusedMoE(torch.nn.Module):
505
shard_dim=shard_dim,
506
loaded_weight=loaded_weight,
507
expert_data=expert_data,
483
- tp_rank=tp_rank)
508
+ tp_rank=tp_rank,
509
+ load_full_w2=getattr(param, "load_full_w2", False))
510
elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value:
511
self._load_per_tensor_weight_scale(shard_id=shard_id,
512
param=param,
vllm/model_executor/layers/quantization/awq_marlin.py
+19
-16
@@ -303,7 +303,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
303
self.quant_config = quant_config
304
305
def create_weights(self, layer: torch.nn.Module, num_experts: int,
306
- hidden_size: int, intermediate_size: int,
306
+ hidden_size: int, intermediate_size_per_partition: int,
307
params_dtype: torch.dtype, **extra_weight_attrs):
308
extra_weight_attrs.update({
309
"is_transposed":
@@ -312,17 +312,18 @@ class AWQMoEMethod(FusedMoEMethodBase):
312
FusedMoeWeightScaleSupported.GROUP.value,
313
})
314
315
- w13_qweight = Parameter(torch.empty(num_experts,
316
- hidden_size,
317
- 2 * intermediate_size //
318
- self.quant_config.pack_factor,
319
- dtype=torch.int32),
320
- requires_grad=False)
315
+ w13_qweight = Parameter(
316
+ torch.empty(num_experts,
317
+ hidden_size,
318
+ 2 * intermediate_size_per_partition //
319
+ self.quant_config.pack_factor,
320
+ dtype=torch.int32),
321
+ requires_grad=False)
322
layer.register_parameter("w13_qweight", w13_qweight)
323
set_weight_attrs(w13_qweight, extra_weight_attrs)
324
325
w2_qweight = Parameter(torch.empty(num_experts,
325
- intermediate_size,
326
+ intermediate_size_per_partition,
327
hidden_size //
328
self.quant_config.pack_factor,
329
dtype=torch.int32),
@@ -331,13 +332,14 @@ class AWQMoEMethod(FusedMoEMethodBase):
332
set_weight_attrs(w2_qweight, extra_weight_attrs)
333
334
num_groups_w13 = hidden_size // self.quant_config.group_size
334
- num_groups_w2 = intermediate_size // self.quant_config.group_size
335
+ num_groups_w2 = (intermediate_size_per_partition //
336
+ self.quant_config.group_size)
337
338
# WEIGHT_SCALES
339
# Allocate 2 scales for w1 and w3 respectively.
340
w13_scales = Parameter(torch.empty(num_experts,
341
num_groups_w13,
340
- intermediate_size * 2,
342
+ intermediate_size_per_partition * 2,
343
dtype=params_dtype),
344
requires_grad=False)
345
layer.register_parameter("w13_scales", w13_scales)
@@ -353,12 +355,13 @@ class AWQMoEMethod(FusedMoEMethodBase):
355
356
# WEIGHT_ZERO_POINT
357
# Allocate 2 zero points for w1 and w3 respectively.
356
- w13_qzeros = Parameter(torch.empty(num_experts,
357
- num_groups_w13,
358
- 2 * intermediate_size //
359
- self.quant_config.pack_factor,
360
- dtype=torch.int32),
361
- requires_grad=False)
358
+ w13_qzeros = Parameter(
359
+ torch.empty(num_experts,
360
+ num_groups_w13,
361
+ 2 * intermediate_size_per_partition //
362
+ self.quant_config.pack_factor,
363
+ dtype=torch.int32),
364
+ requires_grad=False)
365
layer.register_parameter("w13_qzeros", w13_qzeros)
366
set_weight_attrs(w13_qzeros, extra_weight_attrs)
367
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+103
-50
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe import (FusedMoE, FusedMoEMethodBase,
13
FusedMoeWeightScaleSupported)
14
from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
15
WNA16_SUPPORTED_BITS)
16
+from vllm.model_executor.layers.quantization.utils import replace_parameter
17
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
18
all_close_1d, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize)
19
from vllm.model_executor.utils import set_weight_attrs
@@ -75,24 +76,26 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
76
self.static_input_scales = not self.input_quant.dynamic
77
78
def create_weights(self, layer: torch.nn.Module, num_experts: int,
78
- hidden_size: int, intermediate_size: int,
79
+ hidden_size: int, intermediate_size_per_partition: int,
80
params_dtype: torch.dtype, **extra_weight_attrs):
81
82
params_dtype = torch.float8_e4m3fn
83
84
# WEIGHTS
84
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
85
- 2 * intermediate_size,
86
- hidden_size,
87
- dtype=params_dtype),
85
+ w13_weight = torch.nn.Parameter(torch.empty(
86
+ num_experts,
87
+ 2 * intermediate_size_per_partition,
88
+ hidden_size,
89
+ dtype=params_dtype),
90
requires_grad=False)
91
layer.register_parameter("w13_weight", w13_weight)
92
set_weight_attrs(w13_weight, extra_weight_attrs)
93
92
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
93
- hidden_size,
94
- intermediate_size,
95
- dtype=params_dtype),
94
+ w2_weight = torch.nn.Parameter(torch.empty(
95
+ num_experts,
96
+ hidden_size,
97
+ intermediate_size_per_partition,
98
+ dtype=params_dtype),
99
requires_grad=False)
100
layer.register_parameter("w2_weight", w2_weight)
101
set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -254,6 +257,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
257
self.packed_factor = 32 // config.num_bits
258
self.strategy = config.strategy
259
self.group_size = config.group_size
260
+ self.actorder = config.actorder
261
assert config.symmetric, (
262
"Only symmetric quantization is supported for MoE")
263
@@ -266,9 +270,16 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
270
f"{WNA16_SUPPORTED_BITS}")
271
272
def create_weights(self, layer: torch.nn.Module, num_experts: int,
269
- hidden_size: int, intermediate_size: int,
273
+ hidden_size: int, intermediate_size_per_partition: int,
274
params_dtype: torch.dtype, **extra_weight_attrs):
275
276
+ assert params_dtype == torch.float16, (
277
+ "float16 is required for MoE compressed models. Set dtype=torch.float16" # noqa: E501
278
+ )
279
+
280
+ intermediate_size_full = extra_weight_attrs.pop(
281
+ "intermediate_size_full")
282
+
283
# Will transpose the loaded weight along the
284
# intermediate and hidden dim sizes. Will
285
# shard for TP along the transposed dims
@@ -276,35 +287,45 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
287
"is_transposed": True,
288
"quant_method": self.strategy
289
})
279
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
280
- hidden_size //
281
- self.packed_factor,
282
- 2 * intermediate_size,
283
- dtype=torch.int32),
290
+ w13_weight = torch.nn.Parameter(torch.empty(
291
+ num_experts,
292
+ hidden_size // self.packed_factor,
293
+ 2 * intermediate_size_per_partition,
294
+ dtype=torch.int32),
295
requires_grad=False)
296
layer.register_parameter("w13_weight_packed", w13_weight)
297
set_weight_attrs(w13_weight, extra_weight_attrs)
298
288
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
289
- intermediate_size //
290
- self.packed_factor,
291
- hidden_size,
292
- dtype=torch.int32),
299
+ w2_weight = torch.nn.Parameter(torch.empty(
300
+ num_experts,
301
+ intermediate_size_per_partition // self.packed_factor,
302
+ hidden_size,
303
+ dtype=torch.int32),
304
requires_grad=False)
305
layer.register_parameter("w2_weight_packed", w2_weight)
306
set_weight_attrs(w2_weight, extra_weight_attrs)
307
308
+ # In the case where we have actorder/g_idx,
309
+ # we do not partition the w2 scales
310
+ load_full_w2 = self.actorder and self.group_size != -1
311
+ w2_scales_size = (intermediate_size_full
312
+ if load_full_w2 else intermediate_size_per_partition)
313
+
314
+ self.is_k_full = (not self.actorder) or (
315
+ intermediate_size_per_partition == intermediate_size_full)
316
+
317
if self.strategy == "channel":
318
num_groups_w2 = num_groups_w13 = 1
319
self.group_size = -1
320
else:
301
- num_groups_w2 = intermediate_size // self.group_size
321
+ num_groups_w2 = w2_scales_size // self.group_size
322
num_groups_w13 = hidden_size // self.group_size
323
304
- w13_scale = torch.nn.Parameter(torch.ones(num_experts,
305
- num_groups_w13,
306
- 2 * intermediate_size,
307
- dtype=params_dtype),
324
+ w13_scale = torch.nn.Parameter(torch.ones(
325
+ num_experts,
326
+ num_groups_w13,
327
+ 2 * intermediate_size_per_partition,
328
+ dtype=params_dtype),
329
requires_grad=False)
330
layer.register_parameter("w13_weight_scale", w13_scale)
331
set_weight_attrs(w13_scale, extra_weight_attrs)
@@ -316,6 +337,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
337
requires_grad=False)
338
layer.register_parameter("w2_weight_scale", w2_scale)
339
set_weight_attrs(w2_scale, extra_weight_attrs)
340
+ set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2})
341
342
w2_weight_shape = torch.nn.Parameter(torch.empty(num_experts, 2),
343
requires_grad=False)
@@ -335,18 +357,18 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
357
),
358
requires_grad=False,
359
)
338
- layer.register_parameter("w13_g_idx", w13_g_idx)
360
+ layer.register_parameter("w13_weight_g_idx", w13_g_idx)
361
set_weight_attrs(w13_g_idx, extra_weight_attrs)
362
363
w2_g_idx = torch.nn.Parameter(
364
torch.empty(
365
num_experts,
344
- intermediate_size,
366
+ intermediate_size_per_partition,
367
dtype=torch.int32,
368
),
369
requires_grad=False,
370
)
349
- layer.register_parameter("w2_g_idx", w2_g_idx)
371
+ layer.register_parameter("w2_weight_g_idx", w2_g_idx)
372
set_weight_attrs(w2_g_idx, extra_weight_attrs)
373
374
w13_g_idx_sort_indices = torch.nn.Parameter(
@@ -364,7 +386,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
386
w2_g_idx_sort_indices = torch.nn.Parameter(
387
torch.empty(
388
num_experts,
367
- intermediate_size,
389
+ intermediate_size_per_partition,
390
dtype=torch.int32,
391
),
392
requires_grad=False,
@@ -422,24 +444,55 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
444
size_k2 = layer.w2_weight_packed.shape[2]
445
size_k13 = layer.w13_weight_packed.shape[2]
446
425
- num_experts = layer.w13_g_idx.shape[0]
426
- device = layer.w13_g_idx.device
427
- layer.w13_g_idx = torch.nn.Parameter(
428
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
429
- requires_grad=False,
430
- )
431
- layer.w2_g_idx = torch.nn.Parameter(
432
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
433
- requires_grad=False,
434
- )
435
- layer.w13_g_idx_sort_indices = torch.nn.Parameter(
436
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
437
- requires_grad=False,
438
- )
439
- layer.w2_g_idx_sort_indices = torch.nn.Parameter(
440
- torch.empty((num_experts, 0), dtype=torch.int32, device=device),
441
- requires_grad=False,
442
- )
447
+ num_experts = layer.w13_weight_g_idx.shape[0]
448
+ device = layer.w13_weight_g_idx.device
449
+
450
+ # when running models with grouped act order,
451
+ # resort to g_idx values provided in checkpoint
452
+ if self.actorder == "group":
453
+ w13_g_idx_sort_indices = torch.empty_like(layer.w13_weight_g_idx)
454
+ w2_g_idx_sort_indices = torch.empty_like(layer.w2_weight_g_idx)
455
+ w13_sorted_g_idx = torch.empty_like(layer.w13_weight_g_idx)
456
+ w2_sorted_g_idx = torch.empty_like(layer.w2_weight_g_idx)
457
+
458
+ for e in range(num_experts):
459
+ w13_g_idx_sort_indices[e] = torch.argsort(
460
+ layer.w13_weight_g_idx[e]).to(torch.int32)
461
+ w2_g_idx_sort_indices[e] = torch.argsort(
462
+ layer.w2_weight_g_idx[e]).to(torch.int32)
463
+ w13_sorted_g_idx[e] = layer.w13_weight_g_idx[e][
464
+ w13_g_idx_sort_indices[e]]
465
+ w2_sorted_g_idx[e] = layer.w2_weight_g_idx[e][
466
+ w2_g_idx_sort_indices[e]]
467
+
468
+ replace_parameter(layer, "w13_weight_g_idx", w13_sorted_g_idx)
469
+ replace_parameter(layer, "w2_weight_g_idx", w2_sorted_g_idx)
470
+ replace_parameter(layer, "w13_g_idx_sort_indices",
471
+ w13_g_idx_sort_indices)
472
+ replace_parameter(layer, "w2_g_idx_sort_indices",
473
+ w2_g_idx_sort_indices)
474
+
475
+ else:
476
+ layer.w13_weight_g_idx = torch.nn.Parameter(
477
+ torch.empty((num_experts, 0), dtype=torch.int32,
478
+ device=device),
479
+ requires_grad=False,
480
+ )
481
+ layer.w2_weight_g_idx = torch.nn.Parameter(
482
+ torch.empty((num_experts, 0), dtype=torch.int32,
483
+ device=device),
484
+ requires_grad=False,
485
+ )
486
+ layer.w13_g_idx_sort_indices = torch.nn.Parameter(
487
+ torch.empty((num_experts, 0), dtype=torch.int32,
488
+ device=device),
489
+ requires_grad=False,
490
+ )
491
+ layer.w2_g_idx_sort_indices = torch.nn.Parameter(
492
+ torch.empty((num_experts, 0), dtype=torch.int32,
493
+ device=device),
494
+ requires_grad=False,
495
+ )
496
497
marlin_w13_qweight = ops.gptq_marlin_moe_repack(
498
layer.w13_weight_packed,
@@ -511,9 +564,9 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
564
router_logits,
565
topk_weights,
566
topk_ids,
514
- g_idx1=layer.w13_g_idx,
515
- g_idx2=layer.w2_g_idx,
567
+ g_idx1=layer.w13_weight_g_idx,
568
+ g_idx2=layer.w2_weight_g_idx,
569
sort_indices1=layer.w13_g_idx_sort_indices,
570
sort_indices2=layer.w2_g_idx_sort_indices,
571
num_bits=self.num_bits,
519
- )
572
+ is_k_full=self.is_k_full)
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+1
-1
@@ -62,7 +62,7 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
62
**kwargs):
63
64
assert params_dtype == torch.float16, (
65
- "float16 is required for marlin24 compressd models. Set dtype=torch.float16" # noqa: E501
65
+ "float16 is required for marlin24 compressed models. Set dtype=torch.float16" # noqa: E501
66
)
67
68
pack_factor = 32 // self.quant_type.size_bits
vllm/model_executor/layers/quantization/experts_int8.py
+15
-12
@@ -52,7 +52,7 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
52
self.quant_config = quant_config
53
54
def create_weights(self, layer: torch.nn.Module, num_experts: int,
55
- hidden_size: int, intermediate_size: int,
55
+ hidden_size: int, intermediate_size_per_partition: int,
56
params_dtype: torch.dtype, **extra_weight_attrs):
57
58
int8_dtype = torch.int8
@@ -64,26 +64,29 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
64
extra_weight_attrs['weight_loader'] = wrapped_weight_loader
65
66
# Fused gate_up_proj (column parallel)
67
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
68
- 2 * intermediate_size,
69
- hidden_size,
70
- dtype=int8_dtype),
67
+ w13_weight = torch.nn.Parameter(torch.empty(
68
+ num_experts,
69
+ 2 * intermediate_size_per_partition,
70
+ hidden_size,
71
+ dtype=int8_dtype),
72
requires_grad=False)
73
layer.register_parameter("w13_weight", w13_weight)
74
set_weight_attrs(w13_weight, extra_weight_attrs)
75
76
# down_proj (row parallel)
76
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
77
- hidden_size,
78
- intermediate_size,
79
- dtype=int8_dtype),
77
+ w2_weight = torch.nn.Parameter(torch.empty(
78
+ num_experts,
79
+ hidden_size,
80
+ intermediate_size_per_partition,
81
+ dtype=int8_dtype),
82
requires_grad=False)
83
layer.register_parameter("w2_weight", w2_weight)
84
set_weight_attrs(w2_weight, extra_weight_attrs)
85
84
- w13_scale = torch.nn.Parameter(torch.zeros(num_experts,
85
- 2 * intermediate_size,
86
- dtype=torch.float32),
86
+ w13_scale = torch.nn.Parameter(torch.zeros(
87
+ num_experts,
88
+ 2 * intermediate_size_per_partition,
89
+ dtype=torch.float32),
90
requires_grad=False)
91
layer.register_parameter("w13_scale", w13_scale)
92
vllm/model_executor/layers/quantization/fp8.py
+23
-18
@@ -386,8 +386,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
386
self.block_quant = self.quant_config.weight_block_size is not None
387
388
def create_weights(self, layer: Module, num_experts: int, hidden_size: int,
389
- intermediate_size: int, params_dtype: torch.dtype,
390
- **extra_weight_attrs):
389
+ intermediate_size_per_partition: int,
390
+ params_dtype: torch.dtype, **extra_weight_attrs):
391
392
if self.quant_config.is_checkpoint_fp8_serialized:
393
params_dtype = torch.float8_e4m3fn
@@ -402,30 +402,34 @@ class Fp8MoEMethod(FusedMoEMethodBase):
402
# scales, the output_size of the weights for both the gate and up
403
# layers must be divisible by block_n.
404
# Required by column parallel or enabling merged weights
405
- if intermediate_size % block_n != 0:
405
+ if intermediate_size_per_partition % block_n != 0:
406
raise ValueError(
407
f"The output_size of gate's and up's weight = "
408
- f"{intermediate_size} is not divisible by "
408
+ f"{intermediate_size_per_partition} is not divisible by "
409
f"weight quantization block_n = {block_n}.")
410
- if (tp_size > 1 and intermediate_size % block_k != 0):
410
+ if (tp_size > 1
411
+ and intermediate_size_per_partition % block_k != 0):
412
# Required by row parallel
412
- raise ValueError(f"The input_size of down's weight = "
413
- f"{intermediate_size} is not divisible by "
414
- f"weight quantization block_k = {block_k}.")
413
+ raise ValueError(
414
+ f"The input_size of down's weight = "
415
+ f"{intermediate_size_per_partition} is not divisible by "
416
+ f"weight quantization block_k = {block_k}.")
417
418
# WEIGHTS
417
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
418
- 2 * intermediate_size,
419
- hidden_size,
420
- dtype=params_dtype),
419
+ w13_weight = torch.nn.Parameter(torch.empty(
420
+ num_experts,
421
+ 2 * intermediate_size_per_partition,
422
+ hidden_size,
423
+ dtype=params_dtype),
424
requires_grad=False)
425
layer.register_parameter("w13_weight", w13_weight)
426
set_weight_attrs(w13_weight, extra_weight_attrs)
427
425
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
426
- hidden_size,
427
- intermediate_size,
428
- dtype=params_dtype),
428
+ w2_weight = torch.nn.Parameter(torch.empty(
429
+ num_experts,
430
+ hidden_size,
431
+ intermediate_size_per_partition,
432
+ dtype=params_dtype),
433
requires_grad=False)
434
layer.register_parameter("w2_weight", w2_weight)
435
set_weight_attrs(w2_weight, extra_weight_attrs)
@@ -446,7 +450,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
450
w13_weight_scale = torch.nn.Parameter(
451
torch.ones(
452
num_experts,
449
- 2 * ((intermediate_size + block_n - 1) // block_n),
453
+ 2 * ((intermediate_size_per_partition + block_n - 1) //
454
+ block_n),
455
(hidden_size + block_k - 1) // block_k,
456
dtype=torch.float32,
457
),
@@ -456,7 +461,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
461
torch.ones(
462
num_experts,
463
(hidden_size + block_n - 1) // block_n,
459
- (intermediate_size + block_k - 1) // block_k,
464
+ (intermediate_size_per_partition + block_k - 1) // block_k,
465
dtype=torch.float32,
466
),
467
requires_grad=False,
vllm/model_executor/layers/quantization/gptq_marlin.py
+11
-8
@@ -317,7 +317,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
317
layer: torch.nn.Module,
318
num_experts: int,
319
hidden_size: int,
320
- intermediate_size: int,
320
+ intermediate_size_per_partition: int,
321
params_dtype: torch.dtype,
322
**extra_weight_attrs,
323
):
@@ -326,7 +326,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
326
# Supports only sym for now (no zp)
327
if self.quant_config.group_size != -1:
328
scales_size13 = hidden_size // self.quant_config.group_size
329
- scales_size2 = intermediate_size // self.quant_config.group_size
329
+ scales_size2 = (intermediate_size_per_partition //
330
+ self.quant_config.group_size)
331
strategy = FusedMoeWeightScaleSupported.GROUP.value
332
else:
333
scales_size13 = 1
@@ -342,7 +343,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
343
torch.empty(
344
num_experts,
345
hidden_size // self.quant_config.pack_factor,
345
- 2 * intermediate_size,
346
+ 2 * intermediate_size_per_partition,
347
dtype=torch.int32,
348
),
349
requires_grad=False,
@@ -353,7 +354,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
354
w2_qweight = torch.nn.Parameter(
355
torch.empty(
356
num_experts,
356
- intermediate_size // self.quant_config.pack_factor,
357
+ intermediate_size_per_partition //
358
+ self.quant_config.pack_factor,
359
hidden_size,
360
dtype=torch.int32,
361
),
@@ -365,7 +367,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
367
w13_scales = torch.nn.Parameter(
368
torch.empty(num_experts,
369
scales_size13,
368
- 2 * intermediate_size,
370
+ 2 * intermediate_size_per_partition,
371
dtype=torch.half),
372
requires_grad=False,
373
)
@@ -385,7 +387,8 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
387
w13_qzeros = torch.nn.Parameter(
388
torch.empty(num_experts,
389
scales_size13,
388
- 2 * intermediate_size // self.quant_config.pack_factor,
390
+ 2 * intermediate_size_per_partition //
391
+ self.quant_config.pack_factor,
392
dtype=params_dtype),
393
requires_grad=False,
394
)
@@ -414,7 +417,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
417
w2_g_idx = torch.nn.Parameter(
418
torch.empty(
419
num_experts,
417
- intermediate_size,
420
+ intermediate_size_per_partition,
421
dtype=torch.int32,
422
),
423
requires_grad=False,
@@ -435,7 +438,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
438
w2_g_idx_sort_indices = torch.nn.Parameter(
439
torch.empty(
440
num_experts,
438
- intermediate_size,
441
+ intermediate_size_per_partition,
442
dtype=torch.int32,
443
),
444
requires_grad=False,
vllm/model_executor/layers/quantization/quark/quark_moe.py
+11
-9
@@ -60,24 +60,26 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
60
self.static_input_scales = not self.input_quant.get("is_dynamic")
61
62
def create_weights(self, layer: torch.nn.Module, num_experts: int,
63
- hidden_size: int, intermediate_size: int,
63
+ hidden_size: int, intermediate_size_per_partition: int,
64
params_dtype: torch.dtype, **extra_weight_attrs):
65
66
params_dtype = torch.float8_e4m3fn
67
68
# WEIGHTS
69
- w13_weight = torch.nn.Parameter(torch.empty(num_experts,
70
- 2 * intermediate_size,
71
- hidden_size,
72
- dtype=params_dtype),
69
+ w13_weight = torch.nn.Parameter(torch.empty(
70
+ num_experts,
71
+ 2 * intermediate_size_per_partition,
72
+ hidden_size,
73
+ dtype=params_dtype),
74
requires_grad=False)
75
layer.register_parameter("w13_weight", w13_weight)
76
set_weight_attrs(w13_weight, extra_weight_attrs)
77
77
- w2_weight = torch.nn.Parameter(torch.empty(num_experts,
78
- hidden_size,
79
- intermediate_size,
80
- dtype=params_dtype),
78
+ w2_weight = torch.nn.Parameter(torch.empty(
79
+ num_experts,
80
+ hidden_size,
81
+ intermediate_size_per_partition,
82
+ dtype=params_dtype),
83
requires_grad=False)
84
layer.register_parameter("w2_weight", w2_weight)
85
set_weight_attrs(w2_weight, extra_weight_attrs)