A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[BugFix] Fix quantization for all other methods (#11547)
Robert Shaw committed
Dec 27, 2024 at 01:23 UTC
2339d59f9260499599d60599f83978fad1827999
6 files changed
+52
-22
vllm/model_executor/layers/fused_moe/layer.py
+15
-4
@@ -41,9 +41,20 @@ class FusedMoEMethodBase(QuantizeMethodBase):
41
raise NotImplementedError
42
43
@abstractmethod
44
- def apply(self, layer: torch.nn.Module, x: torch.Tensor,
45
- router_logits: torch.Tensor, top_k: int, renormalize: bool,
46
- use_grouped_topk: bool) -> torch.Tensor:
44
+ def apply(
45
+ self,
46
+ layer: torch.nn.Module,
47
+ x: torch.Tensor,
48
+ router_logits: torch.Tensor,
49
+ top_k: int,
50
+ renormalize: bool,
51
+ use_grouped_topk: bool = False,
52
+ topk_group: Optional[int] = None,
53
+ num_expert_group: Optional[int] = None,
54
+ custom_routing_function: Optional[Callable] = None,
55
+ scoring_func: str = "softmax",
56
+ e_score_correction_bias: Optional[torch.Tensor] = None
57
+ ) -> torch.Tensor:
58
raise NotImplementedError
59
60
@@ -79,7 +90,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
90
router_logits: torch.Tensor,
91
top_k: int,
92
renormalize: bool,
82
- use_grouped_topk: bool,
93
+ use_grouped_topk: bool = False,
94
topk_group: Optional[int] = None,
95
num_expert_group: Optional[int] = None,
96
custom_routing_function: Optional[Callable] = None,
vllm/model_executor/layers/quantization/awq_marlin.py
+7
-3
@@ -440,11 +440,13 @@ class AWQMoEMethod(FusedMoEMethodBase):
440
x: torch.Tensor,
441
router_logits: torch.Tensor,
442
top_k: int,
443
- renormalize: bool = True,
443
+ renormalize: bool,
444
use_grouped_topk: bool = False,
445
- num_expert_group: Optional[int] = None,
445
topk_group: Optional[int] = None,
446
+ num_expert_group: Optional[int] = None,
447
custom_routing_function: Optional[Callable] = None,
448
+ scoring_func: str = "softmax",
449
+ e_score_correction_bias: Optional[torch.Tensor] = None,
450
) -> torch.Tensor:
451
topk_weights, topk_ids = FusedMoE.select_experts(
452
hidden_states=x,
@@ -454,7 +456,9 @@ class AWQMoEMethod(FusedMoEMethodBase):
456
renormalize=renormalize,
457
topk_group=topk_group,
458
num_expert_group=num_expert_group,
457
- custom_routing_function=custom_routing_function)
459
+ custom_routing_function=custom_routing_function,
460
+ scoring_func=scoring_func,
461
+ e_score_correction_bias=e_score_correction_bias)
462
463
return torch.ops.vllm.fused_marlin_moe(
464
x,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+15
-7
@@ -203,13 +203,14 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
203
x: torch.Tensor,
204
router_logits: torch.Tensor,
205
top_k: int,
206
- renormalize: bool = True,
206
+ renormalize: bool,
207
use_grouped_topk: bool = False,
208
- num_expert_group: Optional[int] = None,
208
topk_group: Optional[int] = None,
209
+ num_expert_group: Optional[int] = None,
210
custom_routing_function: Optional[Callable] = None,
211
+ scoring_func: str = "softmax",
212
+ e_score_correction_bias: Optional[torch.Tensor] = None,
213
) -> torch.Tensor:
212
-
214
from vllm.model_executor.layers.fused_moe import fused_experts
215
216
topk_weights, topk_ids = FusedMoE.select_experts(
@@ -220,7 +221,9 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
221
renormalize=renormalize,
222
topk_group=topk_group,
223
num_expert_group=num_expert_group,
223
- custom_routing_function=custom_routing_function)
224
+ custom_routing_function=custom_routing_function,
225
+ scoring_func=scoring_func,
226
+ e_score_correction_bias=e_score_correction_bias)
227
228
return fused_experts(x,
229
layer.w13_weight,
@@ -476,12 +479,15 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
479
x: torch.Tensor,
480
router_logits: torch.Tensor,
481
top_k: int,
479
- renormalize: bool = True,
482
+ renormalize: bool,
483
use_grouped_topk: bool = False,
481
- num_expert_group: Optional[int] = None,
484
topk_group: Optional[int] = None,
485
+ num_expert_group: Optional[int] = None,
486
custom_routing_function: Optional[Callable] = None,
487
+ scoring_func: str = "softmax",
488
+ e_score_correction_bias: Optional[torch.Tensor] = None,
489
) -> torch.Tensor:
490
+
491
topk_weights, topk_ids = FusedMoE.select_experts(
492
hidden_states=x,
493
router_logits=router_logits,
@@ -490,7 +496,9 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
496
renormalize=renormalize,
497
topk_group=topk_group,
498
num_expert_group=num_expert_group,
493
- custom_routing_function=custom_routing_function)
499
+ custom_routing_function=custom_routing_function,
500
+ scoring_func=scoring_func,
501
+ e_score_correction_bias=e_score_correction_bias)
502
503
return torch.ops.vllm.fused_marlin_moe(
504
x,
vllm/model_executor/layers/quantization/experts_int8.py
+7
-3
@@ -99,11 +99,13 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
99
x: torch.Tensor,
100
router_logits: torch.Tensor,
101
top_k: int,
102
- renormalize: bool = True,
102
+ renormalize: bool,
103
use_grouped_topk: bool = False,
104
- num_expert_group: Optional[int] = None,
104
topk_group: Optional[int] = None,
105
+ num_expert_group: Optional[int] = None,
106
custom_routing_function: Optional[Callable] = None,
107
+ scoring_func: str = "softmax",
108
+ e_score_correction_bias: Optional[torch.Tensor] = None,
109
) -> torch.Tensor:
110
from vllm.model_executor.layers.fused_moe import fused_experts
111
@@ -115,7 +117,9 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
117
renormalize=renormalize,
118
topk_group=topk_group,
119
num_expert_group=num_expert_group,
118
- custom_routing_function=custom_routing_function)
120
+ custom_routing_function=custom_routing_function,
121
+ scoring_func=scoring_func,
122
+ e_score_correction_bias=e_score_correction_bias)
123
124
return fused_experts(x,
125
layer.w13_weight,
vllm/model_executor/layers/quantization/fp8.py
+1
-2
@@ -601,14 +601,13 @@ class Fp8MoEMethod(FusedMoEMethodBase):
601
router_logits: torch.Tensor,
602
top_k: int,
603
renormalize: bool,
604
- use_grouped_topk: bool,
604
+ use_grouped_topk: bool = False,
605
topk_group: Optional[int] = None,
606
num_expert_group: Optional[int] = None,
607
custom_routing_function: Optional[Callable] = None,
608
scoring_func: str = "softmax",
609
e_score_correction_bias: Optional[torch.Tensor] = None,
610
) -> torch.Tensor:
611
-
611
from vllm.model_executor.layers.fused_moe import fused_experts
612
613
topk_weights, topk_ids = FusedMoE.select_experts(
vllm/model_executor/layers/quantization/gptq_marlin.py
+7
-3
@@ -532,11 +532,13 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
532
x: torch.Tensor,
533
router_logits: torch.Tensor,
534
top_k: int,
535
- renormalize: bool = True,
535
+ renormalize: bool,
536
use_grouped_topk: bool = False,
537
- num_expert_group: Optional[int] = None,
537
topk_group: Optional[int] = None,
538
+ num_expert_group: Optional[int] = None,
539
custom_routing_function: Optional[Callable] = None,
540
+ scoring_func: str = "softmax",
541
+ e_score_correction_bias: Optional[torch.Tensor] = None,
542
) -> torch.Tensor:
543
# The input must currently be float16
544
orig_dtype = x.dtype
@@ -550,7 +552,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
552
renormalize=renormalize,
553
topk_group=topk_group,
554
num_expert_group=num_expert_group,
553
- custom_routing_function=None)
555
+ custom_routing_function=custom_routing_function,
556
+ scoring_func=scoring_func,
557
+ e_score_correction_bias=e_score_correction_bias)
558
559
return torch.ops.vllm.fused_marlin_moe(
560
x,