@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2339d59f92
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[BugFix] Fix quantization for all other methods (#11547)

Robert Shaw committed Dec 27, 2024 at 01:23 UTC 2339d59f9260499599d60599f83978fad1827999
6 files changed +52 -22
vllm/model_executor/layers/fused_moe/layer.py
+15 -4
@@ -41,9 +41,20 @@ class FusedMoEMethodBase(QuantizeMethodBase):
41 raise NotImplementedError
42
43 @abstractmethod
44 - def apply(self, layer: torch.nn.Module, x: torch.Tensor,
45 - router_logits: torch.Tensor, top_k: int, renormalize: bool,
46 - use_grouped_topk: bool) -> torch.Tensor:
44 + def apply(
45 + self,
46 + layer: torch.nn.Module,
47 + x: torch.Tensor,
48 + router_logits: torch.Tensor,
49 + top_k: int,
50 + renormalize: bool,
51 + use_grouped_topk: bool = False,
52 + topk_group: Optional[int] = None,
53 + num_expert_group: Optional[int] = None,
54 + custom_routing_function: Optional[Callable] = None,
55 + scoring_func: str = "softmax",
56 + e_score_correction_bias: Optional[torch.Tensor] = None
57 + ) -> torch.Tensor:
58 raise NotImplementedError
59
60
@@ -79,7 +90,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
90 router_logits: torch.Tensor,
91 top_k: int,
92 renormalize: bool,
82 - use_grouped_topk: bool,
93 + use_grouped_topk: bool = False,
94 topk_group: Optional[int] = None,
95 num_expert_group: Optional[int] = None,
96 custom_routing_function: Optional[Callable] = None,
vllm/model_executor/layers/quantization/awq_marlin.py
+7 -3
@@ -440,11 +440,13 @@ class AWQMoEMethod(FusedMoEMethodBase):
440 x: torch.Tensor,
441 router_logits: torch.Tensor,
442 top_k: int,
443 - renormalize: bool = True,
443 + renormalize: bool,
444 use_grouped_topk: bool = False,
445 - num_expert_group: Optional[int] = None,
445 topk_group: Optional[int] = None,
446 + num_expert_group: Optional[int] = None,
447 custom_routing_function: Optional[Callable] = None,
448 + scoring_func: str = "softmax",
449 + e_score_correction_bias: Optional[torch.Tensor] = None,
450 ) -> torch.Tensor:
451 topk_weights, topk_ids = FusedMoE.select_experts(
452 hidden_states=x,
@@ -454,7 +456,9 @@ class AWQMoEMethod(FusedMoEMethodBase):
456 renormalize=renormalize,
457 topk_group=topk_group,
458 num_expert_group=num_expert_group,
457 - custom_routing_function=custom_routing_function)
459 + custom_routing_function=custom_routing_function,
460 + scoring_func=scoring_func,
461 + e_score_correction_bias=e_score_correction_bias)
462
463 return torch.ops.vllm.fused_marlin_moe(
464 x,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+15 -7
@@ -203,13 +203,14 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
203 x: torch.Tensor,
204 router_logits: torch.Tensor,
205 top_k: int,
206 - renormalize: bool = True,
206 + renormalize: bool,
207 use_grouped_topk: bool = False,
208 - num_expert_group: Optional[int] = None,
208 topk_group: Optional[int] = None,
209 + num_expert_group: Optional[int] = None,
210 custom_routing_function: Optional[Callable] = None,
211 + scoring_func: str = "softmax",
212 + e_score_correction_bias: Optional[torch.Tensor] = None,
213 ) -> torch.Tensor:
212 -
214 from vllm.model_executor.layers.fused_moe import fused_experts
215
216 topk_weights, topk_ids = FusedMoE.select_experts(
@@ -220,7 +221,9 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
221 renormalize=renormalize,
222 topk_group=topk_group,
223 num_expert_group=num_expert_group,
223 - custom_routing_function=custom_routing_function)
224 + custom_routing_function=custom_routing_function,
225 + scoring_func=scoring_func,
226 + e_score_correction_bias=e_score_correction_bias)
227
228 return fused_experts(x,
229 layer.w13_weight,
@@ -476,12 +479,15 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
479 x: torch.Tensor,
480 router_logits: torch.Tensor,
481 top_k: int,
479 - renormalize: bool = True,
482 + renormalize: bool,
483 use_grouped_topk: bool = False,
481 - num_expert_group: Optional[int] = None,
484 topk_group: Optional[int] = None,
485 + num_expert_group: Optional[int] = None,
486 custom_routing_function: Optional[Callable] = None,
487 + scoring_func: str = "softmax",
488 + e_score_correction_bias: Optional[torch.Tensor] = None,
489 ) -> torch.Tensor:
490 +
491 topk_weights, topk_ids = FusedMoE.select_experts(
492 hidden_states=x,
493 router_logits=router_logits,
@@ -490,7 +496,9 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
496 renormalize=renormalize,
497 topk_group=topk_group,
498 num_expert_group=num_expert_group,
493 - custom_routing_function=custom_routing_function)
499 + custom_routing_function=custom_routing_function,
500 + scoring_func=scoring_func,
501 + e_score_correction_bias=e_score_correction_bias)
502
503 return torch.ops.vllm.fused_marlin_moe(
504 x,
vllm/model_executor/layers/quantization/experts_int8.py
+7 -3
@@ -99,11 +99,13 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
99 x: torch.Tensor,
100 router_logits: torch.Tensor,
101 top_k: int,
102 - renormalize: bool = True,
102 + renormalize: bool,
103 use_grouped_topk: bool = False,
104 - num_expert_group: Optional[int] = None,
104 topk_group: Optional[int] = None,
105 + num_expert_group: Optional[int] = None,
106 custom_routing_function: Optional[Callable] = None,
107 + scoring_func: str = "softmax",
108 + e_score_correction_bias: Optional[torch.Tensor] = None,
109 ) -> torch.Tensor:
110 from vllm.model_executor.layers.fused_moe import fused_experts
111
@@ -115,7 +117,9 @@ class ExpertsInt8MoEMethod(FusedMoEMethodBase):
117 renormalize=renormalize,
118 topk_group=topk_group,
119 num_expert_group=num_expert_group,
118 - custom_routing_function=custom_routing_function)
120 + custom_routing_function=custom_routing_function,
121 + scoring_func=scoring_func,
122 + e_score_correction_bias=e_score_correction_bias)
123
124 return fused_experts(x,
125 layer.w13_weight,
vllm/model_executor/layers/quantization/fp8.py
+1 -2
@@ -601,14 +601,13 @@ class Fp8MoEMethod(FusedMoEMethodBase):
601 router_logits: torch.Tensor,
602 top_k: int,
603 renormalize: bool,
604 - use_grouped_topk: bool,
604 + use_grouped_topk: bool = False,
605 topk_group: Optional[int] = None,
606 num_expert_group: Optional[int] = None,
607 custom_routing_function: Optional[Callable] = None,
608 scoring_func: str = "softmax",
609 e_score_correction_bias: Optional[torch.Tensor] = None,
610 ) -> torch.Tensor:
611 -
611 from vllm.model_executor.layers.fused_moe import fused_experts
612
613 topk_weights, topk_ids = FusedMoE.select_experts(
vllm/model_executor/layers/quantization/gptq_marlin.py
+7 -3
@@ -532,11 +532,13 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
532 x: torch.Tensor,
533 router_logits: torch.Tensor,
534 top_k: int,
535 - renormalize: bool = True,
535 + renormalize: bool,
536 use_grouped_topk: bool = False,
537 - num_expert_group: Optional[int] = None,
537 topk_group: Optional[int] = None,
538 + num_expert_group: Optional[int] = None,
539 custom_routing_function: Optional[Callable] = None,
540 + scoring_func: str = "softmax",
541 + e_score_correction_bias: Optional[torch.Tensor] = None,
542 ) -> torch.Tensor:
543 # The input must currently be float16
544 orig_dtype = x.dtype
@@ -550,7 +552,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
552 renormalize=renormalize,
553 topk_group=topk_group,
554 num_expert_group=num_expert_group,
553 - custom_routing_function=None)
555 + custom_routing_function=custom_routing_function,
556 + scoring_func=scoring_func,
557 + e_score_correction_bias=e_score_correction_bias)
558
559 return torch.ops.vllm.fused_marlin_moe(
560 x,