A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Misc] Add CustomOp Interface to UnquantizedFusedMoEMethod (#6289)
Woosuk Kwon committed
Jul 15, 2024 at 12:02 UTC
ec9933f4a5d3f13f43d6c30f7f59d33127de961d
2 files changed
+35
-17
vllm/model_executor/layers/fused_moe/layer.py
+35
-13
@@ -7,7 +7,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
7
get_tensor_model_parallel_world_size,
8
tensor_model_parallel_all_reduce)
9
from vllm.logger import init_logger
10
-from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
10
+from vllm.model_executor.custom_op import CustomOp
11
from vllm.model_executor.layers.quantization.base_config import (
12
QuantizationConfig, QuantizeMethodBase)
13
from vllm.model_executor.utils import set_weight_attrs
@@ -36,7 +36,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
36
raise NotImplementedError
37
38
39
-class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
39
+class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
40
"""MoE method without quantization."""
41
42
def create_weights(self, layer: torch.nn.Module, num_experts: int,
@@ -61,19 +61,37 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
61
layer.register_parameter("w2_weight", w2_weight)
62
set_weight_attrs(w2_weight, extra_weight_attrs)
63
64
- def apply(self,
65
- layer: torch.nn.Module,
66
- x: torch.Tensor,
67
- router_logits: torch.Tensor,
68
- top_k: int,
69
- renormalize: bool = True,
70
- use_grouped_topk: bool = False,
71
- num_expert_group: Optional[int] = None,
72
- topk_group: Optional[int] = None) -> torch.Tensor:
64
+ def apply(
65
+ self,
66
+ layer: torch.nn.Module,
67
+ x: torch.Tensor,
68
+ router_logits: torch.Tensor,
69
+ top_k: int,
70
+ renormalize: bool = True,
71
+ use_grouped_topk: bool = False,
72
+ num_expert_group: Optional[int] = None,
73
+ topk_group: Optional[int] = None,
74
+ ) -> torch.Tensor:
75
+ return self.forward(x, layer.w13_weight, layer.w2_weight,
76
+ router_logits, top_k, renormalize,
77
+ use_grouped_topk, num_expert_group, topk_group)
78
79
+ def forward_cuda(
80
+ self,
81
+ x: torch.Tensor,
82
+ w1: torch.Tensor,
83
+ w2: torch.Tensor,
84
+ router_logits: torch.Tensor,
85
+ top_k: int,
86
+ renormalize: bool,
87
+ use_grouped_topk: bool,
88
+ num_expert_group: Optional[int],
89
+ topk_group: Optional[int],
90
+ ) -> torch.Tensor:
91
+ from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
92
return fused_moe(x,
75
- layer.w13_weight,
76
- layer.w2_weight,
93
+ w1,
94
+ w2,
95
router_logits,
96
top_k,
97
renormalize=renormalize,
@@ -82,6 +100,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
100
num_expert_group=num_expert_group,
101
topk_group=topk_group)
102
103
+ def forward_cpu(self, *args, **kwargs):
104
+ raise NotImplementedError(
105
+ "The CPU backend currently does not support MoE.")
106
+
107
108
class FusedMoE(torch.nn.Module):
109
"""FusedMoE layer for MoE models.
vllm/model_executor/model_loader/loader.py
-4
@@ -279,10 +279,6 @@ class DefaultModelLoader(BaseModelLoader):
279
quant_method = getattr(module, "quant_method", None)
280
if quant_method is not None:
281
quant_method.process_weights_after_loading(module)
282
- # FIXME: Remove this after Mixtral is updated
283
- # to use quant_method.
284
- if hasattr(module, "process_weights_after_loading"):
285
- module.process_weights_after_loading()
282
return model.eval()
283
284