@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec9933f4a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Add CustomOp Interface to UnquantizedFusedMoEMethod (#6289)

Woosuk Kwon committed Jul 15, 2024 at 12:02 UTC ec9933f4a5d3f13f43d6c30f7f59d33127de961d
2 files changed +35 -17
vllm/model_executor/layers/fused_moe/layer.py
+35 -13
@@ -7,7 +7,7 @@ from vllm.distributed import (get_tensor_model_parallel_rank,
7 get_tensor_model_parallel_world_size,
8 tensor_model_parallel_all_reduce)
9 from vllm.logger import init_logger
10 -from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
10 +from vllm.model_executor.custom_op import CustomOp
11 from vllm.model_executor.layers.quantization.base_config import (
12 QuantizationConfig, QuantizeMethodBase)
13 from vllm.model_executor.utils import set_weight_attrs
@@ -36,7 +36,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
36 raise NotImplementedError
37
38
39 -class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
39 +class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
40 """MoE method without quantization."""
41
42 def create_weights(self, layer: torch.nn.Module, num_experts: int,
@@ -61,19 +61,37 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
61 layer.register_parameter("w2_weight", w2_weight)
62 set_weight_attrs(w2_weight, extra_weight_attrs)
63
64 - def apply(self,
65 - layer: torch.nn.Module,
66 - x: torch.Tensor,
67 - router_logits: torch.Tensor,
68 - top_k: int,
69 - renormalize: bool = True,
70 - use_grouped_topk: bool = False,
71 - num_expert_group: Optional[int] = None,
72 - topk_group: Optional[int] = None) -> torch.Tensor:
64 + def apply(
65 + self,
66 + layer: torch.nn.Module,
67 + x: torch.Tensor,
68 + router_logits: torch.Tensor,
69 + top_k: int,
70 + renormalize: bool = True,
71 + use_grouped_topk: bool = False,
72 + num_expert_group: Optional[int] = None,
73 + topk_group: Optional[int] = None,
74 + ) -> torch.Tensor:
75 + return self.forward(x, layer.w13_weight, layer.w2_weight,
76 + router_logits, top_k, renormalize,
77 + use_grouped_topk, num_expert_group, topk_group)
78
79 + def forward_cuda(
80 + self,
81 + x: torch.Tensor,
82 + w1: torch.Tensor,
83 + w2: torch.Tensor,
84 + router_logits: torch.Tensor,
85 + top_k: int,
86 + renormalize: bool,
87 + use_grouped_topk: bool,
88 + num_expert_group: Optional[int],
89 + topk_group: Optional[int],
90 + ) -> torch.Tensor:
91 + from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe
92 return fused_moe(x,
75 - layer.w13_weight,
76 - layer.w2_weight,
93 + w1,
94 + w2,
95 router_logits,
96 top_k,
97 renormalize=renormalize,
@@ -82,6 +100,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase):
100 num_expert_group=num_expert_group,
101 topk_group=topk_group)
102
103 + def forward_cpu(self, *args, **kwargs):
104 + raise NotImplementedError(
105 + "The CPU backend currently does not support MoE.")
106 +
107
108 class FusedMoE(torch.nn.Module):
109 """FusedMoE layer for MoE models.
vllm/model_executor/model_loader/loader.py
-4
@@ -279,10 +279,6 @@ class DefaultModelLoader(BaseModelLoader):
279 quant_method = getattr(module, "quant_method", None)
280 if quant_method is not None:
281 quant_method.process_weights_after_loading(module)
282 - # FIXME: Remove this after Mixtral is updated
283 - # to use quant_method.
284 - if hasattr(module, "process_weights_after_loading"):
285 - module.process_weights_after_loading()
282 return model.eval()
283
284