@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa1e77a19c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Hardware][CPU] Support MOE models on x86 CPU (#11831)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Jan 11, 2025 at 00:07 UTC aa1e77a19ce658abcbaa0836f96878a7ae9dea84
3 files changed +43 -4
docs/source/getting_started/installation/cpu-x86.md
+1 -1
@@ -5,7 +5,7 @@
5 vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
6
7 - Tensor Parallel
8 -- Model Quantization (`INT8 W8A8, AWQ`)
8 +- Model Quantization (`INT8 W8A8, AWQ, GPTQ`)
9 - Chunked-prefill
10 - Prefix-caching
11 - FP8-E5M2 KV-Caching (TODO)
tests/models/decoder_only/language/test_models.py
+4
@@ -48,6 +48,10 @@ from ...utils import check_logprobs_close
48 ),
49 pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
50 pytest.param("bigcode/starcoder2-3b"), # starcoder2
51 + pytest.param(
52 + "ehristoforu/Falcon3-MoE-2x7B-Insruct", # mixtral
53 + marks=[pytest.mark.cpu_model],
54 + )
55 ])
56 @pytest.mark.parametrize("dtype", ["half"])
57 @pytest.mark.parametrize("max_tokens", [32])
vllm/model_executor/layers/fused_moe/layer.py
+38 -3
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig, QuantizeMethodBase)
14 from vllm.model_executor.utils import set_weight_attrs
15 from vllm.platforms import current_platform
16 +from vllm.platforms.interface import CpuArchEnum
17
18 if current_platform.is_cuda_alike():
19 from .fused_moe import fused_experts
@@ -83,6 +84,20 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
84 layer.register_parameter("w2_weight", w2_weight)
85 set_weight_attrs(w2_weight, extra_weight_attrs)
86
87 + def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
88 + super().process_weights_after_loading(layer)
89 +
90 + if current_platform.is_cpu():
91 + if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
92 + import intel_extension_for_pytorch as ipex
93 + layer.ipex_fusion = ipex.llm.modules.GatedMLPMOE(
94 + layer.w13_weight,
95 + layer.w2_weight,
96 + use_prepack=True,
97 + )
98 + else:
99 + raise NotImplementedError("CPU MOE only supports x86 arch.")
100 +
101 def apply(
102 self,
103 layer: torch.nn.Module,
@@ -142,9 +157,29 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
157 topk_ids=topk_ids,
158 inplace=True)
159
145 - def forward_cpu(self, *args, **kwargs):
146 - raise NotImplementedError(
147 - "The CPU backend currently does not support MoE.")
160 + def forward_cpu(
161 + self,
162 + layer: torch.nn.Module,
163 + x: torch.Tensor,
164 + use_grouped_topk: bool,
165 + top_k: int,
166 + router_logits: torch.Tensor,
167 + renormalize: bool,
168 + topk_group: Optional[int] = None,
169 + num_expert_group: Optional[int] = None,
170 + custom_routing_function: Optional[Callable] = None,
171 + **kwargs,
172 + ):
173 + assert custom_routing_function is None
174 + return layer.ipex_fusion(
175 + x,
176 + use_grouped_topk,
177 + top_k,
178 + router_logits,
179 + renormalize,
180 + topk_group,
181 + num_expert_group,
182 + )
183
184 def forward_tpu(
185 self,