A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Hardware][CPU] Support MOE models on x86 CPU (#11831)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jan 11, 2025 at 00:07 UTC
aa1e77a19ce658abcbaa0836f96878a7ae9dea84
3 files changed
+43
-4
docs/source/getting_started/installation/cpu-x86.md
+1
-1
@@ -5,7 +5,7 @@
5
vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
6
7
- Tensor Parallel
8
-- Model Quantization (`INT8 W8A8, AWQ`)
8
+- Model Quantization (`INT8 W8A8, AWQ, GPTQ`)
9
- Chunked-prefill
10
- Prefix-caching
11
- FP8-E5M2 KV-Caching (TODO)
tests/models/decoder_only/language/test_models.py
+4
@@ -48,6 +48,10 @@ from ...utils import check_logprobs_close
48
),
49
pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
50
pytest.param("bigcode/starcoder2-3b"), # starcoder2
51
+ pytest.param(
52
+ "ehristoforu/Falcon3-MoE-2x7B-Insruct", # mixtral
53
+ marks=[pytest.mark.cpu_model],
54
+ )
55
])
56
@pytest.mark.parametrize("dtype", ["half"])
57
@pytest.mark.parametrize("max_tokens", [32])
vllm/model_executor/layers/fused_moe/layer.py
+38
-3
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig, QuantizeMethodBase)
14
from vllm.model_executor.utils import set_weight_attrs
15
from vllm.platforms import current_platform
16
+from vllm.platforms.interface import CpuArchEnum
17
18
if current_platform.is_cuda_alike():
19
from .fused_moe import fused_experts
@@ -83,6 +84,20 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
84
layer.register_parameter("w2_weight", w2_weight)
85
set_weight_attrs(w2_weight, extra_weight_attrs)
86
87
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
88
+ super().process_weights_after_loading(layer)
89
+
90
+ if current_platform.is_cpu():
91
+ if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
92
+ import intel_extension_for_pytorch as ipex
93
+ layer.ipex_fusion = ipex.llm.modules.GatedMLPMOE(
94
+ layer.w13_weight,
95
+ layer.w2_weight,
96
+ use_prepack=True,
97
+ )
98
+ else:
99
+ raise NotImplementedError("CPU MOE only supports x86 arch.")
100
+
101
def apply(
102
self,
103
layer: torch.nn.Module,
@@ -142,9 +157,29 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
157
topk_ids=topk_ids,
158
inplace=True)
159
145
- def forward_cpu(self, *args, **kwargs):
146
- raise NotImplementedError(
147
- "The CPU backend currently does not support MoE.")
160
+ def forward_cpu(
161
+ self,
162
+ layer: torch.nn.Module,
163
+ x: torch.Tensor,
164
+ use_grouped_topk: bool,
165
+ top_k: int,
166
+ router_logits: torch.Tensor,
167
+ renormalize: bool,
168
+ topk_group: Optional[int] = None,
169
+ num_expert_group: Optional[int] = None,
170
+ custom_routing_function: Optional[Callable] = None,
171
+ **kwargs,
172
+ ):
173
+ assert custom_routing_function is None
174
+ return layer.ipex_fusion(
175
+ x,
176
+ use_grouped_topk,
177
+ top_k,
178
+ router_logits,
179
+ renormalize,
180
+ topk_group,
181
+ num_expert_group,
182
+ )
183
184
def forward_tpu(
185
self,