A high-throughput and memory-efficient inference and serving engine for LLMs
[Bug] Fix deepseek v4 OOM issue (#44914)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jun 9, 2026 at 18:47 UTC
d7607ad2730ff26b5cb8730354179a4d42dc45d1
1 file changed
+10
-3
vllm/models/deepseek_v4/quant_config.py
+10
-3
@@ -7,7 +7,11 @@ from __future__ import annotations
7
from typing import TYPE_CHECKING
8
9
from vllm.config import get_current_vllm_config
10
-from vllm.model_executor.layers.fused_moe import MoERunner, UnquantizedFusedMoEMethod
10
+from vllm.model_executor.layers.fused_moe import (
11
+ MoERunner,
12
+ RoutedExperts,
13
+ UnquantizedFusedMoEMethod,
14
+)
15
from vllm.model_executor.layers.quantization import QuantizationMethods
16
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
17
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
@@ -129,7 +133,7 @@ class DeepseekV4FP8Config(Fp8Config):
133
return None
134
135
def get_quant_method(self, layer, prefix):
132
- if isinstance(layer, MoERunner):
136
+ if isinstance(layer, (MoERunner, RoutedExperts)):
137
if is_layer_skipped(
138
prefix=prefix,
139
ignored_layers=self.ignored_layers,
@@ -152,6 +156,9 @@ class DeepseekV4FP8Config(Fp8Config):
156
return super().get_quant_method(layer, prefix)
157
158
def is_mxfp4_quant(self, prefix, layer):
155
- if not isinstance(layer, MoERunner) or self.expert_dtype != "fp4":
159
+ if (
160
+ not isinstance(layer, (MoERunner, RoutedExperts))
161
+ or self.expert_dtype != "fp4"
162
+ ):
163
return False
164
return self.moe_quant_algo != "NVFP4"