A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 09:45 UTC
[Bugfix]Fix EP precision for Qwen3.5, Qwen3-Next (#39181)
Signed-off-by: Song Kai <songkai05@baidu.com>
Kai Song committed
Apr 9, 2026 at 05:47 UTC
f3c7941ec8d3743718ae4513bd98612f90e5d840
2 files changed
+4
vllm/model_executor/models/qwen2_moe.py
+3
@@ -80,6 +80,7 @@ class Qwen2MoeMLP(nn.Module):
80
quant_config: QuantizationConfig | None = None,
81
reduce_results: bool = True,
82
expert_gate: torch.nn.Linear | None = None,
83
+ is_sequence_parallel: bool = False,
84
prefix: str = "",
85
) -> None:
86
super().__init__()
@@ -88,6 +89,7 @@ class Qwen2MoeMLP(nn.Module):
89
[intermediate_size] * 2,
90
bias=False,
91
quant_config=quant_config,
92
+ disable_tp=is_sequence_parallel,
93
prefix=f"{prefix}.gate_up_proj",
94
)
95
self.down_proj = RowParallelLinear(
@@ -96,6 +98,7 @@ class Qwen2MoeMLP(nn.Module):
98
bias=False,
99
quant_config=quant_config,
100
reduce_results=reduce_results,
101
+ disable_tp=is_sequence_parallel,
102
prefix=f"{prefix}.down_proj",
103
)
104
if hidden_act != "silu":
vllm/model_executor/models/qwen3_next.py
+1
@@ -140,6 +140,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
140
quant_config=quant_config,
141
reduce_results=False,
142
expert_gate=self.shared_expert_gate,
143
+ is_sequence_parallel=self.is_sequence_parallel,
144
prefix=f"{prefix}.shared_expert",
145
)
146
else: