A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[XPU] Fix MOE DP accuracy issue on XPU (#25465)
Fanli Lin committed
Sep 23, 2025 at 22:32 UTC
4c966e440e1b70ff22348b23559b410c06e7b9a8
2 files changed
+29
-1
examples/offline_inference/data_parallel.py
+10
-1
@@ -101,6 +101,13 @@ def parse_args():
101
"--quantization",
102
type=str,
103
)
104
+ parser.add_argument(
105
+ "--disable-expert-parallel",
106
+ dest="enable_expert_parallel",
107
+ action="store_false",
108
+ help="Disable expert parallel (default: enabled).",
109
+ )
110
+ parser.set_defaults(enable_expert_parallel=True)
111
return parser.parse_args()
112
113
@@ -113,6 +120,7 @@ def main(
120
dp_master_port,
121
GPUs_per_dp_rank,
122
enforce_eager,
123
+ enable_expert_parallel,
124
trust_remote_code,
125
max_num_seqs,
126
max_model_len,
@@ -168,7 +176,7 @@ def main(
176
model=model,
177
tensor_parallel_size=GPUs_per_dp_rank,
178
enforce_eager=enforce_eager,
171
- enable_expert_parallel=True,
179
+ enable_expert_parallel=enable_expert_parallel,
180
trust_remote_code=trust_remote_code,
181
max_num_seqs=max_num_seqs,
182
max_model_len=max_model_len,
@@ -229,6 +237,7 @@ if __name__ == "__main__":
237
dp_master_port,
238
tp_size,
239
args.enforce_eager,
240
+ args.enable_expert_parallel,
241
args.trust_remote_code,
242
args.max_num_seqs,
243
args.max_model_len,
vllm/distributed/device_communicators/xpu_communicator.py
+19
@@ -25,6 +25,12 @@ class XpuCommunicator(DeviceCommunicatorBase):
25
super().__init__(cpu_group, device, device_group, unique_name)
26
if self.use_all2all:
27
all2all_backend = envs.VLLM_ALL2ALL_BACKEND
28
+ if all2all_backend != "naive":
29
+ logger.warning(
30
+ "`%s` all2all manager is not supported on XPU."
31
+ "Falling back to `naive` all2all manager for XPU.",
32
+ all2all_backend)
33
+ all2all_backend = "naive"
34
if all2all_backend == "naive":
35
from .all2all import NaiveAll2AllManager
36
self.all2all_manager = NaiveAll2AllManager(self.cpu_group)
@@ -67,3 +73,16 @@ class XpuCommunicator(DeviceCommunicatorBase):
73
74
def broadcast(self, input_: torch.Tensor, src: int = 0) -> None:
75
dist.broadcast(input_, src=src, group=self.device_group)
76
+
77
+ def dispatch(
78
+ self, hidden_states: torch.Tensor,
79
+ router_logits: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
80
+ assert self.all2all_manager is not None
81
+ hidden_states, router_logits = self.all2all_manager.dispatch(
82
+ hidden_states, router_logits)
83
+ return hidden_states, router_logits
84
+
85
+ def combine(self, hidden_states: torch.Tensor) -> torch.Tensor:
86
+ assert self.all2all_manager is not None
87
+ hidden_states = self.all2all_manager.combine(hidden_states)
88
+ return hidden_states