A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix][EP+DP] Use pplx-kernel internode instead of intranode (#19034)
Signed-off-by: Tyler Michael Smith <tysmith@redhat.com> Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Jun 2, 2025 at 23:36 UTC
8a57872b2ac9b01004ae1d3a3a689de218ea5be5
2 files changed
+9
-1
vllm/distributed/device_communicators/all2all.py
+4
@@ -83,6 +83,10 @@ class PPLXAll2AllManager(All2AllManagerBase):
83
assert has_pplx, "pplx_kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md to install pplx_kernels." # noqa
84
super().__init__(cpu_group)
85
86
+ # TODO(tms): Disable pplx-a2a intranode as it fails with the error:
87
+ # failed: cuda error /app/pplx/csrc/all_to_all/intranode.cpp:84 'invalid resource handle' # noqa
88
+ self.internode = True
89
+
90
if self.internode:
91
# inter-node communication needs nvshmem,
92
# intra-node communication uses p2p mapping directly
vllm/model_executor/layers/fused_moe/layer.py
+5
-1
@@ -269,9 +269,13 @@ class FusedMoEMethodBase(QuantizeMethodBase):
269
hidden_dim_scale_bytes=(0 if moe.in_dtype.itemsize != 1 else (
270
(moe.hidden_dim + moe.block_size - 1) // moe.block_size *
271
torch.float32.itemsize)),
272
- group_name=all2all_manager.cpu_group.group_name,
272
)
273
274
+ # Intranode pplx a2a takes a group name while internode does not.
275
+ if not all2all_manager.internode:
276
+ all_to_all_args[
277
+ "group_name"] = all2all_manager.cpu_group.group_name
278
+
279
handle = all2all_manager.get_handle(all_to_all_args)
280
281
prepare_finalize = PplxPrepareAndFinalize(