A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix][EP+DP] Fix internode check (#19112)
Signed-off-by: Tyler Michael Smith <tysmith@redhat.com>
Tyler Michael Smith committed
Jun 4, 2025 at 11:39 UTC
d459fae0a2c464e28680bc6d564c1de1b295029e
2 files changed
+1
-8
vllm/distributed/device_communicators/all2all.py
-6
@@ -84,10 +84,6 @@ class PPLXAll2AllManager(All2AllManagerBase):
84
assert has_pplx, "pplx_kernels not found. Please follow https://github.com/vllm-project/vllm/blob/main/tools/ep_kernels/README.md to install pplx_kernels." # noqa
85
super().__init__(cpu_group)
86
87
- # TODO(tms): Disable pplx-a2a intranode as it fails with the error:
88
- # failed: cuda error /app/pplx/csrc/all_to_all/intranode.cpp:84 'invalid resource handle' # noqa
89
- self.internode = True
90
-
87
if self.internode:
88
# inter-node communication needs nvshmem,
89
# intra-node communication uses p2p mapping directly
@@ -178,7 +174,6 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
174
num_rdma_bytes = 1024 * 1024 * 1024
175
num_qps_per_rank = self.num_sms // 2
176
else:
181
- assert self.intranode
177
num_rdma_bytes = 0
178
num_qps_per_rank = 1
179
@@ -243,7 +238,6 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
238
if self.internode:
239
num_rdma_bytes = 1024 * 1024 * 1024
240
else:
246
- assert self.intranode
241
num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint(
242
num_max_dispatch_tokens_per_rank=max_num_tokens_per_dp_rank,
243
hidden=token_hidden_size,
vllm/distributed/device_communicators/base_device_communicator.py
+1
-2
@@ -49,8 +49,7 @@ class All2AllManagerBase:
49
50
# all2all communication often has separate implementations for
51
# intra-node and inter-node communication
52
- self.intranode = in_the_same_node_as(cpu_group, source_rank=0)
53
- self.internode = not self.intranode
52
+ self.internode = not all(in_the_same_node_as(cpu_group, source_rank=0))
53
54
def get_handle(self, kwargs):
55
# get a handle for the all2all communication,