A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 09:00 UTC
[Bugfix] Avoid src/dst as None in irecv/isend_tensor_dict (#35754)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Mar 3, 2026 at 21:56 UTC
440f0e7dc6cb0adfc9c3c98076939668b90c4bf2
2 files changed
+30
-12
.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
+21
-4
@@ -1,26 +1,43 @@
1
#!/bin/bash
2
set -euox pipefail
3
+export VLLM_CPU_CI_ENV=0
4
5
echo "--- PP+TP"
6
vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
7
server_pid=$!
7
-timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
8
+timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
9
vllm bench serve \
10
--backend vllm \
11
--dataset-name random \
12
--model meta-llama/Llama-3.2-3B-Instruct \
13
--num-prompts 20 \
14
+ --result-dir ./test_results \
15
+ --result-filename tp_pp.json \
16
+ --save-result \
17
--endpoint /v1/completions
14
-kill -s SIGTERM $server_pid &
18
+kill -s SIGTERM $server_pid; wait $server_pid || true
19
+failed_req=$(jq '.failed' ./test_results/tp_pp.json)
20
+if [ "$failed_req" -ne 0 ]; then
21
+ echo "Some requests were failed!"
22
+ exit 1
23
+fi
24
25
echo "--- DP+TP"
26
vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
27
server_pid=$!
19
-timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
28
+timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
29
vllm bench serve \
30
--backend vllm \
31
--dataset-name random \
32
--model meta-llama/Llama-3.2-3B-Instruct \
33
--num-prompts 20 \
34
+ --result-dir ./test_results \
35
+ --result-filename dp_pp.json \
36
+ --save-result \
37
--endpoint /v1/completions
26
-kill -s SIGTERM $server_pid &
38
+kill -s SIGTERM $server_pid; wait $server_pid || true
39
+failed_req=$(jq '.failed' ./test_results/dp_pp.json)
40
+if [ "$failed_req" -ne 0 ]; then
41
+ echo "Some requests were failed!"
42
+ exit 1
43
+fi
vllm/distributed/parallel_state.py
+9
-8
@@ -851,6 +851,10 @@ class GroupCoordinator:
851
if self.world_size <= 1:
852
return []
853
854
+ if dst is None:
855
+ dst = (self.rank_in_group + 1) % self.world_size
856
+ assert dst < self.world_size, f"Invalid dst rank ({dst})"
857
+
858
if self.use_cpu_custom_send_recv:
859
if self.device_communicator is None:
860
raise ValueError("No device communicator found")
@@ -868,10 +872,6 @@ class GroupCoordinator:
872
group = self.device_group
873
metadata_group = self.cpu_group
874
871
- if dst is None:
872
- dst = (self.rank_in_group + 1) % self.world_size
873
- assert dst < self.world_size, f"Invalid dst rank ({dst})"
874
-
875
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
876
self.send_object(metadata_list, dst=dst)
877
@@ -948,6 +948,11 @@ class GroupCoordinator:
948
]:
949
if not torch.distributed.is_initialized() or self.world_size == 1:
950
return None, [], []
951
+
952
+ if src is None:
953
+ src = (self.rank_in_group - 1) % self.world_size
954
+ assert src < self.world_size, f"Invalid src rank ({src})"
955
+
956
if self.use_cpu_custom_send_recv:
957
if self.device_communicator is None:
958
raise ValueError("No device communicator found")
@@ -965,10 +970,6 @@ class GroupCoordinator:
970
group = self.device_group
971
metadata_group = self.cpu_group
972
968
- if src is None:
969
- src = (self.rank_in_group - 1) % self.world_size
970
- assert src < self.world_size, f"Invalid src rank ({src})"
971
-
973
recv_metadata_list = self.recv_object(src=src)
974
tensor_dict: dict[str, Any] = {}
975
handles: list[Handle] = []