A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Remove deprecated `PyNcclConnector` (#24151)
Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>
Peter Pan committed
Sep 4, 2025 at 06:49 UTC
b5ee1e3261d9edf94d76ba8b437ebdef7ac599ea
7 files changed
+15
-15
benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh
+2
-2
@@ -62,7 +62,7 @@ benchmark() {
62
--max-model-len 10000 \
63
--gpu-memory-utilization 0.6 \
64
--kv-transfer-config \
65
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
65
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
66
67
68
CUDA_VISIBLE_DEVICES=1 python3 \
@@ -72,7 +72,7 @@ benchmark() {
72
--max-model-len 10000 \
73
--gpu-memory-utilization 0.6 \
74
--kv-transfer-config \
75
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
75
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
76
77
wait_for_server 8100
78
wait_for_server 8200
benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh
+2
-2
@@ -69,7 +69,7 @@ launch_disagg_prefill() {
69
--max-model-len 10000 \
70
--gpu-memory-utilization 0.6 \
71
--kv-transfer-config \
72
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
72
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
73
74
CUDA_VISIBLE_DEVICES=1 python3 \
75
-m vllm.entrypoints.openai.api_server \
@@ -78,7 +78,7 @@ launch_disagg_prefill() {
78
--max-model-len 10000 \
79
--gpu-memory-utilization 0.6 \
80
--kv-transfer-config \
81
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
81
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
82
83
wait_for_server 8100
84
wait_for_server 8200
examples/offline_inference/disaggregated_prefill.py
+6
-6
@@ -30,12 +30,12 @@ def run_prefill(prefill_done):
30
]
31
sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=1)
32
33
- # Using PyNcclConnector to transmit KV caches between vLLM instances.
33
+ # Using P2pNcclConnector to transmit KV caches between vLLM instances.
34
# This instance is the prefill node (kv_producer, rank 0).
35
# The number of parallel instances for KV cache transfer is set to 2,
36
- # as required for PyNcclConnector.
36
+ # as required for P2pNcclConnector.
37
ktc = KVTransferConfig(
38
- kv_connector="PyNcclConnector",
38
+ kv_connector="P2pNcclConnector",
39
kv_role="kv_producer",
40
kv_rank=0,
41
kv_parallel_size=2,
@@ -74,12 +74,12 @@ def run_decode(prefill_done):
74
]
75
sampling_params = SamplingParams(temperature=0, top_p=0.95)
76
77
- # Using PyNcclConnector to transmit KV caches between vLLM instances.
77
+ # Using P2pNcclConnector to transmit KV caches between vLLM instances.
78
# This instance is the decode node (kv_consumer, rank 1).
79
# The number of parallel instances for KV cache transfer is set to 2,
80
- # as required for PyNcclConnector.
80
+ # as required for P2pNcclConnector.
81
ktc = KVTransferConfig(
82
- kv_connector="PyNcclConnector",
82
+ kv_connector="P2pNcclConnector",
83
kv_role="kv_consumer",
84
kv_rank=1,
85
kv_parallel_size=2,
examples/online_serving/disaggregated_prefill.sh
+2
-2
@@ -53,7 +53,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve $MODEL_NAME \
53
--gpu-memory-utilization 0.8 \
54
--trust-remote-code \
55
--kv-transfer-config \
56
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2}' &
56
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2}' &
57
58
# decoding instance, which is the KV consumer
59
CUDA_VISIBLE_DEVICES=1 vllm serve $MODEL_NAME \
@@ -62,7 +62,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve $MODEL_NAME \
62
--gpu-memory-utilization 0.8 \
63
--trust-remote-code \
64
--kv-transfer-config \
65
- '{"kv_connector":"PyNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2}' &
65
+ '{"kv_connector":"P2pNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2}' &
66
67
# wait until prefill and decode instances are ready
68
wait_for_server 8100
tests/kv_transfer/test_lookup_buffer.py
+1
-1
@@ -128,7 +128,7 @@ if __name__ == "__main__":
128
print(f"initialized! My rank is {my_rank}")
129
130
config = KVTransferConfig(
131
- kv_connector='PyNcclConnector',
131
+ kv_connector='P2pNcclConnector',
132
kv_buffer_device='cuda',
133
kv_buffer_size=1e9,
134
kv_rank=my_rank,
tests/kv_transfer/test_send_recv.py
+1
-1
@@ -137,7 +137,7 @@ if __name__ == "__main__":
137
)
138
139
config = KVTransferConfig(
140
- kv_connector='PyNcclConnector',
140
+ kv_connector='P2pNcclConnector',
141
kv_buffer_device='cuda',
142
kv_buffer_size=1e9,
143
kv_rank=my_rank,
vllm/config/__init__.py
+1
-1
@@ -3247,7 +3247,7 @@ class KVTransferConfig:
3247
3248
kv_parallel_size: int = 1
3249
"""The number of parallel instances for KV cache transfer. For
3250
- PyNcclConnector, this should be 2."""
3250
+ P2pNcclConnector, this should be 2."""
3251
3252
kv_ip: str = "127.0.0.1"
3253
"""The KV connector ip, used to build distributed connection."""