A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Update disaggregation benchmark scripts and test logs (#11456)
Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com>
Jiaxin Shan committed
Dec 24, 2024 at 22:58 UTC
fc601665eb372d0dce9a873ea94eebf2ff5b1d27
6 files changed
+43
-29
benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh
+7
-6
@@ -10,7 +10,8 @@ set -ex
10
11
kill_gpu_processes() {
12
# kill all processes on GPU.
13
- pkill -f pt_main_thread
13
+ pgrep pt_main_thread | xargs -r kill -9
14
+ pgrep python3 | xargs -r kill -9
15
sleep 10
16
17
# remove vllm config file
@@ -54,7 +55,7 @@ benchmark() {
55
56
CUDA_VISIBLE_DEVICES=0 python3 \
57
-m vllm.entrypoints.openai.api_server \
57
- --model meta-llama/Meta-Llama-3.1-8B-Instruct \
58
+ --model $model \
59
--port 8100 \
60
--max-model-len 10000 \
61
--gpu-memory-utilization 0.6 \
@@ -64,7 +65,7 @@ benchmark() {
65
66
CUDA_VISIBLE_DEVICES=1 python3 \
67
-m vllm.entrypoints.openai.api_server \
67
- --model meta-llama/Meta-Llama-3.1-8B-Instruct \
68
+ --model $model \
69
--port 8200 \
70
--max-model-len 10000 \
71
--gpu-memory-utilization 0.6 \
@@ -87,7 +88,7 @@ benchmark() {
88
--port 8100 \
89
--save-result \
90
--result-dir $results_folder \
90
- --result-filename disagg_prefill_2xtp4.json \
91
+ --result-filename disagg_prefill_tp1.json \
92
--request-rate "inf"
93
94
@@ -105,7 +106,7 @@ benchmark() {
106
--port 8200 \
107
--save-result \
108
--result-dir $results_folder \
108
- --result-filename disagg_prefill_2xtp4.json \
109
+ --result-filename disagg_prefill_tp1_overhead.json \
110
--request-rate "$qps"
111
kill_gpu_processes
112
@@ -118,7 +119,7 @@ main() {
119
(which jq) || (apt-get -y install jq)
120
(which socat) || (apt-get -y install socat)
121
121
- pip install quart httpx
122
+ pip install quart httpx datasets
123
124
cd "$(dirname "$0")"
125
benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh
+6
-7
@@ -1,13 +1,12 @@
1
#!/bin/bash
2
3
-# Requirement: 8x H100 GPUs.
3
+# Requirement: 2x GPUs.
4
5
6
-# Model: neuralmagic/Meta-Llama-3-70B-Instruct-FP8-KV
7
-# Query: 2048 input tokens, 11 output tokens, QPS 4, 500 requests
8
-# Resource: 8x H100
6
+# Model: meta-llama/Meta-Llama-3.1-8B-Instruct
7
+# Query: 1024 input tokens, 6 output tokens, QPS 2/4/6/8, 100 requests
8
+# Resource: 2x GPU
9
# Approaches:
10
-# 1. Chunked prefill: 1 vllm instance with tp=8
10
# 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
11
# 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
12
# Prefilling instance: max_output_token=1
@@ -114,7 +113,6 @@ benchmark() {
113
--request-rate "$qps"
114
115
sleep 2
117
-
116
}
117
118
@@ -123,8 +121,9 @@ main() {
121
(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
122
(which jq) || (apt-get -y install jq)
123
(which socat) || (apt-get -y install socat)
124
+ (which lsof) || (apt-get -y install lsof)
125
127
- pip install quart httpx matplotlib aiohttp
126
+ pip install quart httpx matplotlib aiohttp datasets
127
128
cd "$(dirname "$0")"
129
tests/kv_transfer/test_lookup_buffer.py
+2
-2
@@ -48,7 +48,7 @@ def test_run(my_rank, buffer, device):
48
assert buffer.buffer_size == 0
49
assert len(buffer.buffer) == 0
50
51
- print("Test run passed!")
51
+ print("My rank: %d, Test run passed!" % (my_rank))
52
53
54
def stress_test(my_rank, buf, device):
@@ -108,7 +108,7 @@ def stress_test(my_rank, buf, device):
108
else:
109
torch.distributed.send(torch.tensor([n]), 0)
110
111
- print("Passed stress test!")
111
+ print("My rank: %d, Passed stress test!" % (my_rank))
112
113
114
if __name__ == "__main__":
tests/kv_transfer/test_lookup_buffer.sh
+7
-2
@@ -1,3 +1,8 @@
1
#!/bin/bash
2
-RANK=0 python test_lookup_buffer.py &
3
-RANK=1 python test_lookup_buffer.py &
\ No newline at end of file
2
+RANK=0 python3 test_lookup_buffer.py &
3
+PID0=$!
4
+RANK=1 python3 test_lookup_buffer.py &
5
+PID1=$!
6
+
7
+wait $PID0
8
+wait $PID1
tests/kv_transfer/test_send_recv.py
+14
-11
@@ -10,39 +10,42 @@ from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
10
11
12
def test_run(my_rank, pipe):
13
+ print(f"rank {my_rank} test_run starts....")
14
# test run
15
x = torch.tensor([1]).to(pipe.device)
16
y = torch.tensor([[2., 3., 4., 8.]]).to(pipe.device)
17
if my_rank == 0:
18
pipe.send_tensor(x)
18
- print("sent tensor x")
19
+ print(f"rank {my_rank} sent tensor x")
20
pipe.send_tensor(y)
20
- print("sent tensor y")
21
+ print(f"rank {my_rank} sent tensor y")
22
x2 = pipe.recv_tensor()
22
- print("received x2 = ", x2)
23
+ print(f"rank {my_rank} received x2 = ", x2)
24
y2 = pipe.recv_tensor()
24
- print("received y2 = ", x2)
25
+ print(f"rank {my_rank} received y2 = ", x2)
26
27
else:
28
x2 = pipe.recv_tensor()
28
- print("received x2 = ", x2)
29
+ print(f"rank {my_rank} received x2 = ", x2)
30
y2 = pipe.recv_tensor()
30
- print("received y2 = ", x2)
31
+ print(f"rank {my_rank} received y2 = ", x2)
32
pipe.send_tensor(x)
32
- print("sent tensor x")
33
+ print(f"rank {my_rank} sent tensor x")
34
pipe.send_tensor(y)
34
- print("sent tensor y")
35
+ print(f"rank {my_rank} sent tensor y")
36
37
assert torch.allclose(x, x2)
38
assert torch.allclose(y, y2)
39
40
+ print(f"rank {my_rank} test_run passed!")
41
40
-def stress_test(my_rank, pipe):
42
42
- torch.distributed.barrier()
43
+def stress_test(my_rank, pipe):
44
+ print(f"rank {my_rank} stress_test starts....")
45
46
tensors: List[torch.Tensor] = []
47
48
+ torch.distributed.barrier()
49
torch.manual_seed(0)
50
51
for i in tqdm(range(500)):
@@ -86,7 +89,6 @@ def stress_test(my_rank, pipe):
89
90
91
def latency_test(my_rank, pipe, nelement, ntensor):
89
-
92
latencies = []
93
94
torch.distributed.barrier()
@@ -149,6 +151,7 @@ if __name__ == "__main__":
151
)
152
153
test_run(my_rank, pipe)
154
+
155
stress_test(my_rank, pipe)
156
157
# Use this function if you want to test the latency of pipe impl.
tests/kv_transfer/test_send_recv.sh
+7
-1
@@ -1,3 +1,9 @@
1
#!/bin/bash
2
+
3
RANK=0 python3 test_send_recv.py &
3
-RANK=1 python3 test_send_recv.py &
\ No newline at end of file
4
+PID0=$!
5
+RANK=1 python3 test_send_recv.py &
6
+PID1=$!
7
+
8
+wait $PID0
9
+wait $PID1