@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fc601665eb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Update disaggregation benchmark scripts and test logs (#11456)

Signed-off-by: Jiaxin Shan <seedjeffwan@gmail.com>

Jiaxin Shan committed Dec 24, 2024 at 22:58 UTC fc601665eb372d0dce9a873ea94eebf2ff5b1d27
6 files changed +43 -29
benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh
+7 -6
@@ -10,7 +10,8 @@ set -ex
10
11 kill_gpu_processes() {
12 # kill all processes on GPU.
13 - pkill -f pt_main_thread
13 + pgrep pt_main_thread | xargs -r kill -9
14 + pgrep python3 | xargs -r kill -9
15 sleep 10
16
17 # remove vllm config file
@@ -54,7 +55,7 @@ benchmark() {
55
56 CUDA_VISIBLE_DEVICES=0 python3 \
57 -m vllm.entrypoints.openai.api_server \
57 - --model meta-llama/Meta-Llama-3.1-8B-Instruct \
58 + --model $model \
59 --port 8100 \
60 --max-model-len 10000 \
61 --gpu-memory-utilization 0.6 \
@@ -64,7 +65,7 @@ benchmark() {
65
66 CUDA_VISIBLE_DEVICES=1 python3 \
67 -m vllm.entrypoints.openai.api_server \
67 - --model meta-llama/Meta-Llama-3.1-8B-Instruct \
68 + --model $model \
69 --port 8200 \
70 --max-model-len 10000 \
71 --gpu-memory-utilization 0.6 \
@@ -87,7 +88,7 @@ benchmark() {
88 --port 8100 \
89 --save-result \
90 --result-dir $results_folder \
90 - --result-filename disagg_prefill_2xtp4.json \
91 + --result-filename disagg_prefill_tp1.json \
92 --request-rate "inf"
93
94
@@ -105,7 +106,7 @@ benchmark() {
106 --port 8200 \
107 --save-result \
108 --result-dir $results_folder \
108 - --result-filename disagg_prefill_2xtp4.json \
109 + --result-filename disagg_prefill_tp1_overhead.json \
110 --request-rate "$qps"
111 kill_gpu_processes
112
@@ -118,7 +119,7 @@ main() {
119 (which jq) || (apt-get -y install jq)
120 (which socat) || (apt-get -y install socat)
121
121 - pip install quart httpx
122 + pip install quart httpx datasets
123
124 cd "$(dirname "$0")"
125
benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh
+6 -7
@@ -1,13 +1,12 @@
1 #!/bin/bash
2
3 -# Requirement: 8x H100 GPUs.
3 +# Requirement: 2x GPUs.
4
5
6 -# Model: neuralmagic/Meta-Llama-3-70B-Instruct-FP8-KV
7 -# Query: 2048 input tokens, 11 output tokens, QPS 4, 500 requests
8 -# Resource: 8x H100
6 +# Model: meta-llama/Meta-Llama-3.1-8B-Instruct
7 +# Query: 1024 input tokens, 6 output tokens, QPS 2/4/6/8, 100 requests
8 +# Resource: 2x GPU
9 # Approaches:
10 -# 1. Chunked prefill: 1 vllm instance with tp=8
10 # 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
11 # 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
12 # Prefilling instance: max_output_token=1
@@ -114,7 +113,6 @@ benchmark() {
113 --request-rate "$qps"
114
115 sleep 2
117 -
116 }
117
118
@@ -123,8 +121,9 @@ main() {
121 (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
122 (which jq) || (apt-get -y install jq)
123 (which socat) || (apt-get -y install socat)
124 + (which lsof) || (apt-get -y install lsof)
125
127 - pip install quart httpx matplotlib aiohttp
126 + pip install quart httpx matplotlib aiohttp datasets
127
128 cd "$(dirname "$0")"
129
tests/kv_transfer/test_lookup_buffer.py
+2 -2
@@ -48,7 +48,7 @@ def test_run(my_rank, buffer, device):
48 assert buffer.buffer_size == 0
49 assert len(buffer.buffer) == 0
50
51 - print("Test run passed!")
51 + print("My rank: %d, Test run passed!" % (my_rank))
52
53
54 def stress_test(my_rank, buf, device):
@@ -108,7 +108,7 @@ def stress_test(my_rank, buf, device):
108 else:
109 torch.distributed.send(torch.tensor([n]), 0)
110
111 - print("Passed stress test!")
111 + print("My rank: %d, Passed stress test!" % (my_rank))
112
113
114 if __name__ == "__main__":
tests/kv_transfer/test_lookup_buffer.sh
+7 -2
@@ -1,3 +1,8 @@
1 #!/bin/bash
2 -RANK=0 python test_lookup_buffer.py &
3 -RANK=1 python test_lookup_buffer.py &
\ No newline at end of file
2 +RANK=0 python3 test_lookup_buffer.py &
3 +PID0=$!
4 +RANK=1 python3 test_lookup_buffer.py &
5 +PID1=$!
6 +
7 +wait $PID0
8 +wait $PID1
tests/kv_transfer/test_send_recv.py
+14 -11
@@ -10,39 +10,42 @@ from vllm.distributed.kv_transfer.kv_pipe.pynccl_pipe import PyNcclPipe
10
11
12 def test_run(my_rank, pipe):
13 + print(f"rank {my_rank} test_run starts....")
14 # test run
15 x = torch.tensor([1]).to(pipe.device)
16 y = torch.tensor([[2., 3., 4., 8.]]).to(pipe.device)
17 if my_rank == 0:
18 pipe.send_tensor(x)
18 - print("sent tensor x")
19 + print(f"rank {my_rank} sent tensor x")
20 pipe.send_tensor(y)
20 - print("sent tensor y")
21 + print(f"rank {my_rank} sent tensor y")
22 x2 = pipe.recv_tensor()
22 - print("received x2 = ", x2)
23 + print(f"rank {my_rank} received x2 = ", x2)
24 y2 = pipe.recv_tensor()
24 - print("received y2 = ", x2)
25 + print(f"rank {my_rank} received y2 = ", x2)
26
27 else:
28 x2 = pipe.recv_tensor()
28 - print("received x2 = ", x2)
29 + print(f"rank {my_rank} received x2 = ", x2)
30 y2 = pipe.recv_tensor()
30 - print("received y2 = ", x2)
31 + print(f"rank {my_rank} received y2 = ", x2)
32 pipe.send_tensor(x)
32 - print("sent tensor x")
33 + print(f"rank {my_rank} sent tensor x")
34 pipe.send_tensor(y)
34 - print("sent tensor y")
35 + print(f"rank {my_rank} sent tensor y")
36
37 assert torch.allclose(x, x2)
38 assert torch.allclose(y, y2)
39
40 + print(f"rank {my_rank} test_run passed!")
41
40 -def stress_test(my_rank, pipe):
42
42 - torch.distributed.barrier()
43 +def stress_test(my_rank, pipe):
44 + print(f"rank {my_rank} stress_test starts....")
45
46 tensors: List[torch.Tensor] = []
47
48 + torch.distributed.barrier()
49 torch.manual_seed(0)
50
51 for i in tqdm(range(500)):
@@ -86,7 +89,6 @@ def stress_test(my_rank, pipe):
89
90
91 def latency_test(my_rank, pipe, nelement, ntensor):
89 -
92 latencies = []
93
94 torch.distributed.barrier()
@@ -149,6 +151,7 @@ if __name__ == "__main__":
151 )
152
153 test_run(my_rank, pipe)
154 +
155 stress_test(my_rank, pipe)
156
157 # Use this function if you want to test the latency of pipe impl.
tests/kv_transfer/test_send_recv.sh
+7 -1
@@ -1,3 +1,9 @@
1 #!/bin/bash
2 +
3 RANK=0 python3 test_send_recv.py &
3 -RANK=1 python3 test_send_recv.py &
\ No newline at end of file
4 +PID0=$!
5 +RANK=1 python3 test_send_recv.py &
6 +PID1=$!
7 +
8 +wait $PID0
9 +wait $PID1