@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7c4f9ee86

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 23:45 UTC

[CI/Build][Doc] Move existing benchmark scripts in CI/document/example to vllm bench CLI (#21355)

Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com>

Ye (Charlotte) Qi committed Jul 26, 2025 at 07:10 UTC e7c4f9ee866296f49ac392fc32a662da0204e70c

14 files changed +102 -87

.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh

+12 -12

@@ -73,7 +73,7 @@ get_current_llm_serving_engine() {

73 echo "Container: vllm"

74 # move to a completely irrelevant directory, to avoid import vllm from current folder

75 export CURRENT_LLM_SERVING_ENGINE=vllm

76 -

76 +

77 return

78 fi

79 }

@@ -227,7 +227,7 @@ run_serving_tests() {

227

228 if [[ "$dataset_name" = "sharegpt" ]]; then

229

230 - client_command="python3 benchmark_serving.py \

230 + client_command="vllm bench serve \

231 --backend $backend \

232 --tokenizer /tokenizer_cache \

233 --model $model \

@@ -248,7 +248,7 @@ run_serving_tests() {

248 sonnet_output_len=$(echo "$common_params" | jq -r '.sonnet_output_len')

249 sonnet_prefix_len=$(echo "$common_params" | jq -r '.sonnet_prefix_len')

250

251 - client_command="python3 benchmark_serving.py \

251 + client_command="vllm bench serve \

252 --backend $backend \

253 --tokenizer /tokenizer_cache \

254 --model $model \

@@ -267,13 +267,13 @@ run_serving_tests() {

267 $client_args"

268

269 else

270 -

270 +

271 echo "The dataset name must be either 'sharegpt' or 'sonnet'. Got $dataset_name."

272 exit 1

273

274 fi

275

276 -

276 +

277

278 echo "Running test case $test_name with qps $qps"

279 echo "Client command: $client_command"

@@ -304,7 +304,7 @@ run_serving_tests() {

304 }

305

306 run_genai_perf_tests() {

307 - # run genai-perf tests

307 + # run genai-perf tests

308

309 # $1: a json file specifying genai-perf test cases

310 local genai_perf_test_file

@@ -313,14 +313,14 @@ run_genai_perf_tests() {

313 # Iterate over genai-perf tests

314 jq -c '.[]' "$genai_perf_test_file" | while read -r params; do

315 # get the test name, and append the GPU type back to it.

316 - test_name=$(echo "$params" | jq -r '.test_name')

317 -

316 + test_name=$(echo "$params" | jq -r '.test_name')

317 +

318 # if TEST_SELECTOR is set, only run the test cases that match the selector

319 if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then

320 echo "Skip test case $test_name."

321 continue

322 fi

323 -

323 +

324 # prepend the current serving engine to the test name

325 test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name}

326

@@ -371,10 +371,10 @@ run_genai_perf_tests() {

371 qps=$num_prompts

372 echo "now qps is $qps"

373 fi

374 -

374 +

375 new_test_name=$test_name"_qps_"$qps

376 backend=$CURRENT_LLM_SERVING_ENGINE

377 -

377 +

378 if [[ "$backend" == *"vllm"* ]]; then

379 backend="vllm"

380 fi

@@ -415,7 +415,7 @@ prepare_dataset() {

415 do

416 cat sonnet.txt >> sonnet_4x.txt

417 done

418 -

418 +

419 }

420

421 main() {

.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh

+3 -3

@@ -206,7 +206,7 @@ run_latency_tests() {

206 fi

207 fi

208

209 - latency_command=" $latency_envs python3 benchmark_latency.py \

209 + latency_command=" $latency_envs vllm bench latency \

210 --output-json $RESULTS_FOLDER/${test_name}.json \

211 $latency_args"

212

@@ -273,7 +273,7 @@ run_throughput_tests() {

273 fi

274 fi

275

276 - throughput_command=" $throughput_envs python3 benchmark_throughput.py \

276 + throughput_command=" $throughput_envs vllm bench throughput \

277 --output-json $RESULTS_FOLDER/${test_name}.json \

278 $throughput_args"

279

@@ -394,7 +394,7 @@ run_serving_tests() {

394

395 # pass the tensor parallel size to the client so that it can be displayed

396 # on the benchmark dashboard

397 - client_command="python3 benchmark_serving.py \

397 + client_command="vllm bench serve \

398 --save-result \

399 --result-dir $RESULTS_FOLDER \

400 --result-filename ${new_test_name}.json \

.buildkite/scripts/hardware_ci/run-cpu-test.sh

+5 -5

@@ -13,9 +13,9 @@ NUMA_NODE=${NUMA_NODE:-1}

13 export CMAKE_BUILD_PARALLEL_LEVEL=32

15 # Setup cleanup

16 -remove_docker_container() {

17 - set -e;

18 - docker rm -f cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"-avx2 || true;

16 +remove_docker_container() {

17 + set -e;

18 + docker rm -f cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"-avx2 || true;

19 }

20 trap remove_docker_container EXIT

21 remove_docker_container

@@ -69,7 +69,7 @@ function cpu_tests() {

69 docker exec cpu-test-"$NUMA_NODE" bash -c "

70 set -e

71 pytest -s -v \

72 - tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs[False-10-32-neuralmagic/Llama-3.2-1B-quantized.w8a8]"

72 + tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs[False-10-32-neuralmagic/Llama-3.2-1B-quantized.w8a8]"

74 # Note: disable it until supports V1

75 # Run AWQ test

@@ -83,7 +83,7 @@ function cpu_tests() {

83 set -e

84 VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS VLLM_CPU_SGL_KERNEL=1 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &

85 timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1

86 - python3 benchmarks/benchmark_serving.py \

86 + vllm bench serve \

87 --backend vllm \

88 --dataset-name random \

89 --model meta-llama/Llama-3.2-3B-Instruct \

.buildkite/scripts/run-benchmarks.sh

+3 -3

@@ -11,10 +11,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../.."

11 (which wget && which curl) || (apt-get update && apt-get install -y wget curl)

13 # run python-based benchmarks and upload the result to buildkite

14 -python3 benchmarks/benchmark_latency.py --output-json latency_results.json 2>&1 | tee benchmark_latency.txt

14 +vllm bench latency --output-json latency_results.json 2>&1 | tee benchmark_latency.txt

15 bench_latency_exit_code=$?

17 -python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 --output-json throughput_results.json 2>&1 | tee benchmark_throughput.txt

17 +vllm bench throughput --input-len 256 --output-len 256 --output-json throughput_results.json 2>&1 | tee benchmark_throughput.txt

18 bench_throughput_exit_code=$?

20 # run server-based benchmarks and upload the result to buildkite

@@ -24,7 +24,7 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/r

25 # wait for server to start, timeout after 600 seconds

26 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1

27 -python3 benchmarks/benchmark_serving.py \

27 +vllm bench serve \

28 --backend vllm \

29 --dataset-name sharegpt \

30 --dataset-path ./ShareGPT_V3_unfiltered_cleaned_split.json \

.buildkite/scripts/tpu/run_bm.sh

+1 -1

@@ -77,7 +77,7 @@ done

77 echo "run benchmark test..."

78 echo "logging to $BM_LOG"

79 echo

80 -python benchmarks/benchmark_serving.py \

80 +vllm bench serve \

81 --backend vllm \

82 --model $MODEL \

83 --dataset-name sonnet \

benchmarks/README.md

+33 -33

@@ -98,7 +98,7 @@ Then run the benchmarking script

98 ```bash

99 # download dataset

100 # wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json

101 -python3 vllm/benchmarks/benchmark_serving.py \

101 +vllm bench serve \

102 --backend vllm \

103 --model NousResearch/Hermes-3-Llama-3.1-8B \

104 --endpoint /v1/completions \

@@ -111,25 +111,25 @@ If successful, you will see the following output

111

112 ```

113 ============ Serving Benchmark Result ============

114 -Successful requests: 10

115 -Benchmark duration (s): 5.78

116 -Total input tokens: 1369

117 -Total generated tokens: 2212

118 -Request throughput (req/s): 1.73

119 -Output token throughput (tok/s): 382.89

120 -Total Token throughput (tok/s): 619.85

114 +Successful requests: 10

115 +Benchmark duration (s): 5.78

116 +Total input tokens: 1369

117 +Total generated tokens: 2212

118 +Request throughput (req/s): 1.73

119 +Output token throughput (tok/s): 382.89

120 +Total Token throughput (tok/s): 619.85

121 ---------------Time to First Token----------------

122 -Mean TTFT (ms): 71.54

123 -Median TTFT (ms): 73.88

124 -P99 TTFT (ms): 79.49

122 +Mean TTFT (ms): 71.54

123 +Median TTFT (ms): 73.88

124 +P99 TTFT (ms): 79.49

125 -----Time per Output Token (excl. 1st token)------

126 -Mean TPOT (ms): 7.91

127 -Median TPOT (ms): 7.96

128 -P99 TPOT (ms): 8.03

126 +Mean TPOT (ms): 7.91

127 +Median TPOT (ms): 7.96

128 +P99 TPOT (ms): 8.03

129 ---------------Inter-token Latency----------------

130 -Mean ITL (ms): 7.74

131 -Median ITL (ms): 7.70

132 -P99 ITL (ms): 8.39

130 +Mean ITL (ms): 7.74

131 +Median ITL (ms): 7.70

132 +P99 ITL (ms): 8.39

133 ==================================================

134 ```

135

@@ -141,7 +141,7 @@ If the dataset you want to benchmark is not supported yet in vLLM, even then you

141 {"prompt": "What is the capital of India?"}

142 {"prompt": "What is the capital of Iran?"}

143 {"prompt": "What is the capital of China?"}

144 -```

144 +```

145

146 ```bash

147 # start server

@@ -150,7 +150,7 @@ VLLM_USE_V1=1 vllm serve meta-llama/Llama-3.1-8B-Instruct --disable-log-requests

150

151 ```bash

152 # run benchmarking script

153 -python3 benchmarks/benchmark_serving.py --port 9001 --save-result --save-detailed \

153 +vllm bench serve --port 9001 --save-result --save-detailed \

154 --backend vllm \

155 --model meta-llama/Llama-3.1-8B-Instruct \

156 --endpoint /v1/completions \

@@ -174,7 +174,7 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct --disable-log-requests

174 ```

175

176 ```bash

177 -python3 vllm/benchmarks/benchmark_serving.py \

177 +vllm bench serve \

178 --backend openai-chat \

179 --model Qwen/Qwen2-VL-7B-Instruct \

180 --endpoint /v1/chat/completions \

@@ -194,7 +194,7 @@ VLLM_USE_V1=1 vllm serve meta-llama/Meta-Llama-3-8B-Instruct \

194 ```

195

196 ``` bash

197 -python3 benchmarks/benchmark_serving.py \

197 +vllm bench serve \

198 --model meta-llama/Meta-Llama-3-8B-Instruct \

199 --dataset-name hf \

200 --dataset-path likaixin/InstructCoder \

@@ -210,7 +210,7 @@ vllm serve Qwen/Qwen2-VL-7B-Instruct --disable-log-requests

210 **`lmms-lab/LLaVA-OneVision-Data`**

211

212 ```bash

213 -python3 vllm/benchmarks/benchmark_serving.py \

213 +vllm bench serve \

214 --backend openai-chat \

215 --model Qwen/Qwen2-VL-7B-Instruct \

216 --endpoint /v1/chat/completions \

@@ -224,7 +224,7 @@ python3 vllm/benchmarks/benchmark_serving.py \

224 **`Aeala/ShareGPT_Vicuna_unfiltered`**

225

226 ```bash

227 -python3 vllm/benchmarks/benchmark_serving.py \

227 +vllm bench serve \

228 --backend openai-chat \

229 --model Qwen/Qwen2-VL-7B-Instruct \

230 --endpoint /v1/chat/completions \

@@ -237,7 +237,7 @@ python3 vllm/benchmarks/benchmark_serving.py \

237 **`AI-MO/aimo-validation-aime`**

238

239 ``` bash

240 -python3 vllm/benchmarks/benchmark_serving.py \

240 +vllm bench serve \

241 --model Qwen/QwQ-32B \

242 --dataset-name hf \

243 --dataset-path AI-MO/aimo-validation-aime \

@@ -248,7 +248,7 @@ python3 vllm/benchmarks/benchmark_serving.py \

248 **`philschmid/mt-bench`**

249

250 ``` bash

251 -python3 vllm/benchmarks/benchmark_serving.py \

251 +vllm bench serve \

252 --model Qwen/QwQ-32B \

253 --dataset-name hf \

254 --dataset-path philschmid/mt-bench \

@@ -261,7 +261,7 @@ When using OpenAI-compatible backends such as `vllm`, optional sampling

261 parameters can be specified. Example client command:

262

263 ```bash

264 -python3 vllm/benchmarks/benchmark_serving.py \

264 +vllm bench serve \

265 --backend vllm \

266 --model NousResearch/Hermes-3-Llama-3.1-8B \

267 --endpoint /v1/completions \

@@ -296,7 +296,7 @@ The following arguments can be used to control the ramp-up:

296 <br/>

297

298 ```bash

299 -python3 vllm/benchmarks/benchmark_throughput.py \

299 +vllm bench throughput \

300 --model NousResearch/Hermes-3-Llama-3.1-8B \

301 --dataset-name sonnet \

302 --dataset-path vllm/benchmarks/sonnet.txt \

@@ -314,7 +314,7 @@ Total num output tokens: 1500

314 **VisionArena Benchmark for Vision Language Models**

315

316 ``` bash

317 -python3 vllm/benchmarks/benchmark_throughput.py \

317 +vllm bench throughput \

318 --model Qwen/Qwen2-VL-7B-Instruct \

319 --backend vllm-chat \

320 --dataset-name hf \

@@ -336,7 +336,7 @@ Total num output tokens: 1280

336 ``` bash

337 VLLM_WORKER_MULTIPROC_METHOD=spawn \

338 VLLM_USE_V1=1 \

339 -python3 vllm/benchmarks/benchmark_throughput.py \

339 +vllm bench throughput \

340 --dataset-name=hf \

341 --dataset-path=likaixin/InstructCoder \

342 --model=meta-llama/Meta-Llama-3-8B-Instruct \

@@ -360,7 +360,7 @@ Total num output tokens: 204800

360 **`lmms-lab/LLaVA-OneVision-Data`**

361

362 ```bash

363 -python3 vllm/benchmarks/benchmark_throughput.py \

363 +vllm bench throughput \

364 --model Qwen/Qwen2-VL-7B-Instruct \

365 --backend vllm-chat \

366 --dataset-name hf \

@@ -373,7 +373,7 @@ python3 vllm/benchmarks/benchmark_throughput.py \

373 **`Aeala/ShareGPT_Vicuna_unfiltered`**

374

375 ```bash

376 -python3 vllm/benchmarks/benchmark_throughput.py \

376 +vllm bench throughput \

377 --model Qwen/Qwen2-VL-7B-Instruct \

378 --backend vllm-chat \

379 --dataset-name hf \

@@ -385,7 +385,7 @@ python3 vllm/benchmarks/benchmark_throughput.py \

385 **`AI-MO/aimo-validation-aime`**

386

387 ```bash

388 -python3 benchmarks/benchmark_throughput.py \

388 +vllm bench throughput \

389 --model Qwen/QwQ-32B \

390 --backend vllm \

391 --dataset-name hf \

@@ -399,7 +399,7 @@ python3 benchmarks/benchmark_throughput.py \

399 ``` bash

400 # download dataset

401 # wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json

402 -python3 vllm/benchmarks/benchmark_throughput.py \

402 +vllm bench throughput \

403 --model meta-llama/Llama-2-7b-hf \

404 --backend vllm \

405 --dataset_path <your data path>/ShareGPT_V3_unfiltered_cleaned_split.json \

benchmarks/auto_tune/auto_tune.sh

+9 -9

@@ -1,6 +1,6 @@

1 #!/bin/bash

3 -# This script aims to tune the best server parameter combinations to maximize throughput for given requirement.

3 +# This script aims to tune the best server parameter combinations to maximize throughput for given requirement.

4 # See details in README (benchmarks/auto_tune/README.md).

6 TAG=$(date +"%Y_%m_%d_%H_%M")

@@ -56,7 +56,7 @@ start_server() {

56 local max_num_batched_tokens=$3

57 local vllm_log=$4

58 local profile_dir=$5

59 -

59 +

60 pkill -f vllm

62 VLLM_USE_V1=1 VLLM_SERVER_DEV_MODE=1 VLLM_TORCH_PROFILER_DIR=$profile_dir vllm serve $MODEL \

@@ -73,9 +73,9 @@ start_server() {

74 # wait for 10 minutes...

75 server_started=0

76 - for i in {1..60}; do

76 + for i in {1..60}; do

77 RESPONSE=$(curl -s -X GET "http://0.0.0.0:8004/health" -w "%{http_code}" -o /dev/stdout)

78 - STATUS_CODE=$(echo "$RESPONSE" | tail -n 1)

78 + STATUS_CODE=$(echo "$RESPONSE" | tail -n 1)

79 if [[ "$STATUS_CODE" -eq 200 ]]; then

80 server_started=1

81 break

@@ -98,10 +98,10 @@ update_best_profile() {

98 selected_profile_file=

99 if [[ "$SYSTEM" == "TPU" ]]; then

100 selected_profile_file="${sorted_paths[$profile_index]}/*.xplane.pb"

101 - fi

101 + fi

102 if [[ "$SYSTEM" == "GPU" ]]; then

103 selected_profile_file="${sorted_paths[$profile_index]}"

104 - fi

104 + fi

105 rm -f $PROFILE_PATH/*

106 cp $selected_profile_file $PROFILE_PATH

107 }

@@ -129,14 +129,14 @@ run_benchmark() {

129 echo "server started."

130 fi

131 echo

132 -

132 +

133 echo "run benchmark test..."

134 meet_latency_requirement=0

135 # get a basic qps by using request-rate inf

136 bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_inf.txt"

137 prefix_len=$(( INPUT_LEN * MIN_CACHE_HIT_PCT / 100 ))

138 adjusted_input_len=$(( INPUT_LEN - prefix_len ))

139 - python3 benchmarks/benchmark_serving.py \

139 + vllm bench serve \

140 --backend vllm \

141 --model $MODEL \

142 --dataset-name random \

@@ -169,7 +169,7 @@ adjusted_input_len=$(( INPUT_LEN - prefix_len ))

169 curl -X POST http://0.0.0.0:8004/reset_prefix_cache

170 sleep 5

171 bm_log="$LOG_FOLDER/bm_log_${max_num_seqs}_${max_num_batched_tokens}_requestrate_${request_rate}.txt"

172 - python3 benchmarks/benchmark_serving.py \

172 + vllm bench serve \

173 --backend vllm \

174 --model $MODEL \

175 --dataset-name random \

benchmarks/benchmark_latency.py

@@ -11,6 +11,7 @@ from typing import Any, Optional

12 import numpy as np

13 from tqdm import tqdm

14 +from typing_extensions import deprecated

16 import vllm.envs as envs

17 from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json

@@ -34,6 +35,10 @@ def save_to_pytorch_benchmark_format(

35 write_to_json(pt_file, pt_records)

38 +@deprecated(

39 + "benchmark_latency.py is deprecated and will be removed in a "

40 + "future version. Please use 'vllm bench latency' instead.",

41 +)

42 def main(args: argparse.Namespace):

43 print(args)

benchmarks/benchmark_serving.py

@@ -38,6 +38,7 @@ from typing import Any, Literal, Optional

38 import numpy as np

39 from tqdm.asyncio import tqdm

40 from transformers import PreTrainedTokenizerBase

41 +from typing_extensions import deprecated

43 from backend_request_func import (

44 ASYNC_REQUEST_FUNCS,

@@ -593,6 +594,10 @@ def save_to_pytorch_benchmark_format(

594 write_to_json(pt_file, pt_records)

595

596

597 +@deprecated(

598 + "benchmark_serving.py is deprecated and will be removed in a future "

599 + "version. Please use 'vllm bench serve' instead.",

600 +)

601 def main(args: argparse.Namespace):

602 print(args)

603 random.seed(args.seed)

benchmarks/benchmark_throughput.py

@@ -15,6 +15,7 @@ import torch

15 import uvloop

16 from tqdm import tqdm

17 from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerBase

18 +from typing_extensions import deprecated

20 from benchmark_dataset import (

21 AIMODataset,

@@ -382,6 +383,10 @@ def get_requests(args, tokenizer):

383 return dataset_cls(**common_kwargs).sample(**sample_kwargs)

384

385

386 +@deprecated(

387 + "benchmark_throughput.py is deprecated and will be removed in a "

388 + "future version. Please use 'vllm bench throughput' instead.",

389 +)

390 def main(args: argparse.Namespace):

391 if args.seed is None:

392 args.seed = 0

docs/contributing/profiling.md

+5 -5

@@ -38,7 +38,7 @@ VLLM_TORCH_PROFILER_DIR=./vllm_profile \

38 benchmark_serving.py:

40 ```bash

41 -python benchmarks/benchmark_serving.py \

41 +vllm bench serve \

42 --backend vllm \

43 --model meta-llama/Meta-Llama-3-70B \

44 --dataset-name sharegpt \

@@ -75,7 +75,7 @@ The following is an example using the `benchmarks/benchmark_latency.py` script:

75 nsys profile -o report.nsys-rep \

76 --trace-fork-before-exec=true \

77 --cuda-graph-trace=node \

78 - python benchmarks/benchmark_latency.py \

78 +vllm bench latency \

79 --model meta-llama/Llama-3.1-8B-Instruct \

80 --num-iters-warmup 5 \

81 --num-iters 1 \

@@ -98,7 +98,7 @@ nsys profile -o report.nsys-rep \

98 vllm serve meta-llama/Llama-3.1-8B-Instruct

100 # client

101 -python benchmarks/benchmark_serving.py \

101 +vllm bench serve \

102 --backend vllm \

103 --model meta-llama/Llama-3.1-8B-Instruct \

104 --num-prompts 1 \

@@ -132,7 +132,7 @@ You can view these profiles either as summaries in the CLI, using `nsys stats [p

132 ...

133 ** CUDA GPU Kernel Summary (cuda_gpu_kern_sum):

134

135 - Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name

135 + Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name

136 -------- --------------- --------- ----------- ----------- -------- --------- ----------- ----------------------------------------------------------------------------------------------------

137 46.3 10,327,352,338 17,505 589,965.9 144,383.0 27,040 3,126,460 944,263.8 sm90_xmma_gemm_bf16bf16_bf16f32_f32_tn_n_tilesize128x128x64_warpgroupsize1x1x1_execute_segment_k_of…

138 14.8 3,305,114,764 5,152 641,520.7 293,408.0 287,296 2,822,716 867,124.9 sm90_xmma_gemm_bf16bf16_bf16f32_f32_tn_n_tilesize256x128x64_warpgroupsize2x1x1_execute_segment_k_of…

@@ -143,7 +143,7 @@ You can view these profiles either as summaries in the CLI, using `nsys stats [p

143 2.6 587,283,113 37,824 15,526.7 3,008.0 2,719 2,517,756 139,091.1 std::enable_if<T2>(int)0&&vllm::_typeConvert<T1>::exists, void>::type vllm::fused_add_rms_norm_kern…

144 1.9 418,362,605 18,912 22,121.5 3,871.0 3,328 2,523,870 175,248.2 void vllm::rotary_embedding_kernel<c10::BFloat16, (bool)1>(const long *, T1 *, T1 *, const T1 *, in…

145 0.7 167,083,069 18,880 8,849.7 2,240.0 1,471 2,499,996 101,436.1 void vllm::reshape_and_cache_flash_kernel<__nv_bfloat16, __nv_bfloat16, (vllm::Fp8KVCacheDataType)0…

146 - ...

146 + ...

147 ```

148

149 GUI example:

docs/design/v1/p2p_nccl_connector.md

+9 -9

@@ -3,14 +3,14 @@ An implementation of xPyD with dynamic scaling based on point-to-point communica

3 # Detailed Design

5 ## Overall Process

6 -As shown in Figure 1, the overall process of this **PD disaggregation** solution is described through a request flow:

7 -

8 -1. The client sends an HTTP request to the Proxy/Router's `/v1/completions` interface.

9 -2. The Proxy/Router selects a **1P1D (1 Prefill instance + 1 Decode instance)** through either through round-robin or random selection, generates a `request_id` (rules to be introduced later), modifies the `max_tokens` in the HTTP request message to **1**, and then forwards the request to the **P instance**.

10 -3. Immediately afterward, the Proxy/Router forwards the **original HTTP request** to the **D instance**.

11 -4. The **P instance** performs **Prefill** and then **actively sends the generated KV cache** to the D instance (using **PUT_ASYNC** mode). The D instance's `zmq_addr` can be resolved through the `request_id`.

12 -5. The **D instance** has a **dedicated thread** for receiving the KV cache (to avoid blocking the main process). The received KV cache is saved into the **GPU memory buffer**, the size of which is determined by the vLLM startup parameter `kv_buffer_size`. When the GPU buffer is full, the KV cache is stored in the **local Tensor memory pool**.

13 -6. During the **Decode**, the D instance's main process retrieves the KV cache (transmitted by the P instance) from either the **GPU buffer** or the **memory pool**, thereby **skipping Prefill**.

6 +As shown in Figure 1, the overall process of this **PD disaggregation** solution is described through a request flow:

7 +

8 +1. The client sends an HTTP request to the Proxy/Router's `/v1/completions` interface.

9 +2. The Proxy/Router selects a **1P1D (1 Prefill instance + 1 Decode instance)** through either through round-robin or random selection, generates a `request_id` (rules to be introduced later), modifies the `max_tokens` in the HTTP request message to **1**, and then forwards the request to the **P instance**.

10 +3. Immediately afterward, the Proxy/Router forwards the **original HTTP request** to the **D instance**.

11 +4. The **P instance** performs **Prefill** and then **actively sends the generated KV cache** to the D instance (using **PUT_ASYNC** mode). The D instance's `zmq_addr` can be resolved through the `request_id`.

12 +5. The **D instance** has a **dedicated thread** for receiving the KV cache (to avoid blocking the main process). The received KV cache is saved into the **GPU memory buffer**, the size of which is determined by the vLLM startup parameter `kv_buffer_size`. When the GPU buffer is full, the KV cache is stored in the **local Tensor memory pool**.

13 +6. During the **Decode**, the D instance's main process retrieves the KV cache (transmitted by the P instance) from either the **GPU buffer** or the **memory pool**, thereby **skipping Prefill**.

14 7. After completing **Decode**, the D instance returns the result to the **Proxy/Router**, which then forwards it to the **client**.

16 ![image1](https://github.com/user-attachments/assets/fb01bde6-755b-49f7-ad45-48a94b1e10a7)

@@ -291,7 +291,7 @@ curl -X POST -s http://10.0.1.1:10001/v1/completions \

291 ??? console "Command"

292

293 ```shell

294 - python3 benchmark_serving.py \

294 + vllm bench serve \

295 --backend vllm \

296 --model base_model \

297 --tokenizer meta-llama/Llama-3.1-8B-Instruct \

examples/online_serving/disaggregated_serving_p2p_nccl_xpyd/disagg_example_p2p_nccl_xpyd.sh

+5 -5

@@ -29,7 +29,7 @@ PROXY_PORT=${PROXY_PORT:-30001}

29 PREFILL_GPUS=${PREFILL_GPUS:-0}

30 DECODE_GPUS=${DECODE_GPUS:-1,2,3}

31 PREFILL_PORTS=${PREFILL_PORTS:-20003}

32 -DECODE_PORTS=${DECODE_PORTS:-20005,20007,20009}

32 +DECODE_PORTS=${DECODE_PORTS:-20005,20007,20009}

34 echo "Warning: P2P NCCL disaggregated prefill XpYd support for vLLM v1 is experimental and subject to change."

35 echo ""

@@ -164,7 +164,7 @@ main() {

164 local gpu_id=${PREFILL_GPU_ARRAY[$i]}

165 local port=${PREFILL_PORT_ARRAY[$i]}

166 local kv_port=$((21001 + i))

167 -

167 +

168 echo " Prefill server $((i+1)): GPU $gpu_id, Port $port, KV Port $kv_port"

169 CUDA_VISIBLE_DEVICES=$gpu_id VLLM_USE_V1=1 vllm serve $MODEL \

170 --enforce-eager \

@@ -193,7 +193,7 @@ main() {

193 local gpu_id=${DECODE_GPU_ARRAY[$i]}

194 local port=${DECODE_PORT_ARRAY[$i]}

195 local kv_port=$((22001 + i))

196 -

196 +

197 echo " Decode server $((i+1)): GPU $gpu_id, Port $port, KV Port $kv_port"

198 VLLM_USE_V1=1 CUDA_VISIBLE_DEVICES=$gpu_id vllm serve $MODEL \

199 --enforce-eager \

@@ -233,7 +233,7 @@ main() {

233 # Run Benchmark

234 # =============================================================================

235 cd ../../../benchmarks/

236 - python3 benchmark_serving.py --port 10001 --seed $(date +%s) \

236 + vllm bench serve --port 10001 --seed $(date +%s) \

237 --model $MODEL \

238 --dataset-name random --random-input-len 7500 --random-output-len 200 \

239 --num-prompts 200 --burstiness 100 --request-rate 2 | tee benchmark.log

@@ -243,4 +243,4 @@ main() {

243 cleanup

244 }

245

246 -main

\ No newline at end of file

246 +main

examples/others/lmcache/disagg_prefill_lmcache_v1/disagg_example_nixl.sh

+2 -2

@@ -122,7 +122,7 @@ main() {

122

123 # begin benchmark

124 cd ../../../../benchmarks/

125 - python3 benchmark_serving.py --port 9000 --seed $(date +%s) \

125 + vllm bench serve --port 9000 --seed $(date +%s) \

126 --model meta-llama/Llama-3.1-8B-Instruct \

127 --dataset-name random --random-input-len 7500 --random-output-len 200 \

128 --num-prompts 200 --burstiness 100 --request-rate 3.6 | tee benchmark.log

@@ -133,4 +133,4 @@ main() {

133

134 }

135

136 -main

\ No newline at end of file

136 +main