@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 697f507a8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 00:30 UTC

[CI/Build][Intel] Enable performance benchmarks for Intel Gaudi 3 (#26919)

Signed-off-by: jakub-sochacki <jakub.sochacki@wp.pl>

Jakub Sochacki committed Oct 31, 2025 at 00:57 UTC 697f507a8ebb13d74e8c0695aec05d9baefb45a0
6 files changed +216 -4
.buildkite/performance-benchmarks/README.md
+2 -1
@@ -7,7 +7,7 @@ vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](http
7
8 ## Performance benchmark quick overview
9
10 -**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100 and Intel® Xeon® Processors, with different models.
10 +**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100, Intel® Xeon® Processors and Intel® Gaudi® 3 Accelerators with different models.
11
12 **Benchmarking Duration**: about 1hr.
13
@@ -34,6 +34,7 @@ Runtime environment variables:
34
35 See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
36 > NOTE: For Intel® Xeon® Processors, use `tests/latency-tests-cpu.json`, `tests/throughput-tests-cpu.json`, `tests/serving-tests-cpu.json` instead.
37 +For Intel® Gaudi® 3 Accelerators, use `tests/latency-tests-hpu.json`, `tests/throughput-tests-hpu.json`, `tests/serving-tests-hpu.json` instead.
38 >
39 ### Latency test
40
.buildkite/performance-benchmarks/performance-benchmarks-descriptions.md
+3 -3
@@ -5,7 +5,7 @@
5 - Input length: 32 tokens.
6 - Output length: 128 tokens.
7 - Batch size: fixed (8).
8 -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
8 +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
9 - CPU Models: llama-3.1 8B.
10 - Evaluation metrics: end-to-end latency (mean, median, p99).
11
@@ -16,7 +16,7 @@
16 - Input length: randomly sample 200 prompts from ShareGPT dataset (with fixed random seed).
17 - Output length: the corresponding output length of these 200 prompts.
18 - Batch size: dynamically determined by vllm to achieve maximum throughput.
19 -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
19 +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
20 - CPU Models: llama-3.1 8B.
21 - Evaluation metrics: throughput.
22
@@ -28,7 +28,7 @@
28 - Output length: the corresponding output length of these 200 prompts.
29 - Batch size: dynamically determined by vllm and the arrival pattern of the requests.
30 - **Average QPS (query per second)**: 1, 4, 16 and inf. QPS = inf means all requests come at once. For other QPS values, the arrival time of each query is determined using a random Poisson process (with fixed random seed).
31 -- GPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
31 +- GPU/HPU Models: llama-3.1 8B, llama-3 70B, mixtral 8x7B.
32 - We also added a speculative decoding test for llama-3 70B on GPU, under QPS 2
33 - CPU Models: llama-3.1 8B.
34 - Evaluation metrics: throughput, TTFT (time to the first token, with mean, median and p99), ITL (inter-token latency, with mean, median and p99).
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+13
@@ -15,6 +15,8 @@ check_gpus() {
15 declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
16 elif command -v amd-smi; then
17 declare -g gpu_count=$(amd-smi list | grep 'GPU' | wc -l)
18 + elif command -v hl-smi; then
19 + declare -g gpu_count=$(hl-smi --list | grep -i "Module ID" | wc -l)
20 fi
21
22 if [[ $gpu_count -gt 0 ]]; then
@@ -23,10 +25,16 @@ check_gpus() {
25 echo "Need at least 1 GPU to run benchmarking."
26 exit 1
27 fi
28 +
29 + declare -g arch_suffix=''
30 +
31 if command -v nvidia-smi; then
32 declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
33 elif command -v amd-smi; then
34 declare -g gpu_type=$(amd-smi static -g 0 -a | grep 'MARKET_NAME' | awk '{print $2}')
35 + elif command -v hl-smi; then
36 + declare -g gpu_type=$(hl-smi -q | grep "Product Name" | head -n 1 | awk -F ':' '{print $2}' | sed 's/^ *//')
37 + arch_suffix='-hpu'
38 fi
39 echo "GPU type is $gpu_type"
40 }
@@ -138,6 +146,10 @@ kill_gpu_processes() {
146 while [ "$(amd-smi metric -g 0 | grep 'USED_VRAM' | awk '{print $2}')" -ge 1000 ]; do
147 sleep 1
148 done
149 + elif command -v hl-smi; then
150 + while [ "$(hl-smi -q | grep "Used" | head -n 1 | awk '{print $3}')" -ge 1000 ]; do
151 + sleep 1
152 + done
153 fi
154
155 # remove vllm config file
@@ -451,6 +463,7 @@ main() {
463 ARCH='-cpu'
464 else
465 check_gpus
466 + ARCH="$arch_suffix"
467 fi
468 check_hf_token
469
.buildkite/performance-benchmarks/tests/latency-tests-hpu.json new
+55
@@ -0,0 +1,55 @@
1 +[
2 + {
3 + "test_name": "latency_llama8B_tp1",
4 + "environment_variables": {
5 + "PT_HPU_LAZY_MODE": 1,
6 + "VLLM_CONTIGUOUS_PA": 1,
7 + "VLLM_DEFRAG": 1
8 + },
9 + "parameters": {
10 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
11 + "tensor_parallel_size": 1,
12 + "load_format": "dummy",
13 + "num-iters-warmup": 5,
14 + "num-iters": 15,
15 + "max-model-len": 256,
16 + "async-scheduling": ""
17 + }
18 + },
19 + {
20 + "test_name": "latency_llama70B_tp4",
21 + "environment_variables": {
22 + "PT_HPU_LAZY_MODE": 1,
23 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
24 + "VLLM_CONTIGUOUS_PA": 1,
25 + "VLLM_DEFRAG": 1
26 + },
27 + "parameters": {
28 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
29 + "tensor_parallel_size": 4,
30 + "load_format": "dummy",
31 + "num-iters-warmup": 5,
32 + "num-iters": 15,
33 + "max-model-len": 256,
34 + "async-scheduling": ""
35 + }
36 + },
37 + {
38 + "test_name": "latency_mixtral8x7B_tp2",
39 + "environment_variables": {
40 + "PT_HPU_LAZY_MODE": 1,
41 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
42 + "VLLM_CONTIGUOUS_PA": 1,
43 + "VLLM_DEFRAG": 1
44 + },
45 + "parameters": {
46 + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
47 + "tensor_parallel_size": 2,
48 + "load_format": "dummy",
49 + "num-iters-warmup": 5,
50 + "num-iters": 15,
51 + "max-model-len": 256,
52 + "async-scheduling": ""
53 + }
54 + }
55 +]
.buildkite/performance-benchmarks/tests/serving-tests-hpu.json new
+82
@@ -0,0 +1,82 @@
1 +[
2 + {
3 + "test_name": "serving_llama8B_tp1_sharegpt",
4 + "qps_list": [1, 4, 16, "inf"],
5 + "server_environment_variables": {
6 + "PT_HPU_LAZY_MODE": 1,
7 + "VLLM_CONTIGUOUS_PA": 1,
8 + "VLLM_DEFRAG": 1
9 + },
10 + "server_parameters": {
11 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
12 + "tensor_parallel_size": 1,
13 + "swap_space": 16,
14 + "disable_log_stats": "",
15 + "load_format": "dummy",
16 + "max-model-len": 2048,
17 + "max-num-seqs": 256,
18 + "async-scheduling": ""
19 + },
20 + "client_parameters": {
21 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
22 + "backend": "vllm",
23 + "dataset_name": "sharegpt",
24 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
25 + "num_prompts": 200
26 + }
27 + },
28 + {
29 + "test_name": "serving_llama70B_tp4_sharegpt",
30 + "qps_list": [1, 4, 16, "inf"],
31 + "server_environment_variables": {
32 + "PT_HPU_LAZY_MODE": 1,
33 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
34 + "VLLM_CONTIGUOUS_PA": 1,
35 + "VLLM_DEFRAG": 1
36 + },
37 + "server_parameters": {
38 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
39 + "tensor_parallel_size": 4,
40 + "swap_space": 16,
41 + "disable_log_stats": "",
42 + "load_format": "dummy",
43 + "max-model-len": 2048,
44 + "max-num-seqs": 256,
45 + "async-scheduling": ""
46 + },
47 + "client_parameters": {
48 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
49 + "backend": "vllm",
50 + "dataset_name": "sharegpt",
51 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
52 + "num_prompts": 200
53 + }
54 + },
55 + {
56 + "test_name": "serving_mixtral8x7B_tp2_sharegpt",
57 + "qps_list": [1, 4, 16, "inf"],
58 + "server_environment_variables": {
59 + "PT_HPU_LAZY_MODE": 1,
60 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
61 + "VLLM_CONTIGUOUS_PA": 1,
62 + "VLLM_DEFRAG": 1
63 + },
64 + "server_parameters": {
65 + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
66 + "tensor_parallel_size": 2,
67 + "swap_space": 16,
68 + "disable_log_stats": "",
69 + "load_format": "dummy",
70 + "max-model-len": 2048,
71 + "max-num-seqs": 256,
72 + "async-scheduling": ""
73 + },
74 + "client_parameters": {
75 + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
76 + "backend": "vllm",
77 + "dataset_name": "sharegpt",
78 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
79 + "num_prompts": 200
80 + }
81 + }
82 +]
.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json new
+61
@@ -0,0 +1,61 @@
1 +[
2 + {
3 + "test_name": "throughput_llama8B_tp1",
4 + "environment_variables": {
5 + "PT_HPU_LAZY_MODE": 1,
6 + "VLLM_CONTIGUOUS_PA": 1,
7 + "VLLM_DEFRAG": 1
8 + },
9 + "parameters": {
10 + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
11 + "tensor_parallel_size": 1,
12 + "load_format": "dummy",
13 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
14 + "num_prompts": 1000,
15 + "backend": "vllm",
16 + "max-model-len": 2048,
17 + "max-num-seqs": 512,
18 + "async-scheduling": ""
19 + }
20 + },
21 + {
22 + "test_name": "throughput_llama70B_tp4",
23 + "environment_variables": {
24 + "PT_HPU_LAZY_MODE": 1,
25 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
26 + "VLLM_CONTIGUOUS_PA": 1,
27 + "VLLM_DEFRAG": 1
28 + },
29 + "parameters": {
30 + "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
31 + "tensor_parallel_size": 4,
32 + "load_format": "dummy",
33 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
34 + "num_prompts": 1000,
35 + "backend": "vllm",
36 + "max-model-len": 2048,
37 + "max-num-seqs": 512,
38 + "async-scheduling": ""
39 + }
40 + },
41 + {
42 + "test_name": "throughput_mixtral8x7B_tp2",
43 + "environment_variables": {
44 + "PT_HPU_LAZY_MODE": 1,
45 + "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
46 + "VLLM_CONTIGUOUS_PA": 1,
47 + "VLLM_DEFRAG": 1
48 + },
49 + "parameters": {
50 + "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
51 + "tensor_parallel_size": 2,
52 + "load_format": "dummy",
53 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
54 + "num_prompts": 1000,
55 + "backend": "vllm",
56 + "max-model-len": 2048,
57 + "max-num-seqs": 512,
58 + "async-scheduling": ""
59 + }
60 + }
61 +]