A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
Update vLLM Benchmark Suite for Xeon based on 0.9.2 release (#21486)
Signed-off-by: Tsai, Louie <louie.tsai@intel.com>
Louie Tsai committed
Jul 29, 2025 at 22:57 UTC
6f8d26188200385fa994526b10b5858a3da1ede7
5 files changed
+437
-1
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+1
@@ -44,6 +44,7 @@ serving_column_mapping = {
44
"test_name": "Test name",
45
"gpu_type": "GPU",
46
"completed": "# of req.",
47
+ "max_concurrency": "# of max concurrency.",
48
"request_throughput": "Tput (req/s)",
49
"total_token_throughput": "Total Token Tput (tok/s)",
50
"output_throughput": "Output Tput (tok/s)",
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+1
-1
@@ -33,7 +33,7 @@ check_gpus() {
33
34
check_cpus() {
35
# check the number of CPUs and NUMA Node and GPU type.
36
- declare -g numa_count=$(python3 -c "from numa import info;numa_size = info.get_num_configured_nodes(); print(numa_size)")
36
+ declare -g numa_count=$(lscpu | grep "NUMA node(s):" | awk '{print $3}')
37
if [[ $numa_count -gt 0 ]]; then
38
echo "NUMA found."
39
echo $numa_count
.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc2.json
new
+209
@@ -0,0 +1,209 @@
1
+[
2
+ {
3
+ "test_name": "serving_llama8B_tp1_sharegpt",
4
+ "qps_list": [1, 4, 16, "inf"],
5
+ "server_environment_variables": {
6
+ "VLLM_RPC_TIMEOUT": 100000,
7
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
8
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
9
+ "VLLM_CPU_SGL_KERNEL": 1,
10
+ "VLLM_CPU_KVCACHE_SPACE": 40
11
+ },
12
+ "server_parameters": {
13
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
14
+ "tensor_parallel_size": 1,
15
+ "dtype": "bfloat16",
16
+ "distributed_executor_backend": "mp",
17
+ "block_size": 128,
18
+ "trust_remote_code": "",
19
+ "disable_log_stats": "",
20
+ "disable_log_requests": "",
21
+ "enforce_eager": "",
22
+ "max_num_batched_tokens": 2048,
23
+ "max_num_seqs": 256,
24
+ "load_format": "dummy"
25
+ },
26
+ "client_parameters": {
27
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
28
+ "backend": "vllm",
29
+ "dataset_name": "sharegpt",
30
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
31
+ "max_concurrency": 60,
32
+ "num_prompts": 200
33
+ }
34
+ },
35
+ {
36
+ "test_name": "serving_llama8B_tp2_sharegpt",
37
+ "qps_list": [1, 4, 16, "inf"],
38
+ "server_environment_variables": {
39
+ "VLLM_RPC_TIMEOUT": 100000,
40
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
41
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
42
+ "VLLM_CPU_SGL_KERNEL": 1,
43
+ "VLLM_CPU_KVCACHE_SPACE": 40
44
+ },
45
+ "server_parameters": {
46
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
47
+ "tensor_parallel_size": 2,
48
+ "dtype": "bfloat16",
49
+ "distributed_executor_backend": "mp",
50
+ "block_size": 128,
51
+ "trust_remote_code": "",
52
+ "disable_log_stats": "",
53
+ "disable_log_requests": "",
54
+ "enforce_eager": "",
55
+ "max_num_batched_tokens": 2048,
56
+ "max_num_seqs": 256,
57
+ "load_format": "dummy"
58
+ },
59
+ "client_parameters": {
60
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
61
+ "backend": "vllm",
62
+ "dataset_name": "sharegpt",
63
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
64
+ "max_concurrency": 60,
65
+ "num_prompts": 200
66
+ }
67
+ },
68
+ {
69
+ "test_name": "serving_llama8B_tp4_sharegpt",
70
+ "qps_list": [1, 4, 16, "inf"],
71
+ "server_environment_variables": {
72
+ "VLLM_RPC_TIMEOUT": 100000,
73
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
74
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
75
+ "VLLM_CPU_SGL_KERNEL": 1,
76
+ "VLLM_CPU_KVCACHE_SPACE": 40
77
+ },
78
+ "server_parameters": {
79
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
80
+ "tensor_parallel_size": 4,
81
+ "dtype": "bfloat16",
82
+ "distributed_executor_backend": "mp",
83
+ "block_size": 128,
84
+ "trust_remote_code": "",
85
+ "disable_log_stats": "",
86
+ "disable_log_requests": "",
87
+ "enforce_eager": "",
88
+ "max_num_batched_tokens": 2048,
89
+ "max_num_seqs": 256,
90
+ "load_format": "dummy"
91
+ },
92
+ "client_parameters": {
93
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
94
+ "backend": "vllm",
95
+ "dataset_name": "sharegpt",
96
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
97
+ "max_concurrency": 60,
98
+ "num_prompts": 200
99
+ }
100
+ },
101
+ {
102
+ "test_name": "serving_llama8B_tp1_random_128_128",
103
+ "qps_list": [1, 4, 16, "inf"],
104
+ "server_environment_variables": {
105
+ "VLLM_RPC_TIMEOUT": 100000,
106
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
107
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
108
+ "VLLM_CPU_SGL_KERNEL": 1,
109
+ "VLLM_CPU_KVCACHE_SPACE": 40
110
+ },
111
+ "server_parameters": {
112
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
113
+ "tensor_parallel_size": 1,
114
+ "dtype": "bfloat16",
115
+ "distributed_executor_backend": "mp",
116
+ "block_size": 128,
117
+ "trust_remote_code": "",
118
+ "enable_chunked_prefill": "",
119
+ "disable_log_stats": "",
120
+ "disable_log_requests": "",
121
+ "enforce_eager": "",
122
+ "max_num_batched_tokens": 2048,
123
+ "max_num_seqs": 256,
124
+ "load_format": "dummy"
125
+ },
126
+ "client_parameters": {
127
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
128
+ "backend": "vllm",
129
+ "dataset_name": "random",
130
+ "random-input-len": 128,
131
+ "random-output-len": 128,
132
+ "ignore-eos": "",
133
+ "max_concurrency": 1000,
134
+ "num_prompts": 1000
135
+ }
136
+ },
137
+ {
138
+ "test_name": "serving_llama8B_tp2_random_128_128",
139
+ "qps_list": [1, 4, 16, "inf"],
140
+ "server_environment_variables": {
141
+ "VLLM_RPC_TIMEOUT": 100000,
142
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
143
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
144
+ "VLLM_CPU_SGL_KERNEL": 1,
145
+ "VLLM_CPU_KVCACHE_SPACE": 40
146
+ },
147
+ "server_parameters": {
148
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
149
+ "tensor_parallel_size": 2,
150
+ "dtype": "bfloat16",
151
+ "distributed_executor_backend": "mp",
152
+ "block_size": 128,
153
+ "trust_remote_code": "",
154
+ "enable_chunked_prefill": "",
155
+ "disable_log_stats": "",
156
+ "disable_log_requests": "",
157
+ "enforce_eager": "",
158
+ "max_num_batched_tokens": 2048,
159
+ "max_num_seqs": 256,
160
+ "load_format": "dummy"
161
+ },
162
+ "client_parameters": {
163
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
164
+ "backend": "vllm",
165
+ "dataset_name": "random",
166
+ "random-input-len": 128,
167
+ "random-output-len": 128,
168
+ "ignore-eos": "",
169
+ "max_concurrency": 1000,
170
+ "num_prompts": 1000
171
+ }
172
+ },
173
+ {
174
+ "test_name": "serving_llama8B_tp4_random_128_128",
175
+ "qps_list": [1, 4, 16, "inf"],
176
+ "server_environment_variables": {
177
+ "VLLM_RPC_TIMEOUT": 100000,
178
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
179
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
180
+ "VLLM_CPU_SGL_KERNEL": 1,
181
+ "VLLM_CPU_KVCACHE_SPACE": 40
182
+ },
183
+ "server_parameters": {
184
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
185
+ "tensor_parallel_size": 4,
186
+ "dtype": "bfloat16",
187
+ "distributed_executor_backend": "mp",
188
+ "block_size": 128,
189
+ "trust_remote_code": "",
190
+ "enable_chunked_prefill": "",
191
+ "disable_log_stats": "",
192
+ "disable_log_requests": "",
193
+ "enforce_eager": "",
194
+ "max_num_batched_tokens": 2048,
195
+ "max_num_seqs": 256,
196
+ "load_format": "dummy"
197
+ },
198
+ "client_parameters": {
199
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
200
+ "backend": "vllm",
201
+ "dataset_name": "random",
202
+ "random-input-len": 128,
203
+ "random-output-len": 128,
204
+ "ignore-eos": "",
205
+ "max_concurrency": 1000,
206
+ "num_prompts": 1000
207
+ }
208
+ }
209
+]
.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc3.json
new
+211
@@ -0,0 +1,211 @@
1
+[
2
+ {
3
+ "test_name": "serving_llama8B_pp1_sharegpt",
4
+ "qps_list": [1, 4, 16, "inf"],
5
+ "server_environment_variables": {
6
+ "VLLM_RPC_TIMEOUT": 100000,
7
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
8
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
9
+ "VLLM_CPU_SGL_KERNEL": 1,
10
+ "VLLM_CPU_KVCACHE_SPACE": 40
11
+ },
12
+ "server_parameters": {
13
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
14
+ "pipeline_parallel_size": 1,
15
+ "dtype": "bfloat16",
16
+ "distributed_executor_backend": "mp",
17
+ "block_size": 128,
18
+ "trust_remote_code": "",
19
+ "disable_log_stats": "",
20
+ "disable_log_requests": "",
21
+ "enforce_eager": "",
22
+ "max_num_batched_tokens": 2048,
23
+ "max_num_seqs": 256,
24
+ "load_format": "dummy"
25
+ },
26
+ "client_parameters": {
27
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
28
+ "backend": "vllm",
29
+ "dataset_name": "sharegpt",
30
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
31
+ "max_concurrency": 60,
32
+ "num_prompts": 200
33
+ }
34
+ },
35
+ {
36
+ "test_name": "serving_llama8B_pp3_sharegpt",
37
+ "qps_list": [1, 4, 16, "inf"],
38
+ "server_environment_variables": {
39
+ "VLLM_RPC_TIMEOUT": 100000,
40
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
41
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
42
+ "VLLM_CPU_SGL_KERNEL": 1,
43
+ "VLLM_CPU_KVCACHE_SPACE": 40
44
+ },
45
+ "server_parameters": {
46
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
47
+ "pipeline_parallel_size": 3,
48
+ "dtype": "bfloat16",
49
+ "distributed_executor_backend": "mp",
50
+ "block_size": 128,
51
+ "trust_remote_code": "",
52
+ "disable_log_stats": "",
53
+ "disable_log_requests": "",
54
+ "enforce_eager": "",
55
+ "max_num_batched_tokens": 2048,
56
+ "max_num_seqs": 256,
57
+ "load_format": "dummy"
58
+ },
59
+ "client_parameters": {
60
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
61
+ "backend": "vllm",
62
+ "dataset_name": "sharegpt",
63
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
64
+ "max_concurrency": 60,
65
+ "num_prompts": 200
66
+ }
67
+ },
68
+ {
69
+ "test_name": "serving_llama8B_tp2pp6_sharegpt",
70
+ "qps_list": [1, 4, 16, "inf"],
71
+ "server_environment_variables": {
72
+ "VLLM_RPC_TIMEOUT": 100000,
73
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
74
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
75
+ "VLLM_CPU_SGL_KERNEL": 1,
76
+ "VLLM_CPU_KVCACHE_SPACE": 40
77
+ },
78
+ "server_parameters": {
79
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
80
+ "tensor_parallel_size": 2,
81
+ "pipeline_parallel_size": 3,
82
+ "dtype": "bfloat16",
83
+ "distributed_executor_backend": "mp",
84
+ "block_size": 128,
85
+ "trust_remote_code": "",
86
+ "disable_log_stats": "",
87
+ "disable_log_requests": "",
88
+ "enforce_eager": "",
89
+ "max_num_batched_tokens": 2048,
90
+ "max_num_seqs": 256,
91
+ "load_format": "dummy"
92
+ },
93
+ "client_parameters": {
94
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
95
+ "backend": "vllm",
96
+ "dataset_name": "sharegpt",
97
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
98
+ "max_concurrency": 60,
99
+ "num_prompts": 200
100
+ }
101
+ },
102
+ {
103
+ "test_name": "serving_llama8B_pp1_random_128_128",
104
+ "qps_list": [1, 4, 16, "inf"],
105
+ "server_environment_variables": {
106
+ "VLLM_RPC_TIMEOUT": 100000,
107
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
108
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
109
+ "VLLM_CPU_SGL_KERNEL": 1,
110
+ "VLLM_CPU_KVCACHE_SPACE": 40
111
+ },
112
+ "server_parameters": {
113
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
114
+ "pipeline_parallel_size": 1,
115
+ "dtype": "bfloat16",
116
+ "distributed_executor_backend": "mp",
117
+ "block_size": 128,
118
+ "trust_remote_code": "",
119
+ "enable_chunked_prefill": "",
120
+ "disable_log_stats": "",
121
+ "disable_log_requests": "",
122
+ "enforce_eager": "",
123
+ "max_num_batched_tokens": 2048,
124
+ "max_num_seqs": 256,
125
+ "load_format": "dummy"
126
+ },
127
+ "client_parameters": {
128
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
129
+ "backend": "vllm",
130
+ "dataset_name": "random",
131
+ "random-input-len": 128,
132
+ "random-output-len": 128,
133
+ "ignore-eos": "",
134
+ "max_concurrency": 1000,
135
+ "num_prompts": 1000
136
+ }
137
+ },
138
+ {
139
+ "test_name": "serving_llama8B_pp3_random_128_128",
140
+ "qps_list": [1, 4, 16, "inf"],
141
+ "server_environment_variables": {
142
+ "VLLM_RPC_TIMEOUT": 100000,
143
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
144
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
145
+ "VLLM_CPU_SGL_KERNEL:": 1,
146
+ "VLLM_CPU_KVCACHE_SPACE": 40
147
+ },
148
+ "server_parameters": {
149
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
150
+ "pipeline_parallel_size": 3,
151
+ "dtype": "bfloat16",
152
+ "distributed_executor_backend": "mp",
153
+ "block_size": 128,
154
+ "trust_remote_code": "",
155
+ "enable_chunked_prefill": "",
156
+ "disable_log_stats": "",
157
+ "disable_log_requests": "",
158
+ "enforce_eager": "",
159
+ "max_num_batched_tokens": 2048,
160
+ "max_num_seqs": 256,
161
+ "load_format": "dummy"
162
+ },
163
+ "client_parameters": {
164
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
165
+ "backend": "vllm",
166
+ "dataset_name": "random",
167
+ "random-input-len": 128,
168
+ "random-output-len": 128,
169
+ "ignore-eos": "",
170
+ "max_concurrency": 1000,
171
+ "num_prompts": 1000
172
+ }
173
+ },
174
+ {
175
+ "test_name": "serving_llama8B_tp2pp3_random_128_128",
176
+ "qps_list": [1, 4, 16, "inf"],
177
+ "server_environment_variables": {
178
+ "VLLM_RPC_TIMEOUT": 100000,
179
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
180
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
181
+ "VLLM_CPU_SGL_KERNEL": 1,
182
+ "VLLM_CPU_KVCACHE_SPACE": 40
183
+ },
184
+ "server_parameters": {
185
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
186
+ "tensor_parallel_size": 2,
187
+ "pipeline_parallel_size": 3,
188
+ "dtype": "bfloat16",
189
+ "distributed_executor_backend": "mp",
190
+ "block_size": 128,
191
+ "trust_remote_code": "",
192
+ "enable_chunked_prefill": "",
193
+ "disable_log_stats": "",
194
+ "disable_log_requests": "",
195
+ "enforce_eager": "",
196
+ "max_num_batched_tokens": 2048,
197
+ "max_num_seqs": 256,
198
+ "load_format": "dummy"
199
+ },
200
+ "client_parameters": {
201
+ "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
202
+ "backend": "vllm",
203
+ "dataset_name": "random",
204
+ "random-input-len": 128,
205
+ "random-output-len": 128,
206
+ "ignore-eos": "",
207
+ "max_concurrency": 1000,
208
+ "num_prompts": 1000
209
+ }
210
+ }
211
+]
.buildkite/nightly-benchmarks/tests/serving-tests-cpu.json
+15
@@ -6,6 +6,7 @@
6
"VLLM_RPC_TIMEOUT": 100000,
7
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
8
"VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
9
+ "VLLM_CPU_SGL_KERNEL": 1,
10
"VLLM_CPU_KVCACHE_SPACE": 40
11
},
12
"server_parameters": {
@@ -18,6 +19,8 @@
19
"disable_log_stats": "",
20
"disable_log_requests": "",
21
"enforce_eager": "",
22
+ "max_num_batched_tokens": 2048,
23
+ "max_num_seqs": 256,
24
"load_format": "dummy"
25
},
26
"client_parameters": {
@@ -36,6 +39,7 @@
39
"VLLM_RPC_TIMEOUT": 100000,
40
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
41
"VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
42
+ "VLLM_CPU_SGL_KERNEL": 1,
43
"VLLM_CPU_KVCACHE_SPACE": 40
44
},
45
"server_parameters": {
@@ -48,6 +52,8 @@
52
"disable_log_stats": "",
53
"disable_log_requests": "",
54
"enforce_eager": "",
55
+ "max_num_batched_tokens": 2048,
56
+ "max_num_seqs": 256,
57
"load_format": "dummy"
58
},
59
"client_parameters": {
@@ -66,6 +72,7 @@
72
"VLLM_RPC_TIMEOUT": 100000,
73
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
74
"VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
75
+ "VLLM_CPU_SGL_KERNEL": 1,
76
"VLLM_CPU_KVCACHE_SPACE": 40
77
},
78
"server_parameters": {
@@ -78,6 +85,8 @@
85
"disable_log_stats": "",
86
"disable_log_requests": "",
87
"enforce_eager": "",
88
+ "max_num_batched_tokens": 2048,
89
+ "max_num_seqs": 256,
90
"load_format": "dummy"
91
},
92
"client_parameters": {
@@ -96,6 +105,7 @@
105
"VLLM_RPC_TIMEOUT": 100000,
106
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
107
"VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
108
+ "VLLM_CPU_SGL_KERNEL": 1,
109
"VLLM_CPU_KVCACHE_SPACE": 40
110
},
111
"server_parameters": {
@@ -109,6 +119,8 @@
119
"disable_log_stats": "",
120
"disable_log_requests": "",
121
"enforce_eager": "",
122
+ "max_num_batched_tokens": 2048,
123
+ "max_num_seqs": 256,
124
"load_format": "dummy"
125
},
126
"client_parameters": {
@@ -129,6 +141,7 @@
141
"VLLM_RPC_TIMEOUT": 100000,
142
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
143
"VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
144
+ "VLLM_CPU_SGL_KERNEL": 1,
145
"VLLM_CPU_KVCACHE_SPACE": 40
146
},
147
"server_parameters": {
@@ -142,6 +155,8 @@
155
"disable_log_stats": "",
156
"disable_log_requests": "",
157
"enforce_eager": "",
158
+ "max_num_batched_tokens": 2048,
159
+ "max_num_seqs": 256,
160
"load_format": "dummy"
161
},
162
"client_parameters": {