A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 15:15 UTC
Update Intel Xeon model list and vLLM Benchmark Suite BKMs (#42607)
Signed-off-by: louie-tsai <louie.tsai@intel.com>
Louie Tsai committed
May 14, 2026 at 22:14 UTC
e30f39c4f13171b8d8047f1c5a4f071d63ec195e
2 files changed
+118
-159
.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json
+76
-143
@@ -31,30 +31,9 @@
31
}
32
},
33
"tests": [
34
- {
35
- "test_name": "serving_llama8B_tp1_sharegpt",
36
- "server_parameters": {
37
- "tensor_parallel_size": 1
38
- },
39
- "client_parameters": {
40
- "dataset_name": "sharegpt",
41
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
42
- }
43
- },
44
- {
45
- "test_name": "serving_llama8B_tp2_sharegpt",
46
- "server_parameters": {
47
- "tensor_parallel_size": 2
48
- },
49
- "client_parameters": {
50
- "dataset_name": "sharegpt",
51
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
52
- }
53
- },
34
{
35
"test_name": "serving_llama8B_tp1_random_128_128",
36
"server_parameters": {
57
- "tensor_parallel_size": 1
37
},
38
"client_parameters": {
39
"dataset_name": "random",
@@ -63,290 +42,244 @@
42
}
43
},
44
{
66
- "test_name": "serving_llama8B_tp2_random_128_128",
45
+ "test_name": "serving_llama8B_int4_tp1_random_128_128",
46
"server_parameters": {
68
- "tensor_parallel_size": 2
47
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
48
},
49
"client_parameters": {
50
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
51
"dataset_name": "random",
52
"random-input-len": 128,
53
"random-output-len": 128
54
}
55
},
56
{
77
- "test_name": "serving_llama8B_tp4_random_128_128",
57
+ "test_name": "serving_llama8B_int8_tp1_random_128_128",
58
"server_parameters": {
79
- "tensor_parallel_size": 4
59
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8"
60
},
61
"client_parameters": {
62
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
63
"dataset_name": "random",
64
"random-input-len": 128,
65
"random-output-len": 128
66
}
67
},
68
{
88
- "test_name": "serving_llama8B_tp1_random_128_2048",
89
- "server_parameters": {
90
- "tensor_parallel_size": 1
91
- },
92
- "client_parameters": {
93
- "dataset_name": "random",
94
- "random-input-len": 128,
95
- "random-output-len": 2048
96
- }
97
- },
98
- {
99
- "test_name": "serving_llama8B_tp2_random_128_2048",
69
+ "test_name": "serving_llama1B_tp1_random_128_128",
70
"server_parameters": {
101
- "tensor_parallel_size": 2
71
+ "model": "meta-llama/Llama-3.2-1B"
72
},
73
"client_parameters": {
74
+ "model": "meta-llama/Llama-3.2-1B",
75
"dataset_name": "random",
76
"random-input-len": 128,
106
- "random-output-len": 2048
77
+ "random-output-len": 128
78
}
79
},
80
{
110
- "test_name": "serving_llama8B_tp4_random_128_2048",
81
+ "test_name": "serving_llama3B_tp1_random_128_128",
82
"server_parameters": {
112
- "tensor_parallel_size": 4
83
+ "model": "meta-llama/Llama-3.2-3B-Instruct"
84
},
85
"client_parameters": {
86
+ "model": "meta-llama/Llama-3.2-3B-Instruct",
87
"dataset_name": "random",
88
"random-input-len": 128,
117
- "random-output-len": 2048
118
- }
119
- },
120
- {
121
- "test_name": "serving_llama8B_tp1_random_2048_128",
122
- "server_parameters": {
123
- "tensor_parallel_size": 1
124
- },
125
- "client_parameters": {
126
- "dataset_name": "random",
127
- "random-input-len": 2048,
89
"random-output-len": 128
90
}
91
},
92
{
132
- "test_name": "serving_llama8B_tp2_random_2048_128",
93
+ "test_name": "serving_llama70B_tp1_random_128_128",
94
"server_parameters": {
134
- "tensor_parallel_size": 2
95
+ "model": "meta-llama/Llama-3.3-70B-Instruct"
96
},
97
"client_parameters": {
98
+ "model": "meta-llama/Llama-3.3-70B-Instruct",
99
"dataset_name": "random",
138
- "random-input-len": 2048,
100
+ "random-input-len": 128,
101
"random-output-len": 128
102
}
103
},
104
{
143
- "test_name": "serving_llama8B_tp4_random_2048_128",
105
+ "test_name": "serving_granite2B_tp1_random_128_128",
106
"server_parameters": {
145
- "tensor_parallel_size": 4
107
+ "model": "ibm-granite/granite-3.2-2b-instruct"
108
},
109
"client_parameters": {
110
+ "model": "ibm-granite/granite-3.2-2b-instruct",
111
"dataset_name": "random",
149
- "random-input-len": 2048,
112
+ "random-input-len": 128,
113
"random-output-len": 128
114
}
115
},
116
{
154
- "test_name": "serving_llama8B_tp1_random_2048_2048",
155
- "server_parameters": {
156
- "tensor_parallel_size": 1
157
- },
158
- "client_parameters": {
159
- "dataset_name": "random",
160
- "random-input-len": 2048,
161
- "random-output-len": 2048
162
- }
163
- },
164
- {
165
- "test_name": "serving_llama8B_tp2_random_2048_2048",
166
- "server_parameters": {
167
- "tensor_parallel_size": 2
168
- },
169
- "client_parameters": {
170
- "dataset_name": "random",
171
- "random-input-len": 2048,
172
- "random-output-len": 2048
173
- }
174
- },
175
- {
176
- "test_name": "serving_llama8B_tp4_random_2048_2048",
177
- "server_parameters": {
178
- "tensor_parallel_size": 4
179
- },
180
- "client_parameters": {
181
- "dataset_name": "random",
182
- "random-input-len": 2048,
183
- "random-output-len": 2048
184
- }
185
- },
186
- {
187
- "test_name": "serving_llama8B_int4_tp1_random_128_128",
117
+ "test_name": "serving_qwen1.7B_tp1_random_128_128",
118
"server_parameters": {
189
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
190
- "tensor_parallel_size": 1
119
+ "model": "Qwen/Qwen3-1.7B"
120
},
121
"client_parameters": {
193
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
122
+ "model": "Qwen/Qwen3-1.7B",
123
"dataset_name": "random",
124
"random-input-len": 128,
125
"random-output-len": 128
126
}
127
},
128
{
200
- "test_name": "serving_llama8B_int4_tp2_random_128_128",
129
+ "test_name": "serving_qwen4B_tp1_random_128_128",
130
"server_parameters": {
202
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
203
- "tensor_parallel_size": 2
131
+ "model": "Qwen/Qwen3-4B"
132
},
133
"client_parameters": {
206
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
134
+ "model": "Qwen/Qwen3-4B",
135
"dataset_name": "random",
136
"random-input-len": 128,
137
"random-output-len": 128
138
}
139
},
140
{
213
- "test_name": "serving_llama8B_int4_tp4_random_128_128",
141
+ "test_name": "serving_qwen8B_tp1_random_128_128",
142
"server_parameters": {
215
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
216
- "tensor_parallel_size": 4
143
+ "model": "Qwen/Qwen3-8B"
144
},
145
"client_parameters": {
219
- "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
146
+ "model": "Qwen/Qwen3-8B",
147
"dataset_name": "random",
148
"random-input-len": 128,
149
"random-output-len": 128
150
}
151
},
152
{
226
- "test_name": "serving_llama8B_int8_tp1_random_128_128",
153
+ "test_name": "serving_qwen14B_tp1_random_128_128",
154
"server_parameters": {
228
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
229
- "tensor_parallel_size": 1
155
+ "model": "Qwen/Qwen3-14B"
156
},
157
"client_parameters": {
232
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
158
+ "model": "Qwen/Qwen3-14B",
159
"dataset_name": "random",
160
"random-input-len": 128,
161
"random-output-len": 128
162
}
163
},
164
{
239
- "test_name": "serving_llama8B_int8_tp2_random_128_128",
165
+ "test_name": "serving_qwen30B_tp1_random_128_128",
166
"server_parameters": {
241
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
242
- "tensor_parallel_size": 2
167
+ "model": "Qwen/Qwen3-30B-A3B"
168
},
169
"client_parameters": {
245
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
170
+ "model": "Qwen/Qwen3-30B-A3B",
171
"dataset_name": "random",
172
"random-input-len": 128,
173
"random-output-len": 128
174
}
175
},
176
{
252
- "test_name": "serving_llama8B_int8_tp4_random_128_128",
177
+ "test_name": "serving_glm9B_tp1_random_128_128",
178
"server_parameters": {
254
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
255
- "tensor_parallel_size": 4
179
+ "model": "zai-org/glm-4-9b-hf"
180
},
181
"client_parameters": {
258
- "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
182
+ "model": "zai-org/glm-4-9b-hf",
183
"dataset_name": "random",
184
"random-input-len": 128,
185
"random-output-len": 128
186
}
187
},
188
{
265
- "test_name": "serving_llama3B_tp1_random_128_128",
189
+ "test_name": "serving_gemma7B_tp1_random_128_128",
190
"server_parameters": {
267
- "model": "meta-llama/Llama-3.2-3B-Instruct",
268
- "tensor_parallel_size": 1
191
+ "model": "google/gemma-7b"
192
},
193
"client_parameters": {
271
- "model": "meta-llama/Llama-3.2-3B-Instruct",
194
+ "model": "google/gemma-7b",
195
"dataset_name": "random",
196
"random-input-len": 128,
197
"random-output-len": 128
198
}
199
},
200
{
278
- "test_name": "serving_granite2B_tp1_random_128_128",
201
+ "test_name": "serving_gemma3-4b_tp1_random_128_128",
202
+ "server_environment_variables": {
203
+ "VLLM_CPU_SGL_KERNEL": 0
204
+ },
205
"server_parameters": {
280
- "model": "ibm-granite/granite-3.2-2b-instruct",
281
- "tensor_parallel_size": 1
206
+ "model": "google/gemma-3-4b-it"
207
},
208
"client_parameters": {
284
- "model": "ibm-granite/granite-3.2-2b-instruct",
209
+ "model": "google/gemma-3-4b-it",
210
"dataset_name": "random",
211
"random-input-len": 128,
212
"random-output-len": 128
213
}
214
},
215
{
291
- "test_name": "serving_qwen1.7B_tp1_random_128_128",
216
+ "test_name": "serving_gemma3-12b_tp1_random_128_128",
217
+ "server_environment_variables": {
218
+ "VLLM_CPU_SGL_KERNEL": 0
219
+ },
220
"server_parameters": {
293
- "model": "Qwen/Qwen3-1.7B",
294
- "tensor_parallel_size": 1
221
+ "model": "google/gemma-3-12b-it"
222
},
223
"client_parameters": {
297
- "model": "Qwen/Qwen3-1.7B",
224
+ "model": "google/gemma-3-12b-it",
225
"dataset_name": "random",
226
"random-input-len": 128,
227
"random-output-len": 128
228
}
229
},
230
{
304
- "test_name": "serving_qwen4B_tp1_random_128_128",
231
+ "test_name": "serving_gemma4-4b_tp1_random_128_128",
232
+ "server_environment_variables": {
233
+ "VLLM_CPU_SGL_KERNEL": 0
234
+ },
235
"server_parameters": {
306
- "model": "Qwen/Qwen3-4B",
307
- "tensor_parallel_size": 1
236
+ "model": "google/gemma-4-E4B-it"
237
},
238
"client_parameters": {
310
- "model": "Qwen/Qwen3-4B",
239
+ "model": "google/gemma-4-E4B-it",
240
"dataset_name": "random",
241
"random-input-len": 128,
242
"random-output-len": 128
243
}
244
},
245
{
317
- "test_name": "serving_qwen8B_tp1_random_128_128",
246
+ "test_name": "serving_gemma4-2b_tp1_random_128_128",
247
+ "server_environment_variables": {
248
+ "VLLM_CPU_SGL_KERNEL": 0
249
+ },
250
"server_parameters": {
319
- "model": "Qwen/Qwen3-8B",
320
- "tensor_parallel_size": 1
251
+ "model": "google/gemma-4-E2B-it"
252
},
253
"client_parameters": {
323
- "model": "Qwen/Qwen3-8B",
254
+ "model": "google/gemma-4-E2B-it",
255
"dataset_name": "random",
256
"random-input-len": 128,
257
"random-output-len": 128
258
}
259
},
260
{
330
- "test_name": "serving_glm9B_tp1_random_128_128",
261
+ "test_name": "serving_gemma4-26b_tp1_random_128_128",
262
+ "server_environment_variables": {
263
+ "VLLM_CPU_SGL_KERNEL": 0,
264
+ "VLLM_CPU_ATTN_SPLIT_KV": 0
265
+ },
266
"server_parameters": {
332
- "model": "zai-org/glm-4-9b-hf",
333
- "tensor_parallel_size": 1
267
+ "model": "google/gemma-4-26B-A4B-it"
268
},
269
"client_parameters": {
336
- "model": "zai-org/glm-4-9b-hf",
270
+ "model": "google/gemma-4-26B-A4B-it",
271
"dataset_name": "random",
272
"random-input-len": 128,
273
"random-output-len": 128
274
}
275
},
276
{
343
- "test_name": "serving_gemma7B_tp1_random_128_128",
277
+ "test_name": "serving_phi4_tp1_random_128_128",
278
"server_parameters": {
345
- "model": "google/gemma-7b",
346
- "tensor_parallel_size": 1
279
+ "model": "microsoft/Phi-4-reasoning"
280
},
281
"client_parameters": {
349
- "model": "google/gemma-7b",
282
+ "model": "microsoft/Phi-4-reasoning",
283
"dataset_name": "random",
284
"random-input-len": 128,
285
"random-output-len": 128
docs/models/hardware_supported_models/cpu.md
+42
-16
@@ -11,24 +11,50 @@
11
12
### Text-only Language Models
13
14
-| Model | Architecture | Supported |
14
+| Model | Architecture | Supported |
15
| ------------------------------------ | ---------------------------------------- | --------- |
16
-| meta-llama/Llama-3.1-8B-Instruct | LlamaForCausalLM | ✅ |
17
-| meta-llama/Llama-3.2-3B-Instruct | LlamaForCausalLM | ✅ |
18
-| ibm-granite/granite-3.2-2b-instruct | GraniteForCausalLM | ✅ |
19
-| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ |
20
-| Qwen/Qwen3-4B | Qwen3ForCausalLM | ✅ |
21
-| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ |
22
-| zai-org/glm-4-9b-hf | GLMForCausalLM | ✅ |
23
-| google/gemma-7b | GemmaForCausalLM | ✅ |
16
+| unsloth/gpt-oss-20b | GptOssForCausalLM | ✅ |
17
+| meta-llama/Llama-3.1-8B-Instruct | LlamaForCausalLM | ✅ |
18
+| meta-llama/Llama-3.2-1B | LlamaForCausalLM | ✅ |
19
+| meta-llama/Llama-3.2-3B-Instruct | LlamaForCausalLM | ✅ |
20
+| meta-llama/Llama-3.3-70B-Instruct | LlamaForCausalLM | ✅ |
21
+| RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8 | LlamaForCausalLM | ✅ |
22
+| RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ |
23
+| RedHatAI/Llama-3.2-1B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ |
24
+| RedHatAI/Llama-3.2-3B-Instruct-quantized.w8a8 | LlamaForCausalLM | ✅ |
25
+| RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8 | LlamaForCausalLM | ✅ |
26
+| hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 | LlamaForCausalLM | ✅ |
27
+| AMead10/Llama-3.2-1B-Instruct-AWQ | LlamaForCausalLM | ✅ |
28
+| AMead10/Llama-3.2-3B-Instruct-AWQ | LlamaForCausalLM | ✅ |
29
+| TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ | LlamaForCausalLM | ✅ |
30
+| TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ | LlamaForCausalLM | ✅ |
31
+| ibm-granite/granite-3.2-2b-instruct | GraniteForCausalLM | ✅ |
32
+| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ |
33
+| Qwen/Qwen3-4B | Qwen3ForCausalLM | ✅ |
34
+| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ |
35
+| Qwen/Qwen3-14B | Qwen3ForCausalLM | ✅ |
36
+| Qwen/Qwen3-14B-AWQ | Qwen3ForCausalLM | ✅ |
37
+| Qwen/Qwen3-30B-A3B | Qwen3MoeForCausalLM | ✅ |
38
+| Qwen/QwQ-32B-AWQ | Qwen2ForCausalLM | ✅ |
39
+| Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4 | Qwen2ForCausalLM | ✅ |
40
+| RedHatAI/QwQ-32B-quantized.w8a8 | Qwen2ForCausalLM | ✅ |
41
+| zai-org/glm-4-9b-hf | GLMForCausalLM | ✅ |
42
+| google/gemma-7b | GemmaForCausalLM | ✅ |
43
+| microsoft/Phi-4-reasoning | Phi3ForCausalLM | ✅ |
44
+| TheBloke/Mistral-7B-Instruct-v0.2-AWQ | MistralForCausalLM | ✅ |
45
46
### Multimodal Language Models
47
27
-| Model | Architecture | Supported |
48
+| Model | Architecture | Supported |
49
| ------------------------------------ | ---------------------------------------- | --------- |
29
-| Qwen/Qwen2.5-VL-7B-Instruct | Qwen2VLForConditionalGeneration | ✅ |
30
-| openai/whisper-large-v3 | WhisperForConditionalGeneration | ✅ |
31
-
32
-✅ Runs and optimized.
33
-🟨 Runs and correct but not optimized to green yet.
34
-❌ Does not pass accuracy test or does not run.
50
+| meta-llama/Llama-4-Scout-17B-16E-Instruct | Llama4ForConditionalGeneration | ✅ |
51
+| google/gemma-3-4b-it | Gemma3ForConditionalGeneration | ✅ |
52
+| google/gemma-3-12b-it | Gemma3ForConditionalGeneration | ✅ |
53
+| google/gemma-4-E4B-it | Gemma4ForConditionalGeneration | ✅ |
54
+| google/gemma-4-E2B-it | Gemma4ForConditionalGeneration | ✅ |
55
+| google/gemma-4-26B-A4B-it | Gemma4ForConditionalGeneration | ✅ |
56
+| microsoft/Phi-4-multimodal-instruct | Phi4MMForCausalLM | ✅ |
57
+| Qwen/Qwen2.5-VL-7B-Instruct | Qwen2VLForConditionalGeneration | ✅ |
58
+| openai/whisper-large-v3 | WhisperForConditionalGeneration | ✅ |
59
+
60
+✅ Runs and optimized.