A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 09:00 UTC
[CI/Build][Intel] Add new performance benchmarks for Intel Gaudi 3 (#31025)
Signed-off-by: Szymon Reginis <sreginis@habana.ai> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Szymon Reginis committed
Mar 3, 2026 at 12:48 UTC
4beebfd14650b1c6a687e7ab496d501423a0e50d
3 files changed
+192
.buildkite/performance-benchmarks/tests/latency-tests-hpu.json
+51
@@ -51,5 +51,56 @@
51
"max-model-len": 256,
52
"async-scheduling": ""
53
}
54
+ },
55
+ {
56
+ "test_name": "latency_deepseek_r1",
57
+ "environment_variables": {
58
+ "PT_HPU_LAZY_MODE": 1,
59
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
60
+ "VLLM_CONTIGUOUS_PA": 1,
61
+ "VLLM_DEFRAG": 1
62
+ },
63
+ "parameters": {
64
+ "model": "deepseek-ai/DeepSeek-R1",
65
+ "tensor_parallel_size": 8,
66
+ "load_format": "dummy",
67
+ "max-model-len": 2048,
68
+ "dtype": "bfloat16"
69
+ }
70
+ },
71
+ {
72
+ "test_name": "latency_llama4_maverick_17b128e_instruct_fp8",
73
+ "environment_variables": {
74
+ "PT_HPU_LAZY_MODE": 1,
75
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
76
+ "VLLM_CONTIGUOUS_PA": 1,
77
+ "VLLM_DEFRAG": 1
78
+ },
79
+ "parameters": {
80
+ "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
81
+ "tensor_parallel_size": 8,
82
+ "max-model-len": 512,
83
+ "max-num-seqs": 128,
84
+ "async-scheduling": "",
85
+ "gpu-memory-utilization": 0.95,
86
+ "enable_expert_parallel": ""
87
+ }
88
+ },
89
+ {
90
+ "test_name": "latency_qwen3_8b",
91
+ "environment_variables": {
92
+ "PT_HPU_LAZY_MODE": 1,
93
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
94
+ "VLLM_CONTIGUOUS_PA": 1,
95
+ "VLLM_DEFRAG": 1
96
+ },
97
+ "parameters": {
98
+ "model": "Qwen/Qwen3-8B",
99
+ "tensor_parallel_size": 1,
100
+ "max-model-len": 2048,
101
+ "max-num-seqs": 128,
102
+ "dtype": "bfloat16",
103
+ "async-scheduling": ""
104
+ }
105
}
106
]
.buildkite/performance-benchmarks/tests/serving-tests-hpu.json
+79
@@ -78,5 +78,84 @@
78
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
79
"num_prompts": 200
80
}
81
+ },
82
+ {
83
+ "test_name": "serving_deepseek_r1",
84
+ "qps_list": [1, 4, 16, "inf"],
85
+ "server_environment_variables": {
86
+ "PT_HPU_LAZY_MODE": 1,
87
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
88
+ "VLLM_CONTIGUOUS_PA": 1,
89
+ "VLLM_DEFRAG": 1
90
+ },
91
+ "server_parameters": {
92
+ "model": "deepseek-ai/DeepSeek-R1",
93
+ "tensor_parallel_size": 8,
94
+ "swap_space": 16,
95
+ "disable_log_stats": "",
96
+ "load_format": "dummy",
97
+ "max-model-len": 2048,
98
+ "max-num-seqs": 200,
99
+ "async-scheduling": "",
100
+ "dtype": "bfloat16"
101
+ },
102
+ "client_parameters": {
103
+ "model": "deepseek-ai/DeepSeek-R1",
104
+ "backend": "vllm",
105
+ "dataset_name": "sharegpt",
106
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
107
+ "num_prompts": 200
108
+ }
109
+ },
110
+ {
111
+ "test_name": "serving_llama4_maverick_17b128e_instruct_fp8",
112
+ "qps_list": [1, 4, 16, "inf"],
113
+ "server_environment_variables": {
114
+ "PT_HPU_LAZY_MODE": 1,
115
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
116
+ "VLLM_CONTIGUOUS_PA": 1,
117
+ "VLLM_DEFRAG": 1
118
+ },
119
+ "server_parameters": {
120
+ "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
121
+ "tensor_parallel_size": 8,
122
+ "disable_log_stats": "",
123
+ "max-model-len": 2048,
124
+ "max-num-seqs": 128,
125
+ "async-scheduling": "",
126
+ "enable_expert_parallel": "",
127
+ "max-num-batched-tokens": 4096
128
+ },
129
+ "client_parameters": {
130
+ "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
131
+ "backend": "vllm",
132
+ "dataset_name": "sharegpt",
133
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
134
+ "num_prompts": 200
135
+ }
136
+ },
137
+ {
138
+ "test_name": "serving_qwen3_8b",
139
+ "qps_list": [1, 4, 10, "inf"],
140
+ "server_environment_variables": {
141
+ "PT_HPU_LAZY_MODE": 1,
142
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
143
+ "VLLM_CONTIGUOUS_PA": 1,
144
+ "VLLM_DEFRAG": 1
145
+ },
146
+ "server_parameters": {
147
+ "model": "Qwen/Qwen-3-8B",
148
+ "tensor_parallel_size": 1,
149
+ "dtype": "bfloat16",
150
+ "disable_log_stats": "",
151
+ "async-scheduling": ""
152
+ },
153
+ "client_parameters": {
154
+ "model": "Qwen/Qwen-3-8B",
155
+ "backend": "vllm",
156
+ "dataset_name": "sharegpt",
157
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
158
+ "num_prompts": 200
159
+ }
160
}
161
]
.buildkite/performance-benchmarks/tests/throughput-tests-hpu.json
+62
@@ -57,5 +57,67 @@
57
"max-num-seqs": 512,
58
"async-scheduling": ""
59
}
60
+ },
61
+ {
62
+ "test_name": "throughput_deepseek_r1",
63
+ "environment_variables": {
64
+ "PT_HPU_LAZY_MODE": 1,
65
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
66
+ "VLLM_CONTIGUOUS_PA": 1,
67
+ "VLLM_DEFRAG": 1
68
+ },
69
+ "parameters": {
70
+ "model": "deepseek-ai/DeepSeek-R1",
71
+ "tensor_parallel_size": 8,
72
+ "load_format": "dummy",
73
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
74
+ "dataset_name": "sharegpt",
75
+ "num_prompts": 1000,
76
+ "backend": "vllm",
77
+ "max-model-len": 2048,
78
+ "max-num-seqs": 384,
79
+ "async-scheduling": ""
80
+ }
81
+ },
82
+ {
83
+ "test_name": "throughput_llama4_maverick_17b128e_instruct_fp8",
84
+ "environment_variables": {
85
+ "PT_HPU_LAZY_MODE": 1,
86
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
87
+ "VLLM_CONTIGUOUS_PA": 1,
88
+ "VLLM_DEFRAG": 1
89
+ },
90
+ "parameters": {
91
+ "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
92
+ "tensor_parallel_size": 8,
93
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
94
+ "dataset_name": "sharegpt",
95
+ "num_prompts": 1000,
96
+ "backend": "vllm",
97
+ "max-model-len": 2048,
98
+ "max-num-seqs": 512,
99
+ "async-scheduling": "",
100
+ "enable_expert_parallel": ""
101
+ }
102
+ },
103
+ {
104
+ "test_name": "throughput_qwen3_8b",
105
+ "environment_variables": {
106
+ "PT_HPU_LAZY_MODE": 1,
107
+ "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
108
+ "VLLM_CONTIGUOUS_PA": 1,
109
+ "VLLM_DEFRAG": 1
110
+ },
111
+ "parameters": {
112
+ "model": "Qwen/Qwen-3-8B",
113
+ "tensor_parallel_size": 1,
114
+ "load_format": "dummy",
115
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
116
+ "dataset_name": "sharegpt",
117
+ "num_prompts": 1000,
118
+ "max-num-seqs": 512,
119
+ "backend": "vllm",
120
+ "async-scheduling": ""
121
+ }
122
}
123
]