A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 04:15 UTC
update GPU json file based on h200 recipes (#43262)
Signed-off-by: louie-tsai <louie.tsai@intel.com>
Louie Tsai committed
May 20, 2026 at 20:57 UTC
5d041cc1fe5181daabf39943efc7b678380d57bd
1 file changed
+104
-69
.buildkite/performance-benchmarks/tests/serving-tests.json
+104
-69
@@ -1,77 +1,112 @@
1
-[
1
+{
2
+ "defaults": {
3
+ "qps_list": [
4
+ "inf"
5
+ ],
6
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
7
+ "server_parameters": {
8
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
9
+ "tensor_parallel_size": 1,
10
+ "dtype": "bfloat16"
11
+ },
12
+ "client_parameters": {
13
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
14
+ "backend": "vllm",
15
+ "ignore-eos": "",
16
+ "temperature": 0,
17
+ "num_prompts": 200
18
+ }
19
+ },
20
+ "tests": [
21
+ {
22
+ "test_name": "serving_llama8B_tp1_sharegpt",
23
+ "server_parameters": {
24
+ "tensor_parallel_size": 1
25
+ },
26
+ "client_parameters": {
27
+ "dataset_name": "sharegpt",
28
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
29
+ }
30
+ },
31
+ {
32
+ "dataset_name": "sharegpt",
33
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
34
+ }
35
+ },
36
+ {
37
+ "test_name": "serving_llama8B_tp1_random_128_128",
38
+ "server_parameters": {
39
+ "tensor_parallel_size": 1
40
+ },
41
+ "client_parameters": {
42
+ "dataset_name": "random",
43
+ "random-input-len": 128,
44
+ "random-output-len": 128
45
+ }
46
+ },
47
+ {
48
+ "test_name": "serving_llama8B_tp1_random_128_2048",
49
+ "server_parameters": {
50
+ "tensor_parallel_size": 1
51
+ },
52
+ "client_parameters": {
53
+ "dataset_name": "random",
54
+ "random-input-len": 128,
55
+ "random-output-len": 2048
56
+ }
57
+ },
58
{
3
- "test_name": "serving_llama8B_tp1_sharegpt",
4
- "qps_list": [1, 4, 16, "inf"],
5
- "server_parameters": {
6
- "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
7
- "tensor_parallel_size": 1,
8
- "disable_log_stats": "",
9
- "load_format": "dummy"
10
- },
11
- "client_parameters": {
12
- "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
13
- "backend": "vllm",
14
- "dataset_name": "sharegpt",
15
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
16
- "temperature": 0,
17
- "num_prompts": 200
18
- }
59
+ "test_name": "serving_llama8B_tp1_random_2048_128",
60
+ "server_parameters": {
61
+ "tensor_parallel_size": 1
62
+ },
63
+ "client_parameters": {
64
+ "dataset_name": "random",
65
+ "random-input-len": 2048,
66
+ "random-output-len": 128
67
+ }
68
},
69
{
21
- "test_name": "serving_llama70B_tp4_sharegpt",
22
- "qps_list": [1, 4, 16, "inf"],
23
- "server_parameters": {
24
- "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
25
- "tensor_parallel_size": 4,
26
- "disable_log_stats": "",
27
- "load_format": "dummy"
28
- },
29
- "client_parameters": {
30
- "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
31
- "backend": "vllm",
32
- "dataset_name": "sharegpt",
33
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
34
- "temperature": 0,
35
- "num_prompts": 200
36
- }
70
+ "test_name": "serving_llama8B_tp1_random_2048_2048",
71
+ "server_parameters": {
72
+ "tensor_parallel_size": 1
73
+ },
74
+ "client_parameters": {
75
+ "dataset_name": "random",
76
+ "random-input-len": 2048,
77
+ "random-output-len": 2048
78
+ }
79
},
80
{
39
- "test_name": "serving_mixtral8x7B_tp2_sharegpt",
40
- "qps_list": [1, 4, 16, "inf"],
41
- "server_parameters": {
42
- "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
43
- "tensor_parallel_size": 2,
44
- "disable_log_stats": "",
45
- "load_format": "dummy"
46
- },
47
- "client_parameters": {
48
- "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
49
- "backend": "vllm",
50
- "dataset_name": "sharegpt",
51
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
52
- "temperature": 0,
53
- "num_prompts": 200
54
- }
81
+ "test_name": "serving_llama70B_tp4_random_128_128",
82
+ "server_parameters": {
83
+ "model": "meta-llama/Llama-3.3-70B-Instruct",
84
+ "async_scheduling": "",
85
+ "no_enable_prefix_caching": "",
86
+ "max_num_batched_tokens": 8192
87
+ },
88
+ "client_parameters": {
89
+ "model": "meta-llama/Llama-3.3-70B-Instruct",
90
+ "dataset_name": "random",
91
+ "random-input-len": 128,
92
+ "random-output-len": 128
93
+ }
94
},
95
{
57
- "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
58
- "qps_list": [2],
59
- "server_parameters": {
60
- "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
61
- "tensor_parallel_size": 4,
62
- "speculative_config": {
63
- "model": "turboderp/Qwama-0.5B-Instruct",
64
- "num_speculative_tokens": 4,
65
- "draft_tensor_parallel_size": 1
66
- }
67
- },
68
- "client_parameters": {
69
- "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
70
- "backend": "vllm",
71
- "dataset_name": "sharegpt",
72
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
73
- "temperature": 0,
74
- "num_prompts": 200
75
- }
96
+ "test_name": "serving_gemma4-e4b_tp1_random_128_128",
97
+ "server_parameters": {
98
+ "model": "google/gemma-4-E4B-it",
99
+ "enable_auto_tool_choice": "",
100
+ "tool_call_parser": "gemma4",
101
+ "chat_template": "examples/tool_chat_template_gemma4.jinja",
102
+ "reasoning_parser": "gemma4"
103
+ },
104
+ "client_parameters": {
105
+ "model": "google/gemma-4-E4B-it",
106
+ "dataset_name": "random",
107
+ "random-input-len": 128,
108
+ "random-output-len": 128
109
+ }
110
}
77
-]
111
+ ]
112
+}