A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
add SLA information into comparison graph for vLLM Benchmark Suite (#25525)
Signed-off-by: Tsai, Louie <louie.tsai@intel.com> Signed-off-by: louie-tsai <louie.tsai@intel.com> Signed-off-by: Louie Tsai <louie.tsai@intel.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Louie Tsai committed
Oct 23, 2025 at 01:04 UTC
3b7bdf983b5bf76da7a2c580acd5edb1075d7bca
7 files changed
+547
-89
.buildkite/nightly-benchmarks/scripts/compare-json-results.py
+182
-33
@@ -7,6 +7,7 @@ from importlib import util
7
8
import pandas as pd
9
10
+pd.options.display.float_format = "{:.2f}".format
11
plotly_found = util.find_spec("plotly.express") is not None
12
13
@@ -109,7 +110,10 @@ def compare_data_columns(
110
if len(compare_frames) >= 2:
111
base = compare_frames[0]
112
current = compare_frames[-1]
112
- ratio = current / base
113
+ if "P99" in data_column or "Median" in data_column:
114
+ ratio = base / current # for latency
115
+ else:
116
+ ratio = current / base
117
ratio = ratio.mask(base == 0) # avoid inf when baseline is 0
118
ratio.name = f"Ratio 1 vs {len(compare_frames)}"
119
frames.append(ratio)
@@ -199,6 +203,71 @@ def split_json_by_tp_pp(
203
return saved_paths
204
205
206
+def _add_limit_line(fig, y_value, label):
207
+ # Visible dashed line + annotation
208
+ fig.add_hline(
209
+ y=y_value,
210
+ line_dash="dash",
211
+ line_color="red" if "ttft" in label.lower() else "blue",
212
+ annotation_text=f"{label}: {y_value} ms",
213
+ annotation_position="top left",
214
+ )
215
+ # Optional: add a legend item (as a transparent helper trace)
216
+ if plot and plotly_found:
217
+ import plotly.graph_objects as go
218
+
219
+ fig.add_trace(
220
+ go.Scatter(
221
+ x=[None],
222
+ y=[None],
223
+ mode="lines",
224
+ line=dict(
225
+ dash="dash", color="red" if "ttft" in label.lower() else "blue"
226
+ ),
227
+ name=f"{label}",
228
+ )
229
+ )
230
+
231
+
232
+def _find_concurrency_col(df: pd.DataFrame) -> str:
233
+ for c in [
234
+ "# of max concurrency.",
235
+ "# of max concurrency",
236
+ "Max Concurrency",
237
+ "max_concurrency",
238
+ "Concurrency",
239
+ ]:
240
+ if c in df.columns:
241
+ return c
242
+ # Fallback: guess an integer-like column (harmless if unused)
243
+ for c in df.columns:
244
+ if df[c].dtype.kind in "iu" and df[c].nunique() > 1 and df[c].min() >= 1:
245
+ return c
246
+ return "# of max concurrency."
247
+
248
+
249
+def _highlight_threshold(
250
+ df: pd.DataFrame, threshold: float
251
+) -> "pd.io.formats.style.Styler":
252
+ """Highlight numeric per-configuration columns with value <= threshold."""
253
+ conc_col = _find_concurrency_col(df)
254
+ key_cols = [
255
+ c
256
+ for c in ["Model", "Dataset Name", "Input Len", "Output Len", conc_col]
257
+ if c in df.columns
258
+ ]
259
+ conf_cols = [
260
+ c for c in df.columns if c not in key_cols and not str(c).startswith("Ratio")
261
+ ]
262
+ conf_cols = [c for c in conf_cols if pd.api.types.is_numeric_dtype(df[c])]
263
+ return df.style.map(
264
+ lambda v: "background-color:#e6ffe6;font-weight:bold;"
265
+ if pd.notna(v) and v <= threshold
266
+ else "",
267
+ subset=conf_cols,
268
+ )
269
+
270
+
271
if __name__ == "__main__":
272
parser = argparse.ArgumentParser()
273
parser.add_argument(
@@ -220,6 +289,26 @@ if __name__ == "__main__":
289
default="# of max concurrency.",
290
help="column name to use as X Axis in comparison graph",
291
)
292
+ parser.add_argument(
293
+ "-l",
294
+ "--latency",
295
+ type=str,
296
+ default="p99",
297
+ help="take median|p99 for latency like TTFT/TPOT",
298
+ )
299
+ parser.add_argument(
300
+ "--ttft-max-ms",
301
+ type=float,
302
+ default=3000.0,
303
+ help="Reference limit for TTFT plots (ms)",
304
+ )
305
+ parser.add_argument(
306
+ "--tpot-max-ms",
307
+ type=float,
308
+ default=100.0,
309
+ help="Reference limit for TPOT plots (ms)",
310
+ )
311
+
312
args = parser.parse_args()
313
314
drop_column = "P99"
@@ -234,12 +323,22 @@ if __name__ == "__main__":
323
"# of max concurrency.",
324
"qps",
325
]
237
- data_cols_to_compare = ["Output Tput (tok/s)", "Median TTFT (ms)", "Median"]
238
- html_msgs_for_data_cols = [
239
- "Compare Output Tokens /n",
240
- "Median TTFT /n",
241
- "Median TPOT /n",
242
- ]
326
+
327
+ if "median" in args.latency:
328
+ data_cols_to_compare = ["Output Tput (tok/s)", "Median TTFT (ms)", "Median"]
329
+ html_msgs_for_data_cols = [
330
+ "Compare Output Tokens /n",
331
+ "Median TTFT /n",
332
+ "Median TPOT /n",
333
+ ]
334
+ drop_column = "P99"
335
+ elif "p99" in args.latency:
336
+ data_cols_to_compare = ["Output Tput (tok/s)", "P99 TTFT (ms)", "P99"]
337
+ html_msgs_for_data_cols = [
338
+ "Compare Output Tokens /n",
339
+ "P99 TTFT /n",
340
+ "P99 TPOT /n",
341
+ ]
342
343
if len(args.file) == 1:
344
files = split_json_by_tp_pp(args.file[0], output_root="splits")
@@ -275,33 +374,83 @@ if __name__ == "__main__":
374
f"Expected subset: {filtered_info_cols}, "
375
f"but DataFrame has: {list(output_df.columns)}"
376
)
278
- output_df_sorted = output_df.sort_values(by=existing_group_cols)
377
+ # output_df_sorted = output_df.sort_values(by=existing_group_cols)
378
+ output_df_sorted = output_df.sort_values(by=args.xaxis)
379
output_groups = output_df_sorted.groupby(existing_group_cols, dropna=False)
380
for name, group in output_groups:
281
- html = group.to_html()
282
- text_file.write(html_msgs_for_data_cols[i])
283
- text_file.write(html)
284
-
285
- if plot and plotly_found:
286
- import plotly.express as px
287
-
288
- df = group[raw_data_cols]
289
- df_sorted = df.sort_values(by=info_cols[y_axis_index])
290
- # Melt DataFrame for plotting
291
- df_melted = df_sorted.melt(
292
- id_vars=info_cols[y_axis_index],
293
- var_name="Configuration",
294
- value_name=data_cols_to_compare[i],
381
+ group_name = (
382
+ ",".join(map(str, name)).replace(",", "_").replace("/", "-")
383
+ )
384
+ group_html_name = "perf_comparison_" + group_name + ".html"
385
+
386
+ metric_name = str(data_cols_to_compare[i]).lower()
387
+ if "tok/s" in metric_name:
388
+ html = group.to_html()
389
+ elif "ttft" in metric_name:
390
+ styler = _highlight_threshold(group, args.ttft_max_ms).format(
391
+ {c: "{:.2f}" for c in group.select_dtypes("number").columns},
392
+ na_rep="—",
393
+ )
394
+ html = styler.to_html(
395
+ table_attributes='border="1" class="dataframe"'
396
+ )
397
+ elif (
398
+ "tpot" in metric_name
399
+ or "median" in metric_name
400
+ or "p99" in metric_name
401
+ ):
402
+ styler = _highlight_threshold(group, args.tpot_max_ms).format(
403
+ {c: "{:.2f}" for c in group.select_dtypes("number").columns},
404
+ na_rep="—",
405
)
296
- title = data_cols_to_compare[i] + " vs " + info_cols[y_axis_index]
297
- # Create Plotly line chart
298
- fig = px.line(
299
- df_melted,
300
- x=info_cols[y_axis_index],
301
- y=data_cols_to_compare[i],
302
- color="Configuration",
303
- title=title,
304
- markers=True,
406
+ html = styler.to_html(
407
+ table_attributes='border="1" class="dataframe"'
408
)
306
- # Export to HTML
307
- text_file.write(fig.to_html(full_html=True, include_plotlyjs="cdn"))
409
+
410
+ text_file.write(html_msgs_for_data_cols[i])
411
+ text_file.write(html)
412
+ with open(group_html_name, "a+") as sub_text_file:
413
+ sub_text_file.write(html_msgs_for_data_cols[i])
414
+ sub_text_file.write(html)
415
+
416
+ if plot and plotly_found:
417
+ import plotly.express as px
418
+
419
+ df = group[raw_data_cols]
420
+ df_sorted = df.sort_values(by=info_cols[y_axis_index])
421
+ # Melt DataFrame for plotting
422
+ df_melted = df_sorted.melt(
423
+ id_vars=info_cols[y_axis_index],
424
+ var_name="Configuration",
425
+ value_name=data_cols_to_compare[i],
426
+ )
427
+ title = (
428
+ data_cols_to_compare[i] + " vs " + info_cols[y_axis_index]
429
+ )
430
+ # Create Plotly line chart
431
+ fig = px.line(
432
+ df_melted,
433
+ x=info_cols[y_axis_index],
434
+ y=data_cols_to_compare[i],
435
+ color="Configuration",
436
+ title=title,
437
+ markers=True,
438
+ )
439
+
440
+ # ---- Add threshold lines based on metric name ----
441
+ if "ttft" in metric_name:
442
+ _add_limit_line(fig, args.ttft_max_ms, "TTFT limit")
443
+ elif (
444
+ "tpot" in metric_name
445
+ or "median" in metric_name
446
+ or "p99" in metric_name
447
+ ):
448
+ _add_limit_line(fig, args.tpot_max_ms, "TPOT limit")
449
+
450
+ # Export to HTML
451
+ text_file.write(
452
+ fig.to_html(full_html=True, include_plotlyjs="cdn")
453
+ )
454
+ sub_text_file.write(
455
+ fig.to_html(full_html=True, include_plotlyjs="cdn")
456
+ )
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+3
-1
@@ -63,9 +63,11 @@ serving_column_mapping = {
63
"mean_ttft_ms": "Mean TTFT (ms)",
64
"median_ttft_ms": "Median TTFT (ms)",
65
"p99_ttft_ms": "P99 TTFT (ms)",
66
+ "std_ttft_ms": "STD TTFT (ms)",
67
"mean_tpot_ms": "Mean TPOT (ms)",
68
"median_tpot_ms": "Median",
69
"p99_tpot_ms": "P99",
70
+ "std_tpot_ms": "STD TPOT (ms)",
71
"mean_itl_ms": "Mean ITL (ms)",
72
"median_itl_ms": "Median ITL (ms)",
73
"p99_itl_ms": "P99 ITL (ms)",
@@ -368,7 +370,7 @@ if __name__ == "__main__":
370
# The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...",
371
# we want to turn it into "8xGPUTYPE"
372
df["GPU"] = df["GPU"].apply(
371
- lambda x: f"{len(x.splitlines())}x{x.splitlines()[0]}"
373
+ lambda x: "{}x{}".format(len(x.split("\n")), x.split("\n")[0])
374
)
375
376
# get markdown tables
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+5
@@ -471,6 +471,11 @@ main() {
471
mkdir -p $RESULTS_FOLDER
472
QUICK_BENCHMARK_ROOT=../.buildkite/nightly-benchmarks/
473
474
+ # dump vllm info via vllm collect-env
475
+ env_output=$(vllm collect-env)
476
+
477
+ echo "$env_output" >"$RESULTS_FOLDER/vllm_env.txt"
478
+
479
# benchmarking
480
run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}"
481
run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}"
.buildkite/nightly-benchmarks/tests/latency-tests-cpu.json
+13
-17
@@ -1,28 +1,24 @@
1
[
2
{
3
- "test_name": "latency_llama8B_tp1",
3
+ "test_name": "latency_llama8B_tp2",
4
"environment_variables": {
5
+ "VLLM_RPC_TIMEOUT": 100000,
6
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
7
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
8
+ "VLLM_CPU_SGL_KERNEL": 1,
9
"VLLM_CPU_KVCACHE_SPACE": 40
10
},
11
"parameters": {
12
"model": "meta-llama/Llama-3.1-8B-Instruct",
10
- "tensor_parallel_size": 1,
11
- "load_format": "dummy",
12
- "num_iters_warmup": 5,
13
- "num_iters": 15
14
- }
15
- },
16
- {
17
- "test_name": "latency_llama8B_tp4",
18
- "environment_variables": {
19
- "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
20
- "VLLM_CPU_KVCACHE_SPACE": 40
21
- },
22
- "parameters": {
23
- "model": "meta-llama/Llama-3.1-8B-Instruct",
24
- "tensor_parallel_size": 4,
25
- "load_format": "dummy",
13
+ "tensor_parallel_size": 2,
14
+ "dtype": "bfloat16",
15
+ "distributed_executor_backend": "mp",
16
+ "block_size": 128,
17
+ "trust_remote_code": "",
18
+ "disable_log_stats": "",
19
+ "enforce_eager": "",
20
+ "max_num_batched_tokens": 2048,
21
+ "max_num_seqs": 256,
22
"num_iters_warmup": 5,
23
"num_iters": 15
24
}
.buildkite/nightly-benchmarks/tests/serving-tests-cpu-snc3.json
+203
@@ -95,6 +95,38 @@
95
"num_prompts": 200
96
}
97
},
98
+ {
99
+ "test_name": "serving_llama8B_bf16_tp4_sharegpt",
100
+ "qps_list": ["inf"],
101
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
102
+ "server_environment_variables": {
103
+ "VLLM_RPC_TIMEOUT": 100000,
104
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
105
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
106
+ "VLLM_CPU_SGL_KERNEL": 1,
107
+ "VLLM_CPU_KVCACHE_SPACE": 40
108
+ },
109
+ "server_parameters": {
110
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
111
+ "tensor_parallel_size": 4,
112
+ "dtype": "bfloat16",
113
+ "distributed_executor_backend": "mp",
114
+ "block_size": 128,
115
+ "trust_remote_code": "",
116
+ "disable_log_stats": "",
117
+ "enforce_eager": "",
118
+ "max_num_batched_tokens": 2048,
119
+ "max_num_seqs": 256,
120
+ "load_format": "dummy"
121
+ },
122
+ "client_parameters": {
123
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
124
+ "backend": "vllm",
125
+ "dataset_name": "sharegpt",
126
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
127
+ "num_prompts": 200
128
+ }
129
+ },
130
{
131
"test_name": "serving_llama8B_bf16_tp2pp3_sharegpt",
132
"qps_list": ["inf"],
@@ -233,6 +265,41 @@
265
"num_prompts": 1000
266
}
267
},
268
+ {
269
+ "test_name": "serving_llama8B_bf16_tp4_random_128_128",
270
+ "qps_list": ["inf"],
271
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
272
+ "server_environment_variables": {
273
+ "VLLM_RPC_TIMEOUT": 100000,
274
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
275
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
276
+ "VLLM_CPU_SGL_KERNEL": 1,
277
+ "VLLM_CPU_KVCACHE_SPACE": 40
278
+ },
279
+ "server_parameters": {
280
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
281
+ "tensor_parallel_size": 4,
282
+ "dtype": "bfloat16",
283
+ "distributed_executor_backend": "mp",
284
+ "block_size": 128,
285
+ "trust_remote_code": "",
286
+ "enable_chunked_prefill": "",
287
+ "disable_log_stats": "",
288
+ "enforce_eager": "",
289
+ "max_num_batched_tokens": 2048,
290
+ "max_num_seqs": 256,
291
+ "load_format": "dummy"
292
+ },
293
+ "client_parameters": {
294
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
295
+ "backend": "vllm",
296
+ "dataset_name": "random",
297
+ "random-input-len": 128,
298
+ "random-output-len": 128,
299
+ "ignore-eos": "",
300
+ "num_prompts": 1000
301
+ }
302
+ },
303
{
304
"test_name": "serving_llama8B_bf16_tp2pp3_random_128_128",
305
"qps_list": ["inf"],
@@ -365,6 +432,38 @@
432
"num_prompts": 200
433
}
434
},
435
+ {
436
+ "test_name": "serving_llama8B_int8_tp4_sharegpt",
437
+ "qps_list": ["inf"],
438
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
439
+ "server_environment_variables": {
440
+ "VLLM_RPC_TIMEOUT": 100000,
441
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
442
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
443
+ "VLLM_CPU_SGL_KERNEL": 1,
444
+ "VLLM_CPU_KVCACHE_SPACE": 40
445
+ },
446
+ "server_parameters": {
447
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
448
+ "tensor_parallel_size": 4,
449
+ "dtype": "bfloat16",
450
+ "distributed_executor_backend": "mp",
451
+ "block_size": 128,
452
+ "trust_remote_code": "",
453
+ "disable_log_stats": "",
454
+ "enforce_eager": "",
455
+ "max_num_batched_tokens": 2048,
456
+ "max_num_seqs": 256,
457
+ "load_format": "dummy"
458
+ },
459
+ "client_parameters": {
460
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
461
+ "backend": "vllm",
462
+ "dataset_name": "sharegpt",
463
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
464
+ "num_prompts": 200
465
+ }
466
+ },
467
{
468
"test_name": "serving_llama8B_int8_tp2pp3_sharegpt",
469
"qps_list": ["inf"],
@@ -503,6 +602,41 @@
602
"num_prompts": 1000
603
}
604
},
605
+ {
606
+ "test_name": "serving_llama8B_int8_tp4_random_128_128",
607
+ "qps_list": ["inf"],
608
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
609
+ "server_environment_variables": {
610
+ "VLLM_RPC_TIMEOUT": 100000,
611
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
612
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
613
+ "VLLM_CPU_SGL_KERNEL": 1,
614
+ "VLLM_CPU_KVCACHE_SPACE": 40
615
+ },
616
+ "server_parameters": {
617
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
618
+ "tensor_parallel_size": 4,
619
+ "dtype": "bfloat16",
620
+ "distributed_executor_backend": "mp",
621
+ "block_size": 128,
622
+ "trust_remote_code": "",
623
+ "enable_chunked_prefill": "",
624
+ "disable_log_stats": "",
625
+ "enforce_eager": "",
626
+ "max_num_batched_tokens": 2048,
627
+ "max_num_seqs": 256,
628
+ "load_format": "dummy"
629
+ },
630
+ "client_parameters": {
631
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
632
+ "backend": "vllm",
633
+ "dataset_name": "random",
634
+ "random-input-len": 128,
635
+ "random-output-len": 128,
636
+ "ignore-eos": "",
637
+ "num_prompts": 1000
638
+ }
639
+ },
640
{
641
"test_name": "serving_llama8B_int8_tp2pp3_random_128_128",
642
"qps_list": ["inf"],
@@ -638,6 +772,39 @@
772
"num_prompts": 200
773
}
774
},
775
+ {
776
+ "test_name": "serving_llama8B_int4_tp4_sharegpt",
777
+ "qps_list": ["inf"],
778
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
779
+ "server_environment_variables": {
780
+ "VLLM_RPC_TIMEOUT": 100000,
781
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
782
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
783
+ "VLLM_CPU_SGL_KERNEL": 1,
784
+ "VLLM_CPU_KVCACHE_SPACE": 40
785
+ },
786
+ "server_parameters": {
787
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
788
+ "quantization": "awq",
789
+ "tensor_parallel_size": 4,
790
+ "dtype": "bfloat16",
791
+ "distributed_executor_backend": "mp",
792
+ "block_size": 128,
793
+ "trust_remote_code": "",
794
+ "disable_log_stats": "",
795
+ "enforce_eager": "",
796
+ "max_num_batched_tokens": 2048,
797
+ "max_num_seqs": 256,
798
+ "load_format": "dummy"
799
+ },
800
+ "client_parameters": {
801
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
802
+ "backend": "vllm",
803
+ "dataset_name": "sharegpt",
804
+ "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
805
+ "num_prompts": 200
806
+ }
807
+ },
808
{
809
"test_name": "serving_llama8B_int4_tp2pp3_sharegpt",
810
"qps_list": ["inf"],
@@ -780,6 +947,42 @@
947
"num_prompts": 1000
948
}
949
},
950
+ {
951
+ "test_name": "serving_llama8B_int4_tp4_random_128_128",
952
+ "qps_list": ["inf"],
953
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
954
+ "server_environment_variables": {
955
+ "VLLM_RPC_TIMEOUT": 100000,
956
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
957
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
958
+ "VLLM_CPU_SGL_KERNEL": 1,
959
+ "VLLM_CPU_KVCACHE_SPACE": 40
960
+ },
961
+ "server_parameters": {
962
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
963
+ "quantization": "awq",
964
+ "tensor_parallel_size": 4,
965
+ "dtype": "bfloat16",
966
+ "distributed_executor_backend": "mp",
967
+ "block_size": 128,
968
+ "trust_remote_code": "",
969
+ "enable_chunked_prefill": "",
970
+ "disable_log_stats": "",
971
+ "enforce_eager": "",
972
+ "max_num_batched_tokens": 2048,
973
+ "max_num_seqs": 256,
974
+ "load_format": "dummy"
975
+ },
976
+ "client_parameters": {
977
+ "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
978
+ "backend": "vllm",
979
+ "dataset_name": "random",
980
+ "random-input-len": 128,
981
+ "random-output-len": 128,
982
+ "ignore-eos": "",
983
+ "num_prompts": 1000
984
+ }
985
+ },
986
{
987
"test_name": "serving_llama8B_int4_tp2pp3_random_128_128",
988
"qps_list": ["inf"],
.buildkite/nightly-benchmarks/tests/serving-tests-cpu.json
+128
-20
@@ -2,7 +2,7 @@
2
{
3
"test_name": "serving_llama8B_tp1_sharegpt",
4
"qps_list": [1, 4, 16, "inf"],
5
- "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
5
+ "max_concurrency_list": [32],
6
"server_environment_variables": {
7
"VLLM_RPC_TIMEOUT": 100000,
8
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
@@ -28,13 +28,13 @@
28
"backend": "vllm",
29
"dataset_name": "sharegpt",
30
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
31
- "num_prompts": 200
31
+ "num_prompts": 32
32
}
33
},
34
{
35
"test_name": "serving_llama8B_tp2_sharegpt",
36
"qps_list": [1, 4, 16, "inf"],
37
- "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
37
+ "max_concurrency_list": [32],
38
"server_environment_variables": {
39
"VLLM_RPC_TIMEOUT": 100000,
40
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
@@ -60,13 +60,13 @@
60
"backend": "vllm",
61
"dataset_name": "sharegpt",
62
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
63
- "num_prompts": 200
63
+ "num_prompts": 32
64
}
65
},
66
{
67
- "test_name": "serving_llama8B_tp4_sharegpt",
67
+ "test_name": "serving_llama8B_tp1_random_128_128",
68
"qps_list": [1, 4, 16, "inf"],
69
- "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
69
+ "max_concurrency_list": [32],
70
"server_environment_variables": {
71
"VLLM_RPC_TIMEOUT": 100000,
72
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
@@ -76,11 +76,12 @@
76
},
77
"server_parameters": {
78
"model": "meta-llama/Llama-3.1-8B-Instruct",
79
- "tensor_parallel_size": 4,
79
+ "tensor_parallel_size": 1,
80
"dtype": "bfloat16",
81
"distributed_executor_backend": "mp",
82
"block_size": 128,
83
"trust_remote_code": "",
84
+ "enable_chunked_prefill": "",
85
"disable_log_stats": "",
86
"enforce_eager": "",
87
"max_num_batched_tokens": 2048,
@@ -90,15 +91,122 @@
91
"client_parameters": {
92
"model": "meta-llama/Llama-3.1-8B-Instruct",
93
"backend": "vllm",
93
- "dataset_name": "sharegpt",
94
- "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
95
- "num_prompts": 200
94
+ "dataset_name": "random",
95
+ "random-input-len": 128,
96
+ "random-output-len": 128,
97
+ "ignore-eos": "",
98
+ "num_prompts": 32
99
+ }
100
+ },
101
+ {
102
+ "test_name": "serving_llama8B_tp2_random_128_128",
103
+ "qps_list": [1, 4, 16, "inf"],
104
+ "max_concurrency_list": [32],
105
+ "server_environment_variables": {
106
+ "VLLM_RPC_TIMEOUT": 100000,
107
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
108
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
109
+ "VLLM_CPU_SGL_KERNEL": 1,
110
+ "VLLM_CPU_KVCACHE_SPACE": 40
111
+ },
112
+ "server_parameters": {
113
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
114
+ "tensor_parallel_size": 2,
115
+ "dtype": "bfloat16",
116
+ "distributed_executor_backend": "mp",
117
+ "block_size": 128,
118
+ "trust_remote_code": "",
119
+ "enable_chunked_prefill": "",
120
+ "disable_log_stats": "",
121
+ "enforce_eager": "",
122
+ "max_num_batched_tokens": 2048,
123
+ "max_num_seqs": 256,
124
+ "load_format": "dummy"
125
+ },
126
+ "client_parameters": {
127
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
128
+ "backend": "vllm",
129
+ "dataset_name": "random",
130
+ "random-input-len": 128,
131
+ "random-output-len": 128,
132
+ "ignore-eos": "",
133
+ "num_prompts": 32
134
+ }
135
+ },
136
+ {
137
+ "test_name": "serving_llama8B_tp1_random_128_2048",
138
+ "qps_list": [1, 4, 16, "inf"],
139
+ "max_concurrency_list": [32],
140
+ "server_environment_variables": {
141
+ "VLLM_RPC_TIMEOUT": 100000,
142
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
143
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
144
+ "VLLM_CPU_SGL_KERNEL": 1,
145
+ "VLLM_CPU_KVCACHE_SPACE": 40
146
+ },
147
+ "server_parameters": {
148
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
149
+ "tensor_parallel_size": 1,
150
+ "dtype": "bfloat16",
151
+ "distributed_executor_backend": "mp",
152
+ "block_size": 128,
153
+ "trust_remote_code": "",
154
+ "enable_chunked_prefill": "",
155
+ "disable_log_stats": "",
156
+ "enforce_eager": "",
157
+ "max_num_batched_tokens": 2048,
158
+ "max_num_seqs": 256,
159
+ "load_format": "dummy"
160
+ },
161
+ "client_parameters": {
162
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
163
+ "backend": "vllm",
164
+ "dataset_name": "random",
165
+ "random-input-len": 128,
166
+ "random-output-len": 2048,
167
+ "ignore-eos": "",
168
+ "num_prompts": 32
169
+ }
170
+ },
171
+ {
172
+ "test_name": "serving_llama8B_tp2_random_128_2048",
173
+ "qps_list": [1, 4, 16, "inf"],
174
+ "max_concurrency_list": [32],
175
+ "server_environment_variables": {
176
+ "VLLM_RPC_TIMEOUT": 100000,
177
+ "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
178
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
179
+ "VLLM_CPU_SGL_KERNEL": 1,
180
+ "VLLM_CPU_KVCACHE_SPACE": 40
181
+ },
182
+ "server_parameters": {
183
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
184
+ "tensor_parallel_size": 2,
185
+ "dtype": "bfloat16",
186
+ "distributed_executor_backend": "mp",
187
+ "block_size": 128,
188
+ "trust_remote_code": "",
189
+ "enable_chunked_prefill": "",
190
+ "disable_log_stats": "",
191
+ "enforce_eager": "",
192
+ "max_num_batched_tokens": 2048,
193
+ "max_num_seqs": 256,
194
+ "load_format": "dummy"
195
+ },
196
+ "client_parameters": {
197
+ "model": "meta-llama/Llama-3.1-8B-Instruct",
198
+ "backend": "vllm",
199
+ "dataset_name": "random",
200
+ "random-input-len": 128,
201
+ "random-output-len": 2048,
202
+ "ignore-eos": "",
203
+ "num_prompts": 32
204
}
205
},
206
{
99
- "test_name": "serving_llama8B_tp4_random_1024_128",
207
+ "test_name": "serving_llama8B_tp1_random_2048_128",
208
"qps_list": [1, 4, 16, "inf"],
101
- "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
209
+ "max_concurrency_list": [32],
210
"server_environment_variables": {
211
"VLLM_RPC_TIMEOUT": 100000,
212
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
@@ -108,7 +216,7 @@
216
},
217
"server_parameters": {
218
"model": "meta-llama/Llama-3.1-8B-Instruct",
111
- "tensor_parallel_size": 4,
219
+ "tensor_parallel_size": 1,
220
"dtype": "bfloat16",
221
"distributed_executor_backend": "mp",
222
"block_size": 128,
@@ -124,16 +232,16 @@
232
"model": "meta-llama/Llama-3.1-8B-Instruct",
233
"backend": "vllm",
234
"dataset_name": "random",
127
- "random-input-len": 1024,
235
+ "random-input-len": 2048,
236
"random-output-len": 128,
237
"ignore-eos": "",
130
- "num_prompts": 100
238
+ "num_prompts": 32
239
}
240
},
241
{
134
- "test_name": "serving_llama8B_pp6_random_1024_128",
242
+ "test_name": "serving_llama8B_tp2_random_2048_128",
243
"qps_list": [1, 4, 16, "inf"],
136
- "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
244
+ "max_concurrency_list": [32],
245
"server_environment_variables": {
246
"VLLM_RPC_TIMEOUT": 100000,
247
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
@@ -143,7 +251,7 @@
251
},
252
"server_parameters": {
253
"model": "meta-llama/Llama-3.1-8B-Instruct",
146
- "pipeline_parallel_size": 6,
254
+ "tensor_parallel_size": 2,
255
"dtype": "bfloat16",
256
"distributed_executor_backend": "mp",
257
"block_size": 128,
@@ -159,10 +267,10 @@
267
"model": "meta-llama/Llama-3.1-8B-Instruct",
268
"backend": "vllm",
269
"dataset_name": "random",
162
- "random-input-len": 1024,
270
+ "random-input-len": 2048,
271
"random-output-len": 128,
272
"ignore-eos": "",
165
- "num_prompts": 100
273
+ "num_prompts": 32
274
}
275
}
276
]
.buildkite/nightly-benchmarks/tests/throughput-tests-cpu.json
+13
-18
@@ -1,29 +1,24 @@
1
[
2
{
3
- "test_name": "throughput_llama8B_tp1",
3
+ "test_name": "throughput_llama8B_tp2",
4
"environment_variables": {
5
+ "VLLM_RPC_TIMEOUT": 100000,
6
"VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
7
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
8
+ "VLLM_CPU_SGL_KERNEL": 1,
9
"VLLM_CPU_KVCACHE_SPACE": 40
10
},
11
"parameters": {
12
"model": "meta-llama/Llama-3.1-8B-Instruct",
10
- "tensor_parallel_size": 1,
11
- "load_format": "dummy",
12
- "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
13
- "num_prompts": 200,
14
- "backend": "vllm"
15
- }
16
- },
17
- {
18
- "test_name": "throughput_llama8B_tp4",
19
- "environment_variables": {
20
- "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
21
- "VLLM_CPU_KVCACHE_SPACE": 40
22
- },
23
- "parameters": {
24
- "model": "meta-llama/Llama-3.1-8B-Instruct",
25
- "tensor_parallel_size": 4,
26
- "load_format": "dummy",
13
+ "tensor_parallel_size": 2,
14
+ "dtype": "bfloat16",
15
+ "distributed_executor_backend": "mp",
16
+ "block_size": 128,
17
+ "trust_remote_code": "",
18
+ "disable_log_stats": "",
19
+ "enforce_eager": "",
20
+ "max_num_batched_tokens": 2048,
21
+ "max_num_seqs": 256,
22
"dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
23
"num_prompts": 200,
24
"backend": "vllm"