A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 07:30 UTC
Indicate compile mode in the benchmark results (#32990)
Signed-off-by: Huy Do <huydhn@gmail.com>
Huy Do committed
Jan 30, 2026 at 12:34 UTC
9ca66ecc1012418f6c6d865239835a0adfcae719
2 files changed
+42
-3
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+8
-3
@@ -393,6 +393,11 @@ run_serving_tests() {
393
fi
394
fi
395
396
+ # save the compilation mode and optimization level on the serving results
397
+ # whenever they are set
398
+ compilation_config_mode=$(echo "$server_params" | jq -r '."compilation_config.mode" // empty')
399
+ optimization_level=$(echo "$server_params" | jq -r '.optimization_level // empty')
400
+
401
# iterate over different QPS
402
for qps in $qps_list; do
403
# remove the surrounding single quote from qps
@@ -406,15 +411,15 @@ run_serving_tests() {
411
for max_concurrency in $max_concurrency_list; do
412
new_test_name=$test_name"_qps_"$qps"_concurrency_"$max_concurrency
413
echo " new test name $new_test_name"
409
- # pass the tensor parallel size to the client so that it can be displayed
410
- # on the benchmark dashboard
414
+ # pass the tensor parallel size, the compilation mode, and the optimization
415
+ # level to the client so that they can be used on the benchmark dashboard
416
client_command="vllm bench serve \
417
--save-result \
418
--result-dir $RESULTS_FOLDER \
419
--result-filename ${new_test_name}.json \
420
--request-rate $qps \
421
--max-concurrency $max_concurrency \
417
- --metadata "tensor_parallel_size=$tp" \
422
+ --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \
423
$client_args $client_remote_args "
424
425
echo "Running test case $test_name with qps $qps"
vllm/benchmarks/lib/utils.py
+34
@@ -8,6 +8,32 @@ import os
8
from typing import Any
9
10
11
+def extract_field(
12
+ args: argparse.Namespace, extra_info: dict[str, Any], field_name: str
13
+) -> str:
14
+ if field_name in extra_info:
15
+ return extra_info[field_name]
16
+
17
+ v = args
18
+ # For example, args.compilation_config.mode
19
+ for nested_field in field_name.split("."):
20
+ if not hasattr(v, nested_field):
21
+ return ""
22
+ v = getattr(v, nested_field)
23
+ return v
24
+
25
+
26
+def use_compile(args: argparse.Namespace, extra_info: dict[str, Any]) -> bool:
27
+ """
28
+ Check if the benchmark is run with torch.compile
29
+ """
30
+ return not (
31
+ extract_field(args, extra_info, "compilation_config.mode") == "0"
32
+ or "eager" in getattr(args, "output_json", "")
33
+ or "eager" in getattr(args, "result_filename", "")
34
+ )
35
+
36
+
37
def convert_to_pytorch_benchmark_format(
38
args: argparse.Namespace, metrics: dict[str, list], extra_info: dict[str, Any]
39
) -> list:
@@ -26,6 +52,14 @@ def convert_to_pytorch_benchmark_format(
52
"name": "vLLM benchmark",
53
"extra_info": {
54
"args": vars(args),
55
+ "compilation_config.mode": extract_field(
56
+ args, extra_info, "compilation_config.mode"
57
+ ),
58
+ "optimization_level": extract_field(
59
+ args, extra_info, "optimization_level"
60
+ ),
61
+ # A boolean field used by vLLM benchmark HUD dashboard
62
+ "use_compile": use_compile(args, extra_info),
63
},
64
},
65
"model": {