A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
Fix some issues with benchmark data output (#13641)
Signed-off-by: Huy Do <huydhn@gmail.com>
Huy Do committed
Feb 23, 2025 at 18:23 UTC
e7ef74e26e4dcd7626db82e40baee2b7991c35ac
7 files changed
+61
-16
.buildkite/nightly-benchmarks/scripts/convert-results-json-to-markdown.py
+21
-6
@@ -84,8 +84,13 @@ if __name__ == "__main__":
84
# this result is generated via `benchmark_serving.py`
85
86
# attach the benchmarking command to raw_result
87
- with open(test_file.with_suffix(".commands")) as f:
88
- command = json.loads(f.read())
87
+ try:
88
+ with open(test_file.with_suffix(".commands")) as f:
89
+ command = json.loads(f.read())
90
+ except OSError as e:
91
+ print(e)
92
+ continue
93
+
94
raw_result.update(command)
95
96
# update the test name of this result
@@ -99,8 +104,13 @@ if __name__ == "__main__":
104
# this result is generated via `benchmark_latency.py`
105
106
# attach the benchmarking command to raw_result
102
- with open(test_file.with_suffix(".commands")) as f:
103
- command = json.loads(f.read())
107
+ try:
108
+ with open(test_file.with_suffix(".commands")) as f:
109
+ command = json.loads(f.read())
110
+ except OSError as e:
111
+ print(e)
112
+ continue
113
+
114
raw_result.update(command)
115
116
# update the test name of this result
@@ -121,8 +131,13 @@ if __name__ == "__main__":
131
# this result is generated via `benchmark_throughput.py`
132
133
# attach the benchmarking command to raw_result
124
- with open(test_file.with_suffix(".commands")) as f:
125
- command = json.loads(f.read())
134
+ try:
135
+ with open(test_file.with_suffix(".commands")) as f:
136
+ command = json.loads(f.read())
137
+ except OSError as e:
138
+ print(e)
139
+ continue
140
+
141
raw_result.update(command)
142
143
# update the test name of this result
.buildkite/nightly-benchmarks/scripts/run-performance-benchmarks.sh
+3
@@ -309,11 +309,14 @@ run_serving_tests() {
309
310
new_test_name=$test_name"_qps_"$qps
311
312
+ # pass the tensor parallel size to the client so that it can be displayed
313
+ # on the benchmark dashboard
314
client_command="python3 benchmark_serving.py \
315
--save-result \
316
--result-dir $RESULTS_FOLDER \
317
--result-filename ${new_test_name}.json \
318
--request-rate $qps \
319
+ --metadata "tensor_parallel_size=$tp" \
320
$client_args"
321
322
echo "Running test case $test_name with qps $qps"
.buildkite/nightly-benchmarks/tests/throughput-tests.json
+1
-1
@@ -32,4 +32,4 @@
32
"backend": "vllm"
33
}
34
}
35
-]
\ No newline at end of file
35
+]
benchmarks/benchmark_latency.py
+2
-3
@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional
11
12
import numpy as np
13
import torch
14
-from benchmark_utils import convert_to_pytorch_benchmark_format
14
+from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
15
from tqdm import tqdm
16
17
from vllm import LLM, SamplingParams
@@ -30,8 +30,7 @@ def save_to_pytorch_benchmark_format(args: argparse.Namespace,
30
for k in ["avg_latency", "percentiles"]})
31
if pt_records:
32
pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
33
- with open(pt_file, "w") as f:
34
- json.dump(pt_records, f)
33
+ write_to_json(pt_file, pt_records)
34
35
36
def main(args: argparse.Namespace):
benchmarks/benchmark_serving.py
+2
-3
@@ -56,7 +56,7 @@ try:
56
except ImportError:
57
from argparse import ArgumentParser as FlexibleArgumentParser
58
59
-from benchmark_utils import convert_to_pytorch_benchmark_format
59
+from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
60
61
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
62
@@ -841,8 +841,7 @@ def save_to_pytorch_benchmark_format(args: argparse.Namespace,
841
if pt_records:
842
# Don't use json suffix here as we don't want CI to pick it up
843
pt_file = f"{os.path.splitext(file_name)[0]}.pytorch.json"
844
- with open(pt_file, "w") as f:
845
- json.dump(pt_records, f)
844
+ write_to_json(pt_file, pt_records)
845
846
847
def main(args: argparse.Namespace):
benchmarks/benchmark_throughput.py
+2
-3
@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple
11
12
import torch
13
import uvloop
14
-from benchmark_utils import convert_to_pytorch_benchmark_format
14
+from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
15
from PIL import Image
16
from tqdm import tqdm
17
from transformers import (AutoModelForCausalLM, AutoTokenizer,
@@ -366,8 +366,7 @@ def save_to_pytorch_benchmark_format(args: argparse.Namespace,
366
if pt_records:
367
# Don't use json suffix here as we don't want CI to pick it up
368
pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json"
369
- with open(pt_file, "w") as f:
370
- json.dump(pt_records, f)
369
+ write_to_json(pt_file, pt_records)
370
371
372
def main(args: argparse.Namespace):
benchmarks/benchmark_utils.py
+30
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import argparse
4
+import json
5
+import math
6
import os
7
from typing import Any, Dict, List
8
@@ -34,6 +36,34 @@ def convert_to_pytorch_benchmark_format(args: argparse.Namespace,
36
"extra_info": extra_info,
37
},
38
}
39
+
40
+ tp = record["benchmark"]["extra_info"]["args"].get(
41
+ "tensor_parallel_size")
42
+ # Save tensor_parallel_size parameter if it's part of the metadata
43
+ if not tp and "tensor_parallel_size" in extra_info:
44
+ record["benchmark"]["extra_info"]["args"][
45
+ "tensor_parallel_size"] = extra_info["tensor_parallel_size"]
46
+
47
records.append(record)
48
49
return records
50
+
51
+
52
+class InfEncoder(json.JSONEncoder):
53
+
54
+ def clear_inf(self, o: Any):
55
+ if isinstance(o, dict):
56
+ return {k: self.clear_inf(v) for k, v in o.items()}
57
+ elif isinstance(o, list):
58
+ return [self.clear_inf(v) for v in o]
59
+ elif isinstance(o, float) and math.isinf(o):
60
+ return "inf"
61
+ return o
62
+
63
+ def iterencode(self, o: Any, *args, **kwargs) -> Any:
64
+ return super().iterencode(self.clear_inf(o), *args, **kwargs)
65
+
66
+
67
+def write_to_json(filename: str, records: List) -> None:
68
+ with open(filename, "w") as f:
69
+ json.dump(records, f, cls=InfEncoder)