A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Misc] Benchmark Serving Script Support Appending Results (#17028)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Lucas Wilkinson committed
Apr 25, 2025 at 01:53 UTC
881f735827a466f1efd7c7100bd423e4081a2ce5
1 file changed
+20
-11
benchmarks/benchmark_serving.py
+20
-11
@@ -713,7 +713,7 @@ def main(args: argparse.Namespace):
713
))
714
715
# Save config and results to json
716
- if args.save_result:
716
+ if args.save_result or args.append_result:
717
result_json: dict[str, Any] = {}
718
719
# Setup
@@ -734,6 +734,14 @@ def main(args: argparse.Namespace):
734
raise ValueError(
735
"Invalid metadata format. Please use KEY=VALUE format."
736
)
737
+ # Traffic
738
+ result_json["request_rate"] = (args.request_rate if args.request_rate
739
+ < float("inf") else "inf")
740
+ result_json["burstiness"] = args.burstiness
741
+ result_json["max_concurrency"] = args.max_concurrency
742
+
743
+ # Merge with benchmark result
744
+ result_json = {**result_json, **benchmark_result}
745
746
if not args.save_detailed:
747
# Remove fields with too many data points
@@ -744,15 +752,6 @@ def main(args: argparse.Namespace):
752
if field in result_json:
753
del result_json[field]
754
747
- # Traffic
748
- result_json["request_rate"] = (args.request_rate if args.request_rate
749
- < float("inf") else "inf")
750
- result_json["burstiness"] = args.burstiness
751
- result_json["max_concurrency"] = args.max_concurrency
752
-
753
- # Merge with benchmark result
754
- result_json = {**result_json, **benchmark_result}
755
-
755
# Save to file
756
base_model_id = model_id.split("/")[-1]
757
max_concurrency_str = (f"-concurrency{args.max_concurrency}"
@@ -762,7 +761,12 @@ def main(args: argparse.Namespace):
761
file_name = args.result_filename
762
if args.result_dir:
763
file_name = os.path.join(args.result_dir, file_name)
765
- with open(file_name, "w", encoding='utf-8') as outfile:
764
+ with open(file_name,
765
+ mode="a+" if args.append_result else "w",
766
+ encoding='utf-8') as outfile:
767
+ # Append a newline.
768
+ if args.append_result and outfile.tell() != 0:
769
+ outfile.write("\n")
770
json.dump(result_json, outfile)
771
save_to_pytorch_benchmark_format(args, result_json, file_name)
772
@@ -894,6 +898,11 @@ if __name__ == "__main__":
898
help="When saving the results, whether to include per request "
899
"information such as response, error, ttfs, tpots, etc.",
900
)
901
+ parser.add_argument(
902
+ "--append-result",
903
+ action="store_true",
904
+ help="Append the benchmark result to the existing json file.",
905
+ )
906
parser.add_argument(
907
"--metadata",
908
metavar="KEY=VALUE",