A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Benchmark] Do not save detailed info to json by default (#14879)
Signed-off-by: simon-mo <simon.mo@hey.com>
Simon Mo committed
Mar 16, 2025 at 21:48 UTC
583a9778e0bc65b031bc3e430d8f13655f727ec7
2 files changed
+19
-1
benchmarks/backend_request_func.py
+4
-1
@@ -14,7 +14,8 @@ from tqdm.asyncio import tqdm
14
from transformers import (AutoTokenizer, PreTrainedTokenizer,
15
PreTrainedTokenizerFast)
16
17
-from vllm.model_executor.model_loader.weight_utils import get_lock
17
+# NOTE(simon): do not import vLLM here so the benchmark script
18
+# can run without vLLM installed.
19
20
AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
21
@@ -427,6 +428,8 @@ def get_model(pretrained_model_name_or_path: str) -> str:
428
if os.getenv('VLLM_USE_MODELSCOPE', 'False').lower() == 'true':
429
from modelscope import snapshot_download
430
431
+ from vllm.model_executor.model_loader.weight_utils import get_lock
432
+
433
# Use file lock to prevent multiple processes from
434
# downloading the same model weights at the same time.
435
with get_lock(pretrained_model_name_or_path):
benchmarks/benchmark_serving.py
+15
@@ -684,6 +684,15 @@ def main(args: argparse.Namespace):
684
"Invalid metadata format. Please use KEY=VALUE format."
685
)
686
687
+ if not args.save_detailed:
688
+ # Remove fields with too many data points
689
+ for field in [
690
+ "input_lens", "output_lens", "ttfts", "itls",
691
+ "generated_texts", "errors"
692
+ ]:
693
+ if field in result_json:
694
+ del result_json[field]
695
+
696
# Traffic
697
result_json["request_rate"] = (args.request_rate if args.request_rate
698
< float("inf") else "inf")
@@ -828,6 +837,12 @@ if __name__ == "__main__":
837
action="store_true",
838
help="Specify to save benchmark results to a json file",
839
)
840
+ parser.add_argument(
841
+ "--save-detailed",
842
+ action="store_true",
843
+ help="When saving the results, whether to include per request "
844
+ "information such as response, error, ttfs, tpots, etc.",
845
+ )
846
parser.add_argument(
847
"--metadata",
848
metavar="KEY=VALUE",