A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Save pytorch profiler output for latency benchmark (#1871)
* Save profiler output * Apply feedback from code review
Antoni Baum committed
Dec 5, 2023 at 20:55 UTC
05ff90b692a6cdac4d8c06e7a4a4606d1b8fe1d6
1 file changed
+25
-9
benchmarks/benchmark_latency.py
+25
-9
@@ -1,6 +1,8 @@
1
"""Benchmark the latency of processing a single batch of requests."""
2
import argparse
3
import time
4
+from pathlib import Path
5
+from typing import Optional
6
7
import numpy as np
8
import torch
@@ -34,12 +36,15 @@ def main(args: argparse.Namespace):
36
print(sampling_params)
37
dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
38
37
- def run_to_completion(profile: bool = False):
38
- if profile:
39
- with torch.profiler.profile(activities=[
40
- torch.profiler.ProfilerActivity.CPU,
41
- torch.profiler.ProfilerActivity.CUDA,
42
- ]) as p:
39
+ def run_to_completion(profile_dir: Optional[str] = None):
40
+ if profile_dir:
41
+ with torch.profiler.profile(
42
+ activities=[
43
+ torch.profiler.ProfilerActivity.CPU,
44
+ torch.profiler.ProfilerActivity.CUDA,
45
+ ],
46
+ on_trace_ready=torch.profiler.tensorboard_trace_handler(
47
+ str(profile_dir))) as p:
48
llm.generate(prompt_token_ids=dummy_prompt_token_ids,
49
sampling_params=sampling_params,
50
use_tqdm=False)
@@ -54,11 +59,14 @@ def main(args: argparse.Namespace):
59
return latency
60
61
print("Warming up...")
57
- run_to_completion(profile=False)
62
+ run_to_completion(profile_dir=None)
63
64
if args.profile:
60
- print("Profiling...")
61
- run_to_completion(profile=True)
65
+ profile_dir = args.profile_result_dir
66
+ if not profile_dir:
67
+ profile_dir = Path(".") / "vllm_benchmark_result" / f"latency_result_{time.time()}"
68
+ print(f"Profiling (results will be saved to '{profile_dir}')...")
69
+ run_to_completion(profile_dir=args.profile_result_dir)
70
return
71
72
# Benchmark.
@@ -107,5 +115,13 @@ if __name__ == '__main__':
115
'--profile',
116
action='store_true',
117
help='profile the generation process of a single batch')
118
+ parser.add_argument(
119
+ '--profile-result-dir',
120
+ type=str,
121
+ default=None,
122
+ help=(
123
+ 'path to save the pytorch profiler output. Can be visualized '
124
+ 'with ui.perfetto.dev or Tensorboard.'
125
+ ))
126
args = parser.parse_args()
127
main(args)