@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05ff90b692
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Save pytorch profiler output for latency benchmark (#1871)

* Save profiler output * Apply feedback from code review

Antoni Baum committed Dec 5, 2023 at 20:55 UTC 05ff90b692a6cdac4d8c06e7a4a4606d1b8fe1d6
1 file changed +25 -9
benchmarks/benchmark_latency.py
+25 -9
@@ -1,6 +1,8 @@
1 """Benchmark the latency of processing a single batch of requests."""
2 import argparse
3 import time
4 +from pathlib import Path
5 +from typing import Optional
6
7 import numpy as np
8 import torch
@@ -34,12 +36,15 @@ def main(args: argparse.Namespace):
36 print(sampling_params)
37 dummy_prompt_token_ids = [[0] * args.input_len] * args.batch_size
38
37 - def run_to_completion(profile: bool = False):
38 - if profile:
39 - with torch.profiler.profile(activities=[
40 - torch.profiler.ProfilerActivity.CPU,
41 - torch.profiler.ProfilerActivity.CUDA,
42 - ]) as p:
39 + def run_to_completion(profile_dir: Optional[str] = None):
40 + if profile_dir:
41 + with torch.profiler.profile(
42 + activities=[
43 + torch.profiler.ProfilerActivity.CPU,
44 + torch.profiler.ProfilerActivity.CUDA,
45 + ],
46 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
47 + str(profile_dir))) as p:
48 llm.generate(prompt_token_ids=dummy_prompt_token_ids,
49 sampling_params=sampling_params,
50 use_tqdm=False)
@@ -54,11 +59,14 @@ def main(args: argparse.Namespace):
59 return latency
60
61 print("Warming up...")
57 - run_to_completion(profile=False)
62 + run_to_completion(profile_dir=None)
63
64 if args.profile:
60 - print("Profiling...")
61 - run_to_completion(profile=True)
65 + profile_dir = args.profile_result_dir
66 + if not profile_dir:
67 + profile_dir = Path(".") / "vllm_benchmark_result" / f"latency_result_{time.time()}"
68 + print(f"Profiling (results will be saved to '{profile_dir}')...")
69 + run_to_completion(profile_dir=args.profile_result_dir)
70 return
71
72 # Benchmark.
@@ -107,5 +115,13 @@ if __name__ == '__main__':
115 '--profile',
116 action='store_true',
117 help='profile the generation process of a single batch')
118 + parser.add_argument(
119 + '--profile-result-dir',
120 + type=str,
121 + default=None,
122 + help=(
123 + 'path to save the pytorch profiler output. Can be visualized '
124 + 'with ui.perfetto.dev or Tensorboard.'
125 + ))
126 args = parser.parse_args()
127 main(args)