@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e74b1736a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add profile option to latency benchmark script (#1839)

Woosuk Kwon committed Nov 29, 2023 at 23:42 UTC e74b1736a1f1673e7823a68442cbf574d4493390
1 file changed +25 -14
benchmarks/benchmark_latency.py
+25 -14
@@ -12,7 +12,6 @@ from vllm import LLM, SamplingParams
12 def main(args: argparse.Namespace):
13 print(args)
14
15 - # Process all the requests in a single batch if possible.
15 # NOTE(woosuk): If the request cannot be processed in a single batch,
16 # the engine will automatically process the request in multiple batches.
17 llm = LLM(
@@ -21,7 +20,6 @@ def main(args: argparse.Namespace):
20 quantization=args.quantization,
21 tensor_parallel_size=args.tensor_parallel_size,
22 max_num_seqs=args.batch_size,
24 - max_num_batched_tokens=args.batch_size * args.input_len,
23 trust_remote_code=args.trust_remote_code,
24 dtype=args.dtype,
25 )
@@ -39,22 +37,31 @@ def main(args: argparse.Namespace):
37
38 def run_to_completion(profile: bool = False):
39 if profile:
42 - torch.cuda.cudart().cudaProfilerStart()
43 - start_time = time.perf_counter()
44 -
45 - llm.generate(prompt_token_ids=dummy_prompt_token_ids,
46 - sampling_params=sampling_params,
47 - use_tqdm=False)
48 -
49 - end_time = time.perf_counter()
50 - latency = end_time - start_time
51 - if profile:
52 - torch.cuda.cudart().cudaProfilerStop()
53 - return latency
40 + with torch.profiler.profile(activities=[
41 + torch.profiler.ProfilerActivity.CPU,
42 + torch.profiler.ProfilerActivity.CUDA,
43 + ]) as p:
44 + llm.generate(prompt_token_ids=dummy_prompt_token_ids,
45 + sampling_params=sampling_params,
46 + use_tqdm=False)
47 + print(p.key_averages())
48 + else:
49 + start_time = time.perf_counter()
50 + llm.generate(prompt_token_ids=dummy_prompt_token_ids,
51 + sampling_params=sampling_params,
52 + use_tqdm=False)
53 + end_time = time.perf_counter()
54 + latency = end_time - start_time
55 + return latency
56
57 print("Warming up...")
58 run_to_completion(profile=False)
59
60 + if args.profile:
61 + print("Profiling...")
62 + run_to_completion(profile=True)
63 + return
64 +
65 # Benchmark.
66 latencies = []
67 for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
@@ -97,5 +104,9 @@ if __name__ == '__main__':
104 'The "auto" option will use FP16 precision '
105 'for FP32 and FP16 models, and BF16 precision '
106 'for BF16 models.')
107 + parser.add_argument(
108 + '--profile',
109 + action='store_true',
110 + help='profile the generation process of a single batch')
111 args = parser.parse_args()
112 main(args)