A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add profile option to latency benchmark script (#1839)
Woosuk Kwon committed
Nov 29, 2023 at 23:42 UTC
e74b1736a1f1673e7823a68442cbf574d4493390
1 file changed
+25
-14
benchmarks/benchmark_latency.py
+25
-14
@@ -12,7 +12,6 @@ from vllm import LLM, SamplingParams
12
def main(args: argparse.Namespace):
13
print(args)
14
15
- # Process all the requests in a single batch if possible.
15
# NOTE(woosuk): If the request cannot be processed in a single batch,
16
# the engine will automatically process the request in multiple batches.
17
llm = LLM(
@@ -21,7 +20,6 @@ def main(args: argparse.Namespace):
20
quantization=args.quantization,
21
tensor_parallel_size=args.tensor_parallel_size,
22
max_num_seqs=args.batch_size,
24
- max_num_batched_tokens=args.batch_size * args.input_len,
23
trust_remote_code=args.trust_remote_code,
24
dtype=args.dtype,
25
)
@@ -39,22 +37,31 @@ def main(args: argparse.Namespace):
37
38
def run_to_completion(profile: bool = False):
39
if profile:
42
- torch.cuda.cudart().cudaProfilerStart()
43
- start_time = time.perf_counter()
44
-
45
- llm.generate(prompt_token_ids=dummy_prompt_token_ids,
46
- sampling_params=sampling_params,
47
- use_tqdm=False)
48
-
49
- end_time = time.perf_counter()
50
- latency = end_time - start_time
51
- if profile:
52
- torch.cuda.cudart().cudaProfilerStop()
53
- return latency
40
+ with torch.profiler.profile(activities=[
41
+ torch.profiler.ProfilerActivity.CPU,
42
+ torch.profiler.ProfilerActivity.CUDA,
43
+ ]) as p:
44
+ llm.generate(prompt_token_ids=dummy_prompt_token_ids,
45
+ sampling_params=sampling_params,
46
+ use_tqdm=False)
47
+ print(p.key_averages())
48
+ else:
49
+ start_time = time.perf_counter()
50
+ llm.generate(prompt_token_ids=dummy_prompt_token_ids,
51
+ sampling_params=sampling_params,
52
+ use_tqdm=False)
53
+ end_time = time.perf_counter()
54
+ latency = end_time - start_time
55
+ return latency
56
57
print("Warming up...")
58
run_to_completion(profile=False)
59
60
+ if args.profile:
61
+ print("Profiling...")
62
+ run_to_completion(profile=True)
63
+ return
64
+
65
# Benchmark.
66
latencies = []
67
for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
@@ -97,5 +104,9 @@ if __name__ == '__main__':
104
'The "auto" option will use FP16 precision '
105
'for FP32 and FP16 models, and BF16 precision '
106
'for BF16 models.')
107
+ parser.add_argument(
108
+ '--profile',
109
+ action='store_true',
110
+ help='profile the generation process of a single batch')
111
args = parser.parse_args()
112
main(args)