| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Benchmark the latency of processing a single batch of requests.""" |
| 4 | |
| 5 | import argparse |
| 6 | import json |
| 7 | import os |
| 8 | import time |
| 9 | from typing import Any |
| 10 | |
| 11 | import numpy as np |
| 12 | from tqdm import tqdm |
| 13 | |
| 14 | from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json |
| 15 | from vllm.engine.arg_utils import EngineArgs |
| 16 | from vllm.inputs import TextPrompt, TokensPrompt |
| 17 | from vllm.sampling_params import BeamSearchParams |
| 18 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 19 | |
| 20 | |
| 21 | def save_to_pytorch_benchmark_format( |
| 22 | args: argparse.Namespace, results: dict[str, Any] |
| 23 | ) -> None: |
| 24 | pt_records = convert_to_pytorch_benchmark_format( |
| 25 | args=args, |
| 26 | metrics={"latency": results["latencies"]}, |
| 27 | extra_info={k: results[k] for k in ["avg_latency", "percentiles"]}, |
| 28 | ) |
| 29 | if pt_records: |
| 30 | pt_file = f"{os.path.splitext(args.output_json)[0]}.pytorch.json" |
| 31 | write_to_json(pt_file, pt_records) |
| 32 | |
| 33 | |
| 34 | def add_cli_args(parser: FlexibleArgumentParser): |
| 35 | parser.add_argument("--input-len", type=int, default=32) |
| 36 | parser.add_argument("--output-len", type=int, default=128) |
| 37 | parser.add_argument("--batch-size", type=int, default=8) |
| 38 | parser.add_argument( |
| 39 | "--n", |
| 40 | type=int, |
| 41 | default=1, |
| 42 | help="Number of generated sequences per prompt.", |
| 43 | ) |
| 44 | parser.add_argument("--use-beam-search", action="store_true") |
| 45 | parser.add_argument( |
| 46 | "--num-iters-warmup", |
| 47 | type=int, |
| 48 | default=10, |
| 49 | help="Number of iterations to run for warmup.", |
| 50 | ) |
| 51 | parser.add_argument( |
| 52 | "--num-iters", type=int, default=30, help="Number of iterations to run." |
| 53 | ) |
| 54 | parser.add_argument( |
| 55 | "--profile", |
| 56 | action="store_true", |
| 57 | help="profile the generation process of a single batch", |
| 58 | ) |
| 59 | parser.add_argument( |
| 60 | "--output-json", |
| 61 | type=str, |
| 62 | default=None, |
| 63 | help="Path to save the latency results in JSON format.", |
| 64 | ) |
| 65 | parser.add_argument( |
| 66 | "--disable-detokenize", |
| 67 | action="store_true", |
| 68 | help=( |
| 69 | "Do not detokenize responses (i.e. do not include " |
| 70 | "detokenization time in the latency measurement)" |
| 71 | ), |
| 72 | ) |
| 73 | |
| 74 | parser = EngineArgs.add_cli_args(parser) |
| 75 | # V1 enables prefix caching by default which skews the latency |
| 76 | # numbers. We need to disable prefix caching by default. |
| 77 | parser.set_defaults(enable_prefix_caching=False) |
| 78 | |
| 79 | |
| 80 | def main(args: argparse.Namespace): |
| 81 | engine_args = EngineArgs.from_cli_args(args) |
| 82 | |
| 83 | # Lazy import to avoid importing LLM when the bench command is not selected. |
| 84 | from vllm import LLM, SamplingParams |
| 85 | |
| 86 | # NOTE(woosuk): If the request cannot be processed in a single batch, |
| 87 | # the engine will automatically process the request in multiple batches. |
| 88 | llm = LLM.from_engine_args(engine_args) |
| 89 | assert llm.llm_engine.model_config.max_model_len >= ( |
| 90 | args.input_len + args.output_len |
| 91 | ), ( |
| 92 | "Please ensure that max_model_len is greater than" |
| 93 | " the sum of input_len and output_len." |
| 94 | ) |
| 95 | |
| 96 | sampling_params = SamplingParams( |
| 97 | n=args.n, |
| 98 | temperature=1.0, |
| 99 | top_p=1.0, |
| 100 | ignore_eos=True, |
| 101 | max_tokens=args.output_len, |
| 102 | detokenize=not args.disable_detokenize, |
| 103 | ) |
| 104 | dummy_prompt_token_ids = np.random.randint( |
| 105 | 10000, size=(args.batch_size, args.input_len) |
| 106 | ) |
| 107 | dummy_prompts: list[TokensPrompt | TextPrompt] = [ |
| 108 | TokensPrompt(prompt_token_ids=batch) |
| 109 | for batch in dummy_prompt_token_ids.tolist() |
| 110 | ] |
| 111 | |
| 112 | def llm_generate(): |
| 113 | if not args.use_beam_search: |
| 114 | llm.generate(dummy_prompts, sampling_params=sampling_params, use_tqdm=False) |
| 115 | else: |
| 116 | llm.beam_search( |
| 117 | dummy_prompts, |
| 118 | BeamSearchParams( |
| 119 | beam_width=args.n, |
| 120 | max_tokens=args.output_len, |
| 121 | ignore_eos=True, |
| 122 | ), |
| 123 | ) |
| 124 | |
| 125 | def run_to_completion(do_profile: bool = False): |
| 126 | if do_profile: |
| 127 | llm.start_profile() |
| 128 | llm_generate() |
| 129 | llm.stop_profile() |
| 130 | else: |
| 131 | start_time = time.perf_counter() |
| 132 | llm_generate() |
| 133 | end_time = time.perf_counter() |
| 134 | latency = end_time - start_time |
| 135 | return latency |
| 136 | |
| 137 | print("Warming up...") |
| 138 | for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"): |
| 139 | run_to_completion(do_profile=False) |
| 140 | |
| 141 | if args.profile: |
| 142 | profiler_config = engine_args.profiler_config |
| 143 | if profiler_config.profiler == "torch": |
| 144 | print( |
| 145 | "Profiling with torch profiler (results will be saved to" |
| 146 | f" {profiler_config.torch_profiler_dir})..." |
| 147 | ) |
| 148 | elif profiler_config.profiler == "cuda": |
| 149 | print("Profiling with cuda profiler ...") |
| 150 | elif profiler_config.profiler == "proton": |
| 151 | print( |
| 152 | "Profiling with Proton (results will be saved under" |
| 153 | f" {profiler_config.proton_profiler_dir})..." |
| 154 | ) |
| 155 | run_to_completion(do_profile=True) |
| 156 | return |
| 157 | |
| 158 | # Benchmark. |
| 159 | latencies = [] |
| 160 | for _ in tqdm(range(args.num_iters), desc="Bench iterations"): |
| 161 | latencies.append(run_to_completion(do_profile=False)) |
| 162 | latencies = np.array(latencies) |
| 163 | percentages = [10, 25, 50, 75, 90, 99] |
| 164 | percentiles = np.percentile(latencies, percentages) |
| 165 | print(f"Avg latency: {np.mean(latencies)} seconds") |
| 166 | for percentage, percentile in zip(percentages, percentiles): |
| 167 | print(f"{percentage}% percentile latency: {percentile} seconds") |
| 168 | |
| 169 | # Output JSON results if specified |
| 170 | if args.output_json: |
| 171 | results = { |
| 172 | "model_id": args.model, |
| 173 | "avg_latency": np.mean(latencies), |
| 174 | "latencies": latencies.tolist(), |
| 175 | "percentiles": dict(zip(percentages, percentiles.tolist())), |
| 176 | } |
| 177 | with open(args.output_json, "w") as f: |
| 178 | json.dump(results, f, indent=4) |
| 179 | save_to_pytorch_benchmark_format(args, results) |