A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[Misc] support nsys profile for bench latency (#29776)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
zhrrr committed
Dec 18, 2025 at 22:52 UTC
eee600c34f87f50e87c7268eab569af16c7c2d22
1 file changed
+14
-12
vllm/benchmarks/latency.py
+14
-12
@@ -79,10 +79,6 @@ def add_cli_args(parser: argparse.ArgumentParser):
79
80
def main(args: argparse.Namespace):
81
engine_args = EngineArgs.from_cli_args(args)
82
- if args.profile and not engine_args.profiler_config.profiler == "torch":
83
- raise ValueError(
84
- "The torch profiler is not enabled. Please provide profiler_config."
85
- )
82
83
# Lazy import to avoid importing LLM when the bench command is not selected.
84
from vllm import LLM, SamplingParams
@@ -125,8 +121,8 @@ def main(args: argparse.Namespace):
121
),
122
)
123
128
- def run_to_completion(profile_dir: str | None = None):
129
- if profile_dir:
124
+ def run_to_completion(do_profile: bool = False):
125
+ if do_profile:
126
llm.start_profile()
127
llm_generate()
128
llm.stop_profile()
@@ -139,18 +135,24 @@ def main(args: argparse.Namespace):
135
136
print("Warming up...")
137
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
142
- run_to_completion(profile_dir=None)
138
+ run_to_completion(do_profile=False)
139
140
if args.profile:
145
- profile_dir = engine_args.profiler_config.torch_profiler_dir
146
- print(f"Profiling (results will be saved to '{profile_dir}')...")
147
- run_to_completion(profile_dir=profile_dir)
141
+ profiler_config = engine_args.profiler_config
142
+ if profiler_config.profiler == "torch":
143
+ print(
144
+ "Profiling with torch profiler (results will be saved to"
145
+ f" {profiler_config.torch_profiler_dir})..."
146
+ )
147
+ elif profiler_config.profiler == "cuda":
148
+ print("Profiling with cuda profiler ...")
149
+ run_to_completion(do_profile=True)
150
return
151
152
# Benchmark.
153
latencies = []
152
- for _ in tqdm(range(args.num_iters), desc="Profiling iterations"):
153
- latencies.append(run_to_completion(profile_dir=None))
154
+ for _ in tqdm(range(args.num_iters), desc="Bench iterations"):
155
+ latencies.append(run_to_completion(do_profile=False))
156
latencies = np.array(latencies)
157
percentages = [10, 25, 50, 75, 90, 99]
158
percentiles = np.percentile(latencies, percentages)