@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f17a1a8f96
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Make Serving Benchmark More User-friendly (#5044)

Roger Wang committed May 25, 2024 at 10:28 UTC f17a1a8f9665bb237a3dddda7dc93f259e5e81e0
2 files changed +32 -3
benchmarks/backend_request_func.py
+6
@@ -89,6 +89,9 @@ async def async_request_tgi(
89 output.latency = most_recent_timestamp - st
90 output.success = True
91 output.generated_text = data["generated_text"]
92 + else:
93 + output.error = response.reason or ""
94 + output.success = False
95 except Exception:
96 output.success = False
97 exc_info = sys.exc_info()
@@ -276,6 +279,9 @@ async def async_request_openai_completions(
279 output.generated_text = generated_text
280 output.success = True
281 output.latency = latency
282 + else:
283 + output.error = response.reason or ""
284 + output.success = False
285 except Exception:
286 output.success = False
287 exc_info = sys.exc_info()
benchmarks/benchmark_serving.py
+26 -3
@@ -215,6 +215,11 @@ def calculate_metrics(
215 else:
216 actual_output_lens.append(0)
217
218 + if completed == 0:
219 + warnings.warn(
220 + "All requests failed. This is likely due to a misconfiguration "
221 + "on the benchmark arguments.",
222 + stacklevel=2)
223 metrics = BenchmarkMetrics(
224 completed=completed,
225 total_input=total_input,
@@ -226,9 +231,9 @@ def calculate_metrics(
231 1000, # ttfts is empty if streaming is not supported by backend
232 median_ttft_ms=np.median(ttfts or 0) * 1000,
233 p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
229 - mean_tpot_ms=np.mean(tpots) * 1000,
230 - median_tpot_ms=np.median(tpots) * 1000,
231 - p99_tpot_ms=np.percentile(tpots, 99) * 1000,
234 + mean_tpot_ms=np.mean(tpots or 0) * 1000,
235 + median_tpot_ms=np.median(tpots or 0) * 1000,
236 + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
237 )
238
239 return metrics, actual_output_lens
@@ -250,6 +255,24 @@ async def benchmark(
255 else:
256 raise ValueError(f"Unknown backend: {backend}")
257
258 + print("Starting initial single prompt test run...")
259 + test_prompt, test_prompt_len, test_output_len = input_requests[0]
260 + test_input = RequestFuncInput(
261 + model=model_id,
262 + prompt=test_prompt,
263 + api_url=api_url,
264 + prompt_len=test_prompt_len,
265 + output_len=test_output_len,
266 + best_of=best_of,
267 + use_beam_search=use_beam_search,
268 + )
269 + test_output = await request_func(request_func_input=test_input)
270 + if not test_output.success:
271 + raise ValueError(
272 + "Initial test run failed - Please make sure benchmark arguments "
273 + f"are correctly specified. Error: {test_output.error}")
274 + else:
275 + print("Initial test run completed. Starting main benchmark run...")
276 print(f"Traffic request rate: {request_rate}")
277
278 pbar = None if disable_tqdm else tqdm(total=len(input_requests))