A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
vllm bench serve shows num of failed requests (#26478)
Signed-off-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
Tomas Ruiz committed
Oct 17, 2025 at 04:55 UTC
965c5f4914595a96d3c2d6ee2d0ed9dd00a8d573
1 file changed
+10
vllm/benchmarks/serve.py
+10
@@ -64,6 +64,7 @@ class TaskType(Enum):
64
@dataclass
65
class BenchmarkMetrics:
66
completed: int
67
+ failed: int
68
total_input: int
69
total_output: int
70
request_throughput: float
@@ -97,6 +98,7 @@ class BenchmarkMetrics:
98
@dataclass
99
class EmbedBenchmarkMetrics:
100
completed: int
101
+ failed: int
102
total_input: int
103
request_throughput: float
104
total_token_throughput: float
@@ -239,12 +241,15 @@ def calculate_metrics_for_embeddings(
241
"""
242
total_input = 0
243
completed = 0
244
+ failed = 0
245
e2els: list[float] = []
246
for i in range(len(outputs)):
247
if outputs[i].success:
248
e2els.append(outputs[i].latency)
249
completed += 1
250
total_input += outputs[i].prompt_len
251
+ else:
252
+ failed += 1
253
254
if completed == 0:
255
warnings.warn(
@@ -254,6 +259,7 @@ def calculate_metrics_for_embeddings(
259
)
260
metrics = EmbedBenchmarkMetrics(
261
completed=completed,
262
+ failed=failed,
263
total_input=total_input,
264
request_throughput=completed / dur_s,
265
total_token_throughput=total_input / dur_s,
@@ -366,6 +372,7 @@ def calculate_metrics(
372
373
# Find the time range across all successful requests
374
successful_outputs = [output for output in outputs if output.success]
375
+ failed_outputs = [output for output in outputs if not output.success]
376
if successful_outputs:
377
min_start_time = min(output.start_time for output in successful_outputs)
378
max_end_time = max(
@@ -427,6 +434,7 @@ def calculate_metrics(
434
435
metrics = BenchmarkMetrics(
436
completed=completed,
437
+ failed=len(failed_outputs),
438
total_input=total_input,
439
total_output=sum(actual_output_lens),
440
request_throughput=completed / dur_s,
@@ -734,6 +742,7 @@ async def benchmark(
742
743
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
744
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
745
+ print("{:<40} {:<10}".format("Failed requests:", metrics.failed))
746
if max_concurrency is not None:
747
print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))
748
if request_rate != float("inf"):
@@ -779,6 +788,7 @@ async def benchmark(
788
result = {
789
"duration": benchmark_duration,
790
"completed": metrics.completed,
791
+ "failed": metrics.failed,
792
"total_input_tokens": metrics.total_input,
793
"total_output_tokens": metrics.total_output,
794
"request_throughput": metrics.request_throughput,