A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
[Bugfix][benchmarks] Fix input token calculation for rerank benchmark metrics (#30596)
Signed-off-by: vensen <vensenmu@gmail.com>
Vensen committed
Dec 14, 2025 at 22:57 UTC
add4b0ca448e0b053a76b7db215aee0e797786d3
3 files changed
+7
-2
vllm/benchmarks/serve.py
+3
-1
@@ -235,7 +235,9 @@ async def get_request(
235
236
237
def calculate_metrics_for_embeddings(
238
- outputs: list[RequestFuncOutput], dur_s: float, selected_percentiles: list[float]
238
+ outputs: list[RequestFuncOutput],
239
+ dur_s: float,
240
+ selected_percentiles: list[float],
241
) -> EmbedBenchmarkMetrics:
242
"""Calculate the metrics for the embedding requests.
243
vllm/entrypoints/pooling/score/protocol.py
+1
@@ -120,6 +120,7 @@ class RerankResult(BaseModel):
120
121
122
class RerankUsage(BaseModel):
123
+ prompt_tokens: int
124
total_tokens: int
125
126
vllm/entrypoints/pooling/score/serving.py
+3
-1
@@ -502,5 +502,7 @@ class ServingScores(OpenAIServing):
502
id=request_id,
503
model=model_name,
504
results=results,
505
- usage=RerankUsage(total_tokens=num_prompt_tokens),
505
+ usage=RerankUsage(
506
+ total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
507
+ ),
508
)