A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmark] Use truncation by default for pooling benchmarks (#26992)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Oct 16, 2025 at 16:02 UTC
17838e50efc9e686fffc901dfed811b15b7d614a
1 file changed
+9
-7
vllm/benchmarks/lib/endpoint_request_func.py
+9
-7
@@ -527,6 +527,9 @@ async def async_request_openai_embeddings(
527
if request_func_input.model_name
528
else request_func_input.model,
529
"input": request_func_input.prompt,
530
+ # Many embedding models have short context length,
531
+ # this is to avoid dropping some of the requests.
532
+ "truncate_prompt_tokens": -1,
533
}
534
_update_payload_common(payload, request_func_input)
535
@@ -564,6 +567,9 @@ async def async_request_vllm_rerank(
567
else request_func_input.model,
568
"query": request_func_input.prompt[0],
569
"documents": request_func_input.prompt[1:],
570
+ # Many reranker models have short context length,
571
+ # this is to avoid dropping some of the requests.
572
+ "truncate_prompt_tokens": -1,
573
}
574
575
headers = {
@@ -599,6 +605,9 @@ async def async_request_openai_embeddings_chat(
605
"messages": [
606
{"role": "user", "content": content},
607
],
608
+ # Many embedding models have short context length,
609
+ # this is to avoid dropping some of the requests.
610
+ "truncate_prompt_tokens": -1,
611
}
612
_update_payload_common(payload, request_func_input)
613
@@ -634,13 +643,6 @@ def _preprocess_clip(request_func_input: RequestFuncInput):
643
# Image input
644
request_func_input.prompt = ""
645
637
- # max_model_len=77 is too short for most datasets,
638
- # so by default we truncate the prompt to max_model_len
639
- if request_func_input.extra_body is None:
640
- request_func_input.extra_body = {}
641
- if "truncate_prompt_tokens" not in request_func_input.extra_body:
642
- request_func_input.extra_body["truncate_prompt_tokens"] = -1
643
-
646
647
def _preprocess_vlm2vec(request_func_input: RequestFuncInput):
648
if request_func_input.multi_modal_content: