A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 17:15 UTC
[bugfix] do not add extra linebreak for score/rerank with chat template (#38617)
Signed-off-by: augusto.yjh <augusto.yjh@antgroup.com> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: wang.yuqi <noooop@126.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Augusto Yao committed
Apr 1, 2026 at 12:50 UTC
ef53395e2ccbe049c332207e31838e03566b2ae8
3 files changed
+14
-6
tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py
+3
-3
@@ -234,7 +234,7 @@ async def test_score_api_queries_str_documents_image_url_plus_text_content(
234
assert score.id is not None
235
assert score.data is not None
236
assert len(score.data) == 1
237
- assert score.usage.prompt_tokens == 108
237
+ assert score.usage.prompt_tokens == 107
238
assert_score(
239
score.data[0].score, TEXT_VS_TEXT_PLUS_IMAGE, backend, "text_vs_text_plus_image"
240
)
@@ -264,7 +264,7 @@ async def test_score_api_queries_str_documents_list(
264
assert score.id is not None
265
assert score.data is not None
266
assert len(score.data) == 4
267
- assert score.usage.prompt_tokens == 368
267
+ assert score.usage.prompt_tokens == 367
268
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "list[0]_text_vs_text")
269
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "list[1]_text_vs_text")
270
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "list[2]_text_vs_image")
@@ -353,7 +353,7 @@ async def test_score_api_queries_list_documents_list(
353
assert score.id is not None
354
assert score.data is not None
355
assert len(score.data) == 4
356
- assert score.usage.prompt_tokens == 368
356
+ assert score.usage.prompt_tokens == 367
357
assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "paired[0]_text_vs_text")
358
assert_score(score.data[1].score, TEXT_VS_TEXT, backend, "paired[1]_text_vs_text")
359
assert_score(score.data[2].score, TEXT_VS_IMAGE, backend, "paired[2]_text_vs_image")
vllm/entrypoints/chat_utils.py
+10
-3
@@ -1187,6 +1187,7 @@ def _get_full_multimodal_text_prompt(
1187
placeholder_storage: dict[str, list],
1188
texts: list[str],
1189
interleave_strings: bool,
1190
+ multimodal_content_part_separator: str = "\n",
1191
) -> str:
1192
"""Combine multimodal prompts for a multimodal language model."""
1193
@@ -1232,9 +1233,11 @@ def _get_full_multimodal_text_prompt(
1233
# NOTE: Default behaviour: we always add missing placeholders
1234
# at the front of the prompt, if interleave_strings=False
1235
if text_prompt:
1235
- return "\n".join(missing_placeholders + [text_prompt])
1236
+ return multimodal_content_part_separator.join(
1237
+ missing_placeholders + [text_prompt]
1238
+ )
1239
else:
1237
- return "\n".join(missing_placeholders)
1240
+ return multimodal_content_part_separator.join(missing_placeholders)
1241
1242
1243
# No need to validate using Pydantic again
@@ -1384,6 +1387,7 @@ def _parse_chat_message_content_parts(
1387
wrap_dicts: bool,
1388
interleave_strings: bool,
1389
mm_processor_kwargs: dict[str, Any] | None = None,
1390
+ multimodal_content_part_separator="\n",
1391
) -> list[ConversationMessage]:
1392
content = list[_ContentPart]()
1393
@@ -1406,7 +1410,10 @@ def _parse_chat_message_content_parts(
1410
mm_placeholder_storage = mm_parser.mm_placeholder_storage()
1411
if mm_placeholder_storage:
1412
text_prompt = _get_full_multimodal_text_prompt(
1409
- mm_placeholder_storage, texts, interleave_strings
1413
+ mm_placeholder_storage,
1414
+ texts,
1415
+ interleave_strings,
1416
+ multimodal_content_part_separator=multimodal_content_part_separator,
1417
)
1418
else:
1419
text_prompt = "\n".join(texts)
vllm/entrypoints/pooling/scoring/utils.py
+1
@@ -150,6 +150,7 @@ def _parse_score_content(
150
mm_tracker=mm_tracker,
151
wrap_dicts=False,
152
interleave_strings=False,
153
+ multimodal_content_part_separator="",
154
)
155
156
if parse_res: