[Bugfix] Raise clear error on interleaved multimodal placeholder overcount (#53999)
Signed-off-by: hungh <hungh@nvidia.com>
Hung Hoang (黄兴） committed
Aug 27, 2026 at 16:49 UTC
acd0af90e854cd8a04060f3bf37bc9331a771f42
2 files changed
+31
-1
tests/entrypoints/unit_tests/test_chat_utils.py
+24
@@ -1663,6 +1663,30 @@ def test_parse_chat_messages_multiple_images_interleave(
1663
_assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1664
1665
1666
+def test_parse_chat_messages_interleave_placeholder_overcount_raises(
1667
+ phi3v_model_config_mm_interleaved,
1668
+ image_url,
1669
+):
1670
+ # Only one image is supplied, but the user's text also contains the internal
1671
+ # image sentinel, so the interleaved prompt references more image
1672
+ # placeholders than actual images. This must surface as a clear validation
1673
+ # error rather than an IndexError from popping an empty list.
1674
+ with pytest.raises(VLLMValidationError, match="placeholders in input prompt"):
1675
+ parse_chat_messages(
1676
+ [
1677
+ {
1678
+ "role": "user",
1679
+ "content": [
1680
+ {"type": "image_url", "image_url": {"url": image_url}},
1681
+ {"type": "text", "text": "<##IMAGE##>"},
1682
+ ],
1683
+ }
1684
+ ],
1685
+ phi3v_model_config_mm_interleaved,
1686
+ content_format="string",
1687
+ )
1688
+
1689
+
1690
@pytest.mark.asyncio
1691
async def test_parse_chat_messages_multiple_images_interleave_async(
1692
phi3v_model_config_mm_interleaved,
vllm/entrypoints/chat_utils.py
+7
-1
@@ -1407,7 +1407,13 @@ def _get_interleaved_text_prompt(
1407
) -> str:
1408
for idx, elem in enumerate(texts):
1409
if elem in placeholder_storage:
1410
- texts[idx] = placeholder_storage[elem].pop(0)
1410
+ slots = placeholder_storage[elem]
1411
+ if not slots:
1412
+ raise VLLMValidationError(
1413
+ f"Found more '{elem}' placeholders in input prompt than "
1414
+ "actual multimodal data items."
1415
+ )
1416
+ texts[idx] = slots.pop(0)
1417
1418
return "\n".join(texts)
1419