A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 21:15 UTC
Support multiple image/audio embeddings per requests (#29988)
Signed-off-by: Jeremy Teboul <jeremyteboul@fb.com> Co-authored-by: Jeremy Teboul <jeremyteboul@fb.com>
jeremyteboul committed
Dec 6, 2025 at 20:34 UTC
dce6d229f7d405a4757aa8f0a76ba62f0e39eaa4
3 files changed
+198
-20
docs/features/multimodal_inputs.md
+8
-2
@@ -445,7 +445,7 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd
445
446
For Qwen3-VL, the `image_embeds` should contain both the base image embedding and deepstack features.
447
448
-#### Audio Embeddings
448
+#### Audio Embedding Inputs
449
450
You can pass pre-computed audio embeddings similar to image embeddings:
451
@@ -892,5 +892,11 @@ For Online Serving, you can also skip sending media if you expect cache hits wit
892
```
893
894
!!! note
895
- Only one message can contain `{"type": "image_embeds"}`.
895
+ Multiple messages can now contain `{"type": "image_embeds"}`, enabling you to pass multiple image embeddings in a single request (similar to regular images). The number of embeddings is limited by `--limit-mm-per-prompt`.
896
+
897
+ **Important**: The embedding shape format differs based on the number of embeddings:
898
+
899
+ - **Single embedding**: 3D tensor of shape `(1, feature_size, hidden_size)`
900
+ - **Multiple embeddings**: List of 2D tensors, each of shape `(feature_size, hidden_size)`
901
+
902
If used with a model that requires additional parameters, you must also provide a tensor for each of them, e.g. `image_grid_thw`, `image_sizes`, etc.
tests/entrypoints/test_chat_utils.py
+178
@@ -6,6 +6,7 @@ from collections.abc import Mapping
6
from typing import Literal
7
8
import pytest
9
+import torch
10
from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy
11
12
from vllm.assets.audio import AudioAsset
@@ -915,6 +916,183 @@ async def test_parse_chat_messages_audio_embeds_async(
916
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
917
918
919
+def test_parse_chat_messages_multiple_image_embeds(
920
+ phi3v_model_config_image_embeds,
921
+):
922
+ """Test that multiple image_embeds in a single message are now supported.
923
+
924
+ This test validates the fix for the limitation that previously only allowed
925
+ one message with {'type': 'image_embeds'}. Now multiple image embeddings
926
+ can be provided in a single request, similar to regular images.
927
+ """
928
+ # Create two sample image embedding tensors
929
+ image_embedding_1 = torch.randn(256, 1024)
930
+ image_embedding_2 = torch.randn(128, 1024)
931
+
932
+ # Encode them as base64 using the convenience function
933
+ base64_image_embedding_1 = tensor2base64(image_embedding_1)
934
+ base64_image_embedding_2 = tensor2base64(image_embedding_2)
935
+
936
+ conversation, mm_data, mm_uuids = parse_chat_messages(
937
+ [
938
+ {
939
+ "role": "user",
940
+ "content": [
941
+ {
942
+ "type": "image_embeds",
943
+ "image_embeds": base64_image_embedding_1,
944
+ },
945
+ {
946
+ "type": "image_embeds",
947
+ "image_embeds": base64_image_embedding_2,
948
+ },
949
+ {"type": "text", "text": "Describe these two images."},
950
+ ],
951
+ }
952
+ ],
953
+ phi3v_model_config_image_embeds,
954
+ content_format="string",
955
+ )
956
+
957
+ # Verify conversation structure
958
+ assert conversation == [
959
+ {
960
+ "role": "user",
961
+ "content": "<|image_1|>\n<|image_2|>\nDescribe these two images.",
962
+ }
963
+ ]
964
+
965
+ # Verify mm_data contains a list of embeddings (not a single embedding)
966
+ assert mm_data is not None
967
+ assert "image" in mm_data
968
+ assert isinstance(mm_data["image"], list)
969
+ assert len(mm_data["image"]) == 2
970
+
971
+ # Verify each embedding has the correct shape
972
+ assert isinstance(mm_data["image"][0], torch.Tensor)
973
+ assert mm_data["image"][0].shape == image_embedding_1.shape
974
+ assert isinstance(mm_data["image"][1], torch.Tensor)
975
+ assert mm_data["image"][1].shape == image_embedding_2.shape
976
+
977
+ # Verify UUIDs (None since we didn't provide any)
978
+ _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
979
+
980
+
981
+def test_parse_chat_messages_multiple_image_embeds_with_uuids(
982
+ phi3v_model_config_image_embeds,
983
+):
984
+ """Test multiple image_embeds with UUIDs.
985
+
986
+ This validates that UUIDs are properly tracked for multiple embeddings.
987
+ """
988
+ uuid1 = "image-uuid-1"
989
+ uuid2 = "image-uuid-2"
990
+
991
+ conversation, mm_data, mm_uuids = parse_chat_messages(
992
+ [
993
+ {
994
+ "role": "user",
995
+ "content": [
996
+ {
997
+ "type": "image_embeds",
998
+ "image_embeds": None,
999
+ "uuid": uuid1,
1000
+ },
1001
+ {
1002
+ "type": "image_embeds",
1003
+ "image_embeds": None,
1004
+ "uuid": uuid2,
1005
+ },
1006
+ {"type": "text", "text": "Compare these images."},
1007
+ ],
1008
+ }
1009
+ ],
1010
+ phi3v_model_config_image_embeds,
1011
+ content_format="string",
1012
+ )
1013
+
1014
+ # Verify conversation structure
1015
+ assert conversation == [
1016
+ {
1017
+ "role": "user",
1018
+ "content": "<|image_1|>\n<|image_2|>\nCompare these images.",
1019
+ }
1020
+ ]
1021
+
1022
+ # Verify mm_data contains a list with None values (UUID references)
1023
+ assert mm_data is not None
1024
+ assert "image" in mm_data
1025
+ assert isinstance(mm_data["image"], list)
1026
+ assert len(mm_data["image"]) == 2
1027
+ assert mm_data["image"][0] is None
1028
+ assert mm_data["image"][1] is None
1029
+
1030
+ # Verify UUIDs are correctly tracked
1031
+ _assert_mm_uuids(mm_uuids, 2, expected_uuids=[uuid1, uuid2])
1032
+
1033
+
1034
+@pytest.mark.asyncio
1035
+async def test_parse_chat_messages_multiple_image_embeds_async(
1036
+ phi3v_model_config_image_embeds,
1037
+):
1038
+ """Test multiple image_embeds with async parsing.
1039
+
1040
+ This validates the AsyncMultiModalItemTracker also supports multiple embeddings.
1041
+ """
1042
+ # Create two sample image embedding tensors
1043
+ image_embedding_1 = torch.randn(200, 768)
1044
+ image_embedding_2 = torch.randn(150, 768)
1045
+
1046
+ # Encode them as base64 using the convenience function
1047
+ base64_image_embedding_1 = tensor2base64(image_embedding_1)
1048
+ base64_image_embedding_2 = tensor2base64(image_embedding_2)
1049
+
1050
+ conversation, mm_future, mm_uuids = parse_chat_messages_futures(
1051
+ [
1052
+ {
1053
+ "role": "user",
1054
+ "content": [
1055
+ {
1056
+ "type": "image_embeds",
1057
+ "image_embeds": base64_image_embedding_1,
1058
+ },
1059
+ {
1060
+ "type": "image_embeds",
1061
+ "image_embeds": base64_image_embedding_2,
1062
+ },
1063
+ {"type": "text", "text": "What do these images show?"},
1064
+ ],
1065
+ }
1066
+ ],
1067
+ phi3v_model_config_image_embeds,
1068
+ content_format="string",
1069
+ )
1070
+
1071
+ # Verify conversation structure
1072
+ assert conversation == [
1073
+ {
1074
+ "role": "user",
1075
+ "content": "<|image_1|>\n<|image_2|>\nWhat do these images show?",
1076
+ }
1077
+ ]
1078
+
1079
+ # Await the future and verify mm_data
1080
+ mm_data = await mm_future
1081
+ assert mm_data is not None
1082
+ assert "image" in mm_data
1083
+ assert isinstance(mm_data["image"], list)
1084
+ assert len(mm_data["image"]) == 2
1085
+
1086
+ # Verify each embedding has the correct shape
1087
+ assert isinstance(mm_data["image"][0], torch.Tensor)
1088
+ assert mm_data["image"][0].shape == image_embedding_1.shape
1089
+ assert isinstance(mm_data["image"][1], torch.Tensor)
1090
+ assert mm_data["image"][1].shape == image_embedding_2.shape
1091
+
1092
+ # Verify UUIDs
1093
+ _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1094
+
1095
+
1096
@pytest.mark.asyncio
1097
async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
1098
phi3v_model_config_image_embeds,
vllm/entrypoints/chat_utils.py
+12
-18
@@ -694,16 +694,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
694
raise ValueError("Mixing raw image and embedding inputs is not allowed")
695
696
if "image_embeds" in uuids_by_modality:
697
- image_embeds_uuids = uuids_by_modality["image_embeds"]
698
- if len(image_embeds_uuids) > 1:
699
- raise ValueError("Only one message can have {'type': 'image_embeds'}")
697
mm_uuids["image"] = uuids_by_modality["image_embeds"]
698
if "image" in uuids_by_modality:
699
mm_uuids["image"] = uuids_by_modality["image"] # UUIDs of images
700
if "audio_embeds" in uuids_by_modality:
704
- audio_embeds_uuids = uuids_by_modality["audio_embeds"]
705
- if len(audio_embeds_uuids) > 1:
706
- raise ValueError("Only one message can have {'type': 'audio_embeds'}")
701
mm_uuids["audio"] = uuids_by_modality["audio_embeds"]
702
if "audio" in uuids_by_modality:
703
mm_uuids["audio"] = uuids_by_modality["audio"] # UUIDs of audios
@@ -729,16 +723,16 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
723
724
if "image_embeds" in items_by_modality:
725
image_embeds_lst = items_by_modality["image_embeds"]
732
- if len(image_embeds_lst) > 1:
733
- raise ValueError("Only one message can have {'type': 'image_embeds'}")
734
- mm_inputs["image"] = image_embeds_lst[0]
726
+ mm_inputs["image"] = (
727
+ image_embeds_lst if len(image_embeds_lst) != 1 else image_embeds_lst[0]
728
+ )
729
if "image" in items_by_modality:
730
mm_inputs["image"] = items_by_modality["image"] # A list of images
731
if "audio_embeds" in items_by_modality:
732
audio_embeds_lst = items_by_modality["audio_embeds"]
739
- if len(audio_embeds_lst) > 1:
740
- raise ValueError("Only one message can have {'type': 'audio_embeds'}")
741
- mm_inputs["audio"] = audio_embeds_lst[0]
733
+ mm_inputs["audio"] = (
734
+ audio_embeds_lst if len(audio_embeds_lst) != 1 else audio_embeds_lst[0]
735
+ )
736
if "audio" in items_by_modality:
737
mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
738
if "video" in items_by_modality:
@@ -771,16 +765,16 @@ class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
765
766
if "image_embeds" in items_by_modality:
767
image_embeds_lst = items_by_modality["image_embeds"]
774
- if len(image_embeds_lst) > 1:
775
- raise ValueError("Only one message can have {'type': 'image_embeds'}")
776
- mm_inputs["image"] = image_embeds_lst[0]
768
+ mm_inputs["image"] = (
769
+ image_embeds_lst if len(image_embeds_lst) != 1 else image_embeds_lst[0]
770
+ )
771
if "image" in items_by_modality:
772
mm_inputs["image"] = items_by_modality["image"] # A list of images
773
if "audio_embeds" in items_by_modality:
774
audio_embeds_lst = items_by_modality["audio_embeds"]
781
- if len(audio_embeds_lst) > 1:
782
- raise ValueError("Only one message can have {'type': 'audio_embeds'}")
783
- mm_inputs["audio"] = audio_embeds_lst[0]
775
+ mm_inputs["audio"] = (
776
+ audio_embeds_lst if len(audio_embeds_lst) != 1 else audio_embeds_lst[0]
777
+ )
778
if "audio" in items_by_modality:
779
mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
780
if "video" in items_by_modality: