@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 11b556878b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 9, 14:45 UTC

[Refactor] Use data parser for matching data items to multi-modal UUIDs (#32955)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 26, 2026 at 15:00 UTC 11b556878b958043e9c026919d4cba0236c85143
14 files changed +702 -605
docs/features/multimodal_inputs.md
+139 -117
@@ -20,67 +20,6 @@ To input multi-modal data, follow this schema in [vllm.inputs.PromptType][]:
20 - `prompt`: The prompt should follow the format that is documented on HuggingFace.
21 - `multi_modal_data`: This is a dictionary that follows the schema defined in [vllm.multimodal.inputs.MultiModalDataDict][].
22
23 -### Stable UUIDs for Caching (multi_modal_uuids)
24 -
25 -When using multi-modal inputs, vLLM normally hashes each media item by content to enable caching across requests. You can optionally pass `multi_modal_uuids` to provide your own stable IDs for each item so caching can reuse work across requests without rehashing the raw content.
26 -
27 -??? code
28 -
29 - ```python
30 - from vllm import LLM
31 - from PIL import Image
32 -
33 - # Qwen2.5-VL example with two images
34 - llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
35 -
36 - prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
37 - img_a = Image.open("/path/to/a.jpg")
38 - img_b = Image.open("/path/to/b.jpg")
39 -
40 - outputs = llm.generate({
41 - "prompt": prompt,
42 - "multi_modal_data": {"image": [img_a, img_b]},
43 - # Provide stable IDs for caching.
44 - # Requirements (matched by this example):
45 - # - Include every modality present in multi_modal_data.
46 - # - For lists, provide the same number of entries.
47 - # - Use None to fall back to content hashing for that item.
48 - "multi_modal_uuids": {"image": ["sku-1234-a", None]},
49 - })
50 -
51 - for o in outputs:
52 - print(o.outputs[0].text)
53 - ```
54 -
55 -Using UUIDs, you can also skip sending media data entirely if you expect cache hits for respective items. Note that the request will fail if the skipped media doesn't have a corresponding UUID, or if the UUID fails to hit the cache.
56 -
57 -??? code
58 -
59 - ```python
60 - from vllm import LLM
61 - from PIL import Image
62 -
63 - # Qwen2.5-VL example with two images
64 - llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
65 -
66 - prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
67 - img_b = Image.open("/path/to/b.jpg")
68 -
69 - outputs = llm.generate({
70 - "prompt": prompt,
71 - "multi_modal_data": {"image": [None, img_b]},
72 - # Since img_a is expected to be cached, we can skip sending the actual
73 - # image entirely.
74 - "multi_modal_uuids": {"image": ["sku-1234-a", None]},
75 - })
76 -
77 - for o in outputs:
78 - print(o.outputs[0].text)
79 - ```
80 -
81 -!!! warning
82 - If both multimodal processor caching and prefix caching are disabled, user-provided `multi_modal_uuids` are ignored.
83 -
23 ### Image Inputs
24
25 You can pass a single image to the `'image'` field of the multi-modal dictionary, as shown in the following examples:
@@ -397,7 +336,8 @@ No manual conversion is needed - vLLM handles the channel normalization automati
336 ### Embedding Inputs
337
338 To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
400 -pass a tensor of shape `(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.
339 +pass a tensor of shape `(..., hidden_size of LM)` to the corresponding field of the multi-modal dictionary.
340 +The exact shape depends on the model being used.
341
342 You must enable this feature via `enable_mm_embeds=True`.
343
@@ -418,8 +358,7 @@ You must enable this feature via `enable_mm_embeds=True`.
358 # Refer to the HuggingFace repo for the correct format to use
359 prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
360
421 - # Embeddings for single image
422 - # torch.Tensor of shape (1, image_feature_size, hidden_size of LM)
361 + # For most models, `image_embeds` has shape: (num_images, image_feature_size, hidden_size)
362 image_embeds = torch.load(...)
363
364 outputs = llm.generate({
@@ -430,21 +369,8 @@ You must enable this feature via `enable_mm_embeds=True`.
369 for o in outputs:
370 generated_text = o.outputs[0].text
371 print(generated_text)
433 - ```
434 -
435 -For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embeddings:
436 -
437 -??? code
438 -
439 - ```python
440 - # Construct the prompt based on your model
441 - prompt = ...
372
443 - # Embeddings for multiple images
444 - # torch.Tensor of shape (num_images, image_feature_size, hidden_size of LM)
445 - image_embeds = torch.load(...)
446 -
447 - # Qwen2-VL
373 + # Additional examples for models that require extra fields
374 llm = LLM(
375 "Qwen/Qwen2-VL-2B-Instruct",
376 limit_mm_per_prompt={"image": 4},
@@ -452,13 +378,15 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd
378 )
379 mm_data = {
380 "image": {
455 - "image_embeds": image_embeds,
381 + # Shape: (total_feature_size, hidden_size)
382 + # total_feature_size = sum(image_feature_size for image in images)
383 + "image_embeds": torch.load(...),
384 + # Shape: (num_images, 3)
385 # image_grid_thw is needed to calculate positional encoding.
457 - "image_grid_thw": torch.load(...), # torch.Tensor of shape (1, 3),
386 + "image_grid_thw": torch.load(...),
387 }
388 }
389
461 - # MiniCPM-V
390 llm = LLM(
391 "openbmb/MiniCPM-V-2_6",
392 trust_remote_code=True,
@@ -467,20 +395,14 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd
395 )
396 mm_data = {
397 "image": {
470 - "image_embeds": image_embeds,
398 + # Shape: (num_images, num_slices, hidden_size)
399 + # num_slices can differ for each image
400 + "image_embeds": [torch.load(...) for image in images],
401 + # Shape: (num_images, 2)
402 # image_sizes is needed to calculate details of the sliced image.
472 - "image_sizes": [image.size for image in images], # list of image sizes
403 + "image_sizes": [image.size for image in images],
404 }
405 }
475 -
476 - outputs = llm.generate({
477 - "prompt": prompt,
478 - "multi_modal_data": mm_data,
479 - })
480 -
481 - for o in outputs:
482 - generated_text = o.outputs[0].text
483 - print(generated_text)
406 ```
407
408 For Qwen3-VL, the `image_embeds` should contain both the base image embedding and deepstack features.
@@ -501,8 +423,8 @@ You can pass pre-computed audio embeddings similar to image embeddings:
423 # Refer to the HuggingFace repo for the correct format to use
424 prompt = "USER: <audio>\nWhat is in this audio?\nASSISTANT:"
425
504 - # Load pre-computed audio embeddings
505 - # torch.Tensor of shape (1, audio_feature_size, hidden_size of LM)
426 + # Load pre-computed audio embeddings, usually with shape:
427 + # (num_audios, audio_feature_size, hidden_size of LM)
428 audio_embeds = torch.load(...)
429
430 outputs = llm.generate({
@@ -515,6 +437,67 @@ You can pass pre-computed audio embeddings similar to image embeddings:
437 print(generated_text)
438 ```
439
440 +### Cached Inputs
441 +
442 +When using multi-modal inputs, vLLM normally hashes each media item by content to enable caching across requests. You can optionally pass `multi_modal_uuids` to provide your own stable IDs for each item so caching can reuse work across requests without rehashing the raw content.
443 +
444 +??? code
445 +
446 + ```python
447 + from vllm import LLM
448 + from PIL import Image
449 +
450 + # Qwen2.5-VL example with two images
451 + llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
452 +
453 + prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
454 + img_a = Image.open("/path/to/a.jpg")
455 + img_b = Image.open("/path/to/b.jpg")
456 +
457 + outputs = llm.generate({
458 + "prompt": prompt,
459 + "multi_modal_data": {"image": [img_a, img_b]},
460 + # Provide stable IDs for caching.
461 + # Requirements (matched by this example):
462 + # - Include every modality present in multi_modal_data.
463 + # - For lists, provide the same number of entries.
464 + # - Use None to fall back to content hashing for that item.
465 + "multi_modal_uuids": {"image": ["sku-1234-a", None]},
466 + })
467 +
468 + for o in outputs:
469 + print(o.outputs[0].text)
470 + ```
471 +
472 +Using UUIDs, you can also skip sending media data entirely if you expect cache hits for respective items. Note that the request will fail if the skipped media doesn't have a corresponding UUID, or if the UUID fails to hit the cache.
473 +
474 +??? code
475 +
476 + ```python
477 + from vllm import LLM
478 + from PIL import Image
479 +
480 + # Qwen2.5-VL example with two images
481 + llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
482 +
483 + prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
484 + img_b = Image.open("/path/to/b.jpg")
485 +
486 + outputs = llm.generate({
487 + "prompt": prompt,
488 + "multi_modal_data": {"image": [None, img_b]},
489 + # Since img_a is expected to be cached, we can skip sending the actual
490 + # image entirely.
491 + "multi_modal_uuids": {"image": ["sku-1234-a", None]},
492 + })
493 +
494 + for o in outputs:
495 + print(o.outputs[0].text)
496 + ```
497 +
498 +!!! warning
499 + If both multimodal processor caching and prefix caching are disabled, user-provided `multi_modal_uuids` are ignored.
500 +
501 ## Online Serving
502
503 Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions API](https://platform.openai.com/docs/api-reference/chat). Media inputs also support optional UUIDs users can provide to uniquely identify each media, which is used to cache the media results across requests.
@@ -879,7 +862,11 @@ Full example: [examples/online_serving/openai_chat_completion_client_for_multimo
862 ### Embedding Inputs
863
864 To input pre-computed embeddings belonging to a data type (i.e. image, video, or audio) directly to the language model,
882 -pass a tensor of shape `(num_items, feature_size, hidden_size of LM)` to the corresponding field of the multi-modal dictionary.
865 +pass a tensor of shape `(..., hidden_size of LM)` for each item to the corresponding field of the multi-modal dictionary.
866 +
867 +!!! important
868 + Unlike offline inference, the embeddings for each item must be passed separately
869 + in order for placeholder tokens to be applied correctly by the chat template.
870
871 You must enable this feature via the `--enable-mm-embeds` flag in `vllm serve`.
872
@@ -897,11 +884,6 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
884 ```python
885 from vllm.utils.serial_utils import tensor2base64
886
900 - image_embedding = torch.load(...)
901 - grid_thw = torch.load(...) # Required by Qwen/Qwen2-VL-2B-Instruct
902 -
903 - base64_image_embedding = tensor2base64(image_embedding)
904 -
887 client = OpenAI(
888 # defaults to os.environ.get("OPENAI_API_KEY")
889 api_key=openai_api_key,
@@ -912,29 +894,33 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
894 model = "llava-hf/llava-1.5-7b-hf"
895 embeds = {
896 "type": "image_embeds",
915 - "image_embeds": f"{base64_image_embedding}",
897 + "image_embeds": tensor2base64(torch.load(...)), # Shape: (image_feature_size, hidden_size)
898 "uuid": image_url, # Optional
899 }
900
919 - # Pass additional parameters (available to Qwen2-VL and MiniCPM-V)
901 +
902 + # Additional examples for models that require extra fields
903 model = "Qwen/Qwen2-VL-2B-Instruct"
904 embeds = {
905 "type": "image_embeds",
906 "image_embeds": {
924 - "image_embeds": f"{base64_image_embedding}", # Required
925 - "image_grid_thw": f"{base64_image_grid_thw}", # Required by Qwen/Qwen2-VL-2B-Instruct
907 + "image_embeds": tensor2base64(torch.load(...)), # Shape: (image_feature_size, hidden_size)
908 + "image_grid_thw": tensor2base64(torch.load(...)), # Shape: (3,)
909 },
910 "uuid": image_url, # Optional
911 }
912 +
913 model = "openbmb/MiniCPM-V-2_6"
914 embeds = {
915 "type": "image_embeds",
916 "image_embeds": {
933 - "image_embeds": f"{base64_image_embedding}", # Required
934 - "image_sizes": f"{base64_image_sizes}", # Required by openbmb/MiniCPM-V-2_6
917 + "image_embeds": tensor2base64(torch.load(...)), # Shape: (num_slices, hidden_size)
918 + "image_sizes": tensor2base64(torch.load(...)), # Shape: (2,)
919 },
920 "uuid": image_url, # Optional
921 }
922 +
923 + # Single image input
924 chat_completion = client.chat.completions.create(
925 messages=[
926 {
@@ -954,9 +940,55 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
940 ],
941 model=model,
942 )
943 +
944 + # Multi image input
945 + chat_completion = client.chat.completions.create(
946 + messages=[
947 + {
948 + "role": "system",
949 + "content": "You are a helpful assistant.",
950 + },
951 + {
952 + "role": "user",
953 + "content": [
954 + {
955 + "type": "text",
956 + "text": "What's in this image?",
957 + },
958 + embeds,
959 + embeds,
960 + ],
961 + },
962 + ],
963 + model=model,
964 + )
965 +
966 + # Multi image input (interleaved)
967 + chat_completion = client.chat.completions.create(
968 + messages=[
969 + {
970 + "role": "system",
971 + "content": "You are a helpful assistant.",
972 + },
973 + {
974 + "role": "user",
975 + "content": [
976 + embeds,
977 + {
978 + "type": "text",
979 + "text": "What's in this image?",
980 + },
981 + embeds,
982 + ],
983 + },
984 + ],
985 + model=model,
986 + )
987 ```
988
959 -For Online Serving, you can also skip sending media if you expect cache hits with provided UUIDs. You can do so by sending media like this:
989 +### Cached Inputs
990 +
991 +Just like with offline inference, you can skip sending media if you expect cache hits with provided UUIDs. You can do so by sending media like this:
992
993 ??? code
994
@@ -990,13 +1022,3 @@ For Online Serving, you can also skip sending media if you expect cache hits wit
1022 },
1023
1024 ```
993 -
994 -!!! note
995 - Multiple messages can now contain `{"type": "image_embeds"}`, enabling you to pass multiple image embeddings in a single request (similar to regular images). The number of embeddings is limited by `--limit-mm-per-prompt`.
996 -
997 - **Important**: The embedding shape format differs based on the number of embeddings:
998 -
999 - - **Single embedding**: 3D tensor of shape `(1, feature_size, hidden_size)`
1000 - - **Multiple embeddings**: List of 2D tensors, each of shape `(feature_size, hidden_size)`
1001 -
1002 - If used with a model that requires additional parameters, you must also provide a tensor for each of them, e.g. `image_grid_thw`, `image_sizes`, etc.
examples/pooling/plugin/prithvi_geospatial_mae_offline.py
+4 -2
@@ -59,8 +59,10 @@ class PrithviMAE:
59 input_data = input_data[0]
60
61 mm_data = {
62 - "pixel_values": input_data,
63 - "location_coords": location_coords,
62 + "image": {
63 + "pixel_values": input_data,
64 + "location_coords": location_coords,
65 + }
66 }
67
68 prompt = {"prompt_token_ids": [1], "multi_modal_data": mm_data}
tests/entrypoints/openai/test_vision_embeds.py
+103 -23
@@ -13,31 +13,10 @@ from vllm.utils.serial_utils import tensor2base64
13 from ...utils import RemoteOpenAIServer
14
15
16 -def _terratorch_dummy_messages():
17 - pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)
18 - location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)
19 -
20 - return [
21 - {
22 - "role": "user",
23 - "content": [
24 - {
25 - "type": "image_embeds",
26 - "image_embeds": {
27 - "pixel_values": tensor2base64(pixel_values),
28 - "location_coords": tensor2base64(location_coords),
29 - },
30 - }
31 - ],
32 - }
33 - ]
34 -
35 -
36 -@pytest.mark.asyncio
16 @pytest.mark.parametrize(
17 "model_name", ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"]
18 )
40 -def test_single_request(model_name: str):
19 +def test_single_content(model_name: str):
20 args = [
21 "--runner",
22 "pooling",
@@ -59,7 +38,24 @@ def test_single_request(model_name: str):
38 server.url_for("pooling"),
39 json={
40 "model": model_name,
62 - "messages": _terratorch_dummy_messages(),
41 + "messages": [
42 + {
43 + "role": "user",
44 + "content": [
45 + {
46 + "type": "image_embeds",
47 + "image_embeds": {
48 + "pixel_values": tensor2base64(
49 + torch.ones((6, 512, 512), dtype=torch.float16)
50 + ),
51 + "location_coords": tensor2base64(
52 + torch.ones((1, 2), dtype=torch.float16)
53 + ),
54 + },
55 + },
56 + ],
57 + }
58 + ],
59 "encoding_format": "base64",
60 },
61 )
@@ -69,3 +65,87 @@ def test_single_request(model_name: str):
65
66 np_response = np.frombuffer(base64.b64decode(output), dtype=np.float32)
67 assert len(np_response) == 524288
68 +
69 +
70 +@pytest.mark.parametrize("model_name", ["Qwen/Qwen3-VL-2B-Instruct"])
71 +def test_multi_content(model_name: str):
72 + args = [
73 + "--enforce-eager",
74 + "--max-num-seqs",
75 + "32",
76 + "--max-model-len",
77 + "8192",
78 + "--enable-mm-embeds",
79 + ]
80 +
81 + with RemoteOpenAIServer(model_name, args) as server:
82 + client = server.get_client()
83 +
84 + # Image only
85 + chat_completion = client.chat.completions.create(
86 + model=model_name,
87 + messages=[
88 + {
89 + "role": "user",
90 + "content": [
91 + {
92 + "type": "image_embeds",
93 + "image_embeds": {
94 + "image_embeds": tensor2base64(torch.zeros(220, 8192)),
95 + "image_grid_thw": tensor2base64(
96 + torch.tensor([1, 22, 40])
97 + ),
98 + },
99 + },
100 + {
101 + "type": "image_embeds",
102 + "image_embeds": {
103 + "image_embeds": tensor2base64(torch.zeros(220, 8192)),
104 + "image_grid_thw": tensor2base64(
105 + torch.tensor([1, 22, 40])
106 + ),
107 + },
108 + },
109 + ],
110 + }
111 + ],
112 + max_tokens=5,
113 + )
114 +
115 + assert chat_completion.id is not None
116 + assert len(chat_completion.choices) == 1
117 +
118 + # Interleaved text and image
119 + chat_completion = client.chat.completions.create(
120 + model=model_name,
121 + messages=[
122 + {
123 + "role": "user",
124 + "content": [
125 + {
126 + "type": "image_embeds",
127 + "image_embeds": {
128 + "image_embeds": tensor2base64(torch.zeros(220, 8192)),
129 + "image_grid_thw": tensor2base64(
130 + torch.tensor([1, 22, 40])
131 + ),
132 + },
133 + },
134 + {"type": "text", "text": "OCR:"},
135 + {
136 + "type": "image_embeds",
137 + "image_embeds": {
138 + "image_embeds": tensor2base64(torch.zeros(220, 8192)),
139 + "image_grid_thw": tensor2base64(
140 + torch.tensor([1, 22, 40])
141 + ),
142 + },
143 + },
144 + ],
145 + }
146 + ],
147 + max_tokens=5,
148 + )
149 +
150 + assert chat_completion.id is not None
151 + assert len(chat_completion.choices) == 1
tests/entrypoints/test_chat_utils.py
+36 -18
@@ -68,6 +68,16 @@ def phi3v_model_config_image_embeds():
68 )
69
70
71 +@pytest.fixture(scope="function")
72 +def qwen25omni_model_config_image_embeds():
73 + return ModelConfig(
74 + QWEN25OMNI_MODEL_ID,
75 + runner="generate",
76 + limit_mm_per_prompt={"image": 2},
77 + enable_mm_embeds=True,
78 + )
79 +
80 +
81 @pytest.fixture(scope="function")
82 def qwen2_audio_model_config():
83 return ModelConfig(
@@ -823,7 +833,8 @@ def test_parse_chat_messages_audio_embeds_with_string(
833 import torch
834
835 # Create a sample audio embedding tensor
826 - audio_embedding = torch.randn(1, 128, 768)
836 + hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
837 + audio_embedding = torch.randn(1, 128, hidden_size)
838
839 # Encode it as base64
840 base64_audio_embedding = tensor2base64(audio_embedding)
@@ -865,7 +876,8 @@ async def test_parse_chat_messages_audio_embeds_async(
876 import torch
877
878 # Create a sample audio embedding tensor
868 - audio_embedding = torch.randn(1, 128, 768)
879 + hidden_size = audio_embeds_model_config.get_inputs_embeds_size()
880 + audio_embedding = torch.randn(1, 128, hidden_size)
881
882 # Encode it as base64
883 base64_audio_embedding = tensor2base64(audio_embedding)
@@ -908,8 +920,9 @@ def test_parse_chat_messages_multiple_image_embeds(
920 can be provided in a single request, similar to regular images.
921 """
922 # Create two sample image embedding tensors
911 - image_embedding_1 = torch.randn(256, 1024)
912 - image_embedding_2 = torch.randn(128, 1024)
923 + hidden_size = phi3v_model_config_image_embeds.get_inputs_embeds_size()
924 + image_embedding_1 = torch.randn(256, hidden_size)
925 + image_embedding_2 = torch.randn(128, hidden_size)
926
927 # Encode them as base64 using the convenience function
928 base64_image_embedding_1 = tensor2base64(image_embedding_1)
@@ -1022,8 +1035,9 @@ async def test_parse_chat_messages_multiple_image_embeds_async(
1035 This validates the AsyncMultiModalItemTracker also supports multiple embeddings.
1036 """
1037 # Create two sample image embedding tensors
1025 - image_embedding_1 = torch.randn(200, 768)
1026 - image_embedding_2 = torch.randn(150, 768)
1038 + hidden_size = phi3v_model_config_image_embeds.get_inputs_embeds_size()
1039 + image_embedding_1 = torch.randn(200, hidden_size)
1040 + image_embedding_2 = torch.randn(150, hidden_size)
1041
1042 # Encode them as base64 using the convenience function
1043 base64_image_embedding_1 = tensor2base64(image_embedding_1)
@@ -1145,13 +1159,14 @@ def test_parse_chat_messages_empty_dict_image_embeds(
1159
1160
1161 def test_parse_chat_messages_multiple_dict_image_embeds(
1148 - phi3v_model_config_image_embeds,
1162 + qwen25omni_model_config_image_embeds,
1163 ):
1164 """Test that multiple dictionaries for image_embeds is handled without errors."""
1165 # Create two sample image embedding tensors
1166 batch_size = 2
1153 - image_embedding_1 = torch.randn(batch_size, 256, 1024)
1154 - image_embedding_2 = torch.randn(batch_size, 3)
1167 + hidden_size = qwen25omni_model_config_image_embeds.get_inputs_embeds_size()
1168 + image_embeds = torch.randn(batch_size * 220, hidden_size)
1169 + image_grid_thw = torch.tensor([[1, 22, 40] for _ in range(batch_size)])
1170
1171 conversation, mm_data, mm_uuids = parse_chat_messages(
1172 [
@@ -1161,18 +1176,20 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
1176 {
1177 "type": "image_embeds",
1178 "image_embeds": {
1164 - "image_embedding_1": tensor2base64(p),
1165 - "image_embedding_2": tensor2base64(i),
1179 + "image_embeds": tensor2base64(embeds),
1180 + "image_grid_thw": tensor2base64(grid_thw),
1181 },
1182 }
1168 - for p, i in zip(image_embedding_1, image_embedding_2)
1183 + for embeds, grid_thw in zip(
1184 + image_embeds.chunk(batch_size), image_grid_thw
1185 + )
1186 ]
1187 + [
1188 {"type": "text", "text": "Describe these two images."},
1189 ],
1190 }
1191 ],
1175 - phi3v_model_config_image_embeds,
1192 + qwen25omni_model_config_image_embeds,
1193 content_format="string",
1194 )
1195
@@ -1180,7 +1197,8 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
1197 assert conversation == [
1198 {
1199 "role": "user",
1183 - "content": "<|image_1|>\n<|image_2|>\nDescribe these two images.",
1200 + "content": "<|vision_start|><|IMAGE|><|vision_end|>\n"
1201 + "<|vision_start|><|IMAGE|><|vision_end|>\nDescribe these two images.",
1202 }
1203 ]
1204
@@ -1191,10 +1209,10 @@ def test_parse_chat_messages_multiple_dict_image_embeds(
1209 assert len(mm_data["image"]) == batch_size
1210
1211 # Verify each embedding has the correct shape
1194 - assert isinstance(mm_data["image"]["image_embedding_1"], torch.Tensor)
1195 - assert mm_data["image"]["image_embedding_1"].shape == image_embedding_1.shape
1196 - assert isinstance(mm_data["image"]["image_embedding_2"], torch.Tensor)
1197 - assert mm_data["image"]["image_embedding_2"].shape == image_embedding_2.shape
1212 + assert isinstance(mm_data["image"]["image_embeds"], torch.Tensor)
1213 + assert mm_data["image"]["image_embeds"].shape == image_embeds.shape
1214 + assert isinstance(mm_data["image"]["image_grid_thw"], torch.Tensor)
1215 + assert mm_data["image"]["image_grid_thw"].shape == image_grid_thw.shape
1216
1217 # Verify UUIDs (None since we didn't provide any)
1218 _assert_mm_uuids(mm_uuids, batch_size, expected_uuids=[None, None])
tests/models/multimodal/pooling/test_prithvi_mae.py
+6 -9
@@ -7,14 +7,6 @@ import torch
7 from ....conftest import VllmRunner
8
9
10 -def generate_test_mm_data():
11 - mm_data = {
12 - "pixel_values": torch.full((6, 512, 512), 1.0, dtype=torch.float16),
13 - "location_coords": torch.full((1, 2), 1.0, dtype=torch.float16),
14 - }
15 - return mm_data
16 -
17 -
10 def _run_test(
11 vllm_runner: type[VllmRunner],
12 model: str,
@@ -23,7 +15,12 @@ def _run_test(
15 {
16 # This model deals with no text input
17 "prompt_token_ids": [1],
26 - "multi_modal_data": generate_test_mm_data(),
18 + "multi_modal_data": {
19 + "image": {
20 + "pixel_values": torch.ones((6, 512, 512), dtype=torch.float16),
21 + "location_coords": torch.ones((1, 2), dtype=torch.float16),
22 + }
23 + },
24 }
25 for _ in range(10)
26 ]
tests/plugins/prithvi_io_processor_plugin/prithvi_io_processor/prithvi_processor.py
+4 -2
@@ -349,8 +349,10 @@ class PrithviMultimodalDataProcessor(IOProcessor):
349 {
350 "prompt_token_ids": [1],
351 "multi_modal_data": {
352 - "pixel_values": window.to(torch.float16)[0],
353 - "location_coords": location_coords.to(torch.float16),
352 + "image": {
353 + "pixel_values": window.to(torch.float16)[0],
354 + "location_coords": location_coords.to(torch.float16),
355 + }
356 },
357 }
358 )
tests/v1/engine/test_process_multi_modal_uuids.py
+26 -63
@@ -5,14 +5,8 @@ import pytest
5
6 from vllm.assets.image import ImageAsset
7 from vllm.assets.video import VideoAsset
8 -from vllm.config import (
9 - CacheConfig,
10 - DeviceConfig,
11 - ModelConfig,
12 - MultiModalConfig,
13 - VllmConfig,
14 -)
15 -from vllm.multimodal import MultiModalRegistry, MultiModalUUIDDict
8 +from vllm.config import CacheConfig, ModelConfig, VllmConfig
9 +from vllm.multimodal import MultiModalUUIDDict
10 from vllm.sampling_params import SamplingParams
11 from vllm.v1.engine.input_processor import InputProcessor
12
@@ -21,55 +15,26 @@ stop_pil_image = ImageAsset("stop_sign").pil_image
15 baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays
16
17
24 -def _mock_input_processor(
25 - monkeypatch, *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True
18 +def _build_input_processor(
19 + *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True
20 ) -> InputProcessor:
27 - """
28 - Create a Processor instance with minimal configuration suitable for unit
29 - tests without accessing external resources.
30 - """
31 - monkeypatch.setattr(
32 - ModelConfig, "try_get_generation_config", lambda self: {}, raising=True
33 - )
34 - monkeypatch.setattr(
35 - ModelConfig, "__post_init__", lambda self, *args: None, raising=True
36 - )
37 - monkeypatch.setattr(
38 - ModelConfig,
39 - "verify_with_parallel_config",
40 - lambda self, parallel_config: None,
41 - raising=True,
42 - )
43 - monkeypatch.setattr(
44 - MultiModalRegistry,
45 - "processor_cache_from_config",
46 - lambda self, vllm_config: None,
47 - raising=True,
48 - )
49 -
50 - monkeypatch.setattr(VllmConfig, "__post_init__", lambda self: None, raising=True)
51 -
21 model_config = ModelConfig(
53 - tokenizer="dummy",
22 + model="Qwen/Qwen2.5-VL-3B-Instruct",
23 skip_tokenizer_init=True,
24 max_model_len=128,
25 mm_processor_cache_gb=mm_cache_gb,
57 - generation_config="vllm",
26 )
59 - model_config.runner_type = "generate"
60 - model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb)
27
28 vllm_config = VllmConfig(
29 model_config=model_config,
30 cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),
65 - device_config=DeviceConfig(device="cpu"),
31 )
32
33 return InputProcessor(vllm_config)
34
35
71 -def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
72 - input_processor = _mock_input_processor(monkeypatch)
36 +def test_multi_modal_uuids_length_mismatch_raises():
37 + input_processor = _build_input_processor()
38
39 prompt = {
40 "prompt": "USER: <image>\nDescribe\nASSISTANT:",
@@ -78,7 +43,7 @@ def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
43 "multi_modal_uuids": {"image": ["hash_cherry"]},
44 }
45
81 - with pytest.raises(ValueError, match="must have same length as data"):
46 + with pytest.raises(ValueError, match="must have same length as"):
47 input_processor.process_inputs(
48 request_id="req-1",
49 prompt=prompt, # type: ignore[arg-type]
@@ -86,21 +51,21 @@ def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):
51 )
52
53
89 -def test_multi_modal_uuids_missing_modality_raises(monkeypatch):
90 - input_processor = _mock_input_processor(monkeypatch)
54 +def test_multi_modal_uuids_missing_modality_raises():
55 + input_processor = _build_input_processor()
56
57 prompt = {
58 "prompt": "USER: <image><video>\nDescribe\nASSISTANT:",
59 # Two modalities provided in data
60 "multi_modal_data": {
61 "image": [cherry_pil_image],
97 - "video": [baby_reading_np_ndarrays],
62 + "video": None,
63 },
64 # Only image uuids provided; video missing should raise
65 "multi_modal_uuids": {"image": ["hash_cherry"]},
66 }
67
103 - with pytest.raises(ValueError, match="must be provided if multi_modal_data"):
68 + with pytest.raises(ValueError, match="is empty but .* is missing"):
69 input_processor.process_inputs(
70 request_id="req-2",
71 prompt=prompt, # type: ignore[arg-type]
@@ -119,8 +84,7 @@ def test_multi_modal_uuids_missing_modality_raises(monkeypatch):
84 def test_multi_modal_uuids_accepts_none_and_passes_through(
85 monkeypatch, mm_cache_gb: float, enable_prefix_caching: bool
86 ):
122 - input_processor = _mock_input_processor(
123 - monkeypatch,
87 + input_processor = _build_input_processor(
88 mm_cache_gb=mm_cache_gb,
89 enable_prefix_caching=enable_prefix_caching,
90 )
@@ -163,8 +127,8 @@ def test_multi_modal_uuids_accepts_none_and_passes_through(
127 def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
128 # When both processor cache is 0 and prefix caching disabled, the
129 # processor builds overrides from request id instead of using user UUIDs.
166 - input_processor = _mock_input_processor(
167 - monkeypatch, mm_cache_gb=0.0, enable_prefix_caching=False
130 + input_processor = _build_input_processor(
131 + mm_cache_gb=0.0, enable_prefix_caching=False
132 )
133
134 captured: dict[str, MultiModalUUIDDict] = {}
@@ -180,12 +144,12 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
144 )
145
146 request_id = "req-42"
183 - mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": "hash_video"}
147 + mm_uuids = {"image": ["hash_cherry", "hash_stop"], "video": ["hash_video"]}
148 prompt = {
149 "prompt": "USER: <image><image><video>\nDescribe\nASSISTANT:",
150 "multi_modal_data": {
151 "image": [cherry_pil_image, stop_pil_image],
188 - "video": baby_reading_np_ndarrays,
152 + "video": [baby_reading_np_ndarrays],
153 },
154 "multi_modal_uuids": mm_uuids,
155 }
@@ -197,16 +161,15 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
161 )
162
163 # Expect request-id-based overrides are passed through
200 - mm_uuids = captured["mm_uuids"]
164 assert set(mm_uuids.keys()) == {"image", "video"}
165 assert len(mm_uuids["image"]) == 2
166 assert len(mm_uuids["video"]) == 1
204 - assert mm_uuids["image"][0].startswith(f"{request_id}-image-") and mm_uuids[
205 - "image"
206 - ][0].endswith("-0")
207 - assert mm_uuids["image"][1].startswith(f"{request_id}-image-") and mm_uuids[
208 - "image"
209 - ][1].endswith("-1")
210 - assert mm_uuids["video"][0].startswith(f"{request_id}-video-") and mm_uuids[
211 - "video"
212 - ][0].endswith("-0")
167 + assert captured["mm_uuids"]["image"][0].startswith(
168 + f"{request_id}-image-"
169 + ) and captured["mm_uuids"]["image"][0].endswith("-0")
170 + assert captured["mm_uuids"]["image"][1].startswith(
171 + f"{request_id}-image-"
172 + ) and captured["mm_uuids"]["image"][1].endswith("-1")
173 + assert captured["mm_uuids"]["video"][0].startswith(
174 + f"{request_id}-video-"
175 + ) and captured["mm_uuids"]["video"][0].endswith("-0")
vllm/entrypoints/chat_utils.py
+224 -176
@@ -8,6 +8,7 @@ from abc import ABC, abstractmethod
8 from collections import Counter, defaultdict
9 from collections.abc import Awaitable, Callable, Iterable
10 from functools import cached_property, lru_cache, partial
11 +from itertools import accumulate
12 from pathlib import Path
13 from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
14
@@ -41,6 +42,12 @@ from vllm.config import ModelConfig
42 from vllm.logger import init_logger
43 from vllm.model_executor.models import SupportsMultiModal
44 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
45 +from vllm.multimodal.inputs import (
46 + MultiModalBatchedField,
47 + MultiModalFlatField,
48 + MultiModalSharedField,
49 +)
50 +from vllm.multimodal.processing import BaseMultiModalProcessor
51 from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
52 from vllm.utils import random_uuid
53 from vllm.utils.collection_utils import is_list_of
@@ -48,7 +55,9 @@ from vllm.utils.import_utils import LazyLoader
55
56 if TYPE_CHECKING:
57 import torch
58 + import transformers
59 else:
60 + transformers = LazyLoader("transformers", globals(), "transformers")
61 torch = LazyLoader("torch", globals(), "torch")
62
63 logger = init_logger(__name__)
@@ -331,42 +340,113 @@ ModalityStr = Literal["image", "audio", "video", "image_embeds", "audio_embeds"]
340 _T = TypeVar("_T")
341
342
334 -def _extract_embeds(tensors: list[torch.Tensor]):
335 - if len(tensors) == 0:
336 - return tensors
343 +# Backward compatibility for single item input
344 +class _BatchedSingleItemField(MultiModalSharedField):
345 + pass
346
338 - if len(tensors) == 1:
339 - tensors[0]._is_single_item = True # type: ignore
340 - return tensors[0] # To keep backwards compatibility for single item input
347
342 - first_shape = tensors[0].shape
343 - if all(t.shape == first_shape for t in tensors):
344 - return torch.stack(tensors)
348 +def _detect_field(
349 + tensors: list[torch.Tensor],
350 + mm_processor: BaseMultiModalProcessor,
351 +):
352 + first_item = tensors[0]
353 + hidden_size = mm_processor.info.ctx.model_config.get_inputs_embeds_size()
354
346 - return tensors
355 + if (
356 + len(tensors) == 1
357 + and first_item.ndim == 3
358 + and first_item.shape[0] == 1
359 + and first_item.shape[-1] == hidden_size
360 + ):
361 + logger.warning(
362 + "Batched multi-modal embedding inputs are deprecated for Chat API. "
363 + "Please pass a separate content part for each multi-modal item."
364 + )
365 + return _BatchedSingleItemField(batch_size=1)
366
367 + first_shape = first_item.shape
368 + if all(t.shape == first_shape for t in tensors):
369 + return MultiModalBatchedField()
370 +
371 + size_per_item = [len(tensor) for tensor in tensors]
372 + slice_idxs = [0, *accumulate(size_per_item)]
373 + slices = [
374 + (slice(slice_idxs[i], slice_idxs[i + 1]),) for i in range(len(size_per_item))
375 + ]
376 + return MultiModalFlatField(slices=slices)
377 +
378 +
379 +def _merge_embeds(
380 + data_items: list[dict[str, "torch.Tensor"]],
381 + mm_processor: BaseMultiModalProcessor,
382 +):
383 + if not data_items:
384 + return {}
385 +
386 + first_keys = set(data_items[0].keys())
387 + if any(set(item.keys()) != first_keys for item in data_items[1:]):
388 + raise ValueError(
389 + "All dictionaries in the list of embeddings must have the same keys."
390 + )
391
349 -def _get_embeds_data(items_by_modality: dict[str, list[Any]], modality: str):
350 - embeds_key = f"{modality}_embeds"
351 - embeds = items_by_modality[embeds_key]
392 + fields = {
393 + key: _detect_field([item[key] for item in data_items], mm_processor)
394 + for key in first_keys
395 + }
396 + data_merged = {
397 + key: field._reduce_data([item[key] for item in data_items], pin_memory=False)
398 + for key, field in fields.items()
399 + }
400
353 - if len(embeds) == 0:
354 - return embeds
355 - if is_list_of(embeds, torch.Tensor):
356 - return _extract_embeds(embeds)
357 - if is_list_of(embeds, dict):
358 - if not embeds:
359 - return {}
401 + try:
402 + # TODO: Support per-request mm_processor_kwargs
403 + parsed_configs = mm_processor._get_mm_fields_config(
404 + transformers.BatchFeature(data_merged),
405 + {},
406 + )
407 + parsed_fields = {key: parsed_configs[key].field for key in first_keys}
408 + keys_to_update = [
409 + key
410 + for key in first_keys
411 + if (
412 + fields[key] != parsed_fields[key]
413 + and not isinstance(fields[key], _BatchedSingleItemField)
414 + )
415 + ]
416
361 - first_keys = set(embeds[0].keys())
362 - if any(set(item.keys()) != first_keys for item in embeds[1:]):
363 - raise ValueError(
364 - "All dictionaries in the list of embeddings must have the same keys."
417 + for key in keys_to_update:
418 + data_merged[key] = parsed_fields[key]._reduce_data(
419 + [item[key] for item in data_items], pin_memory=False
420 )
421 + except Exception:
422 + logger.exception(
423 + "Error when parsing merged embeddings. "
424 + "Falling back to auto-detected fields."
425 + )
426
367 - return {k: _extract_embeds([item[k] for item in embeds]) for k in first_keys}
427 + return data_merged
428
369 - return embeds
429 +
430 +def _get_embeds_data(
431 + modality: str,
432 + data_items: list[Any],
433 + mm_processor: BaseMultiModalProcessor,
434 +):
435 + if len(data_items) == 0:
436 + return data_items
437 +
438 + if all(item is None for item in data_items):
439 + return data_items
440 +
441 + if is_list_of(data_items, torch.Tensor):
442 + embeds_key = f"{modality}_embeds"
443 + dict_items = [{embeds_key: item} for item in data_items]
444 + return _merge_embeds(dict_items, mm_processor)[embeds_key]
445 +
446 + if is_list_of(data_items, dict):
447 + return _merge_embeds(data_items, mm_processor)
448 +
449 + raise NotImplementedError(type(data_items))
450
451
452 class BaseMultiModalItemTracker(ABC, Generic[_T]):
@@ -381,8 +461,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
461
462 self._model_config = model_config
463
384 - self._items_by_modality = defaultdict[str, list[_T | None]](list)
385 - self._uuids_by_modality = defaultdict[str, list[str | None]](list)
464 + self._items_by_modality = defaultdict[str, list[_T]](list)
465
466 @property
467 def model_config(self) -> ModelConfig:
@@ -411,12 +490,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
490 def mm_processor(self):
491 return self.mm_registry.create_processor(self.model_config)
492
414 - def add(
415 - self,
416 - modality: ModalityStr,
417 - item: _T | None,
418 - uuid: str | None = None,
419 - ) -> str | None:
493 + def add(self, modality: ModalityStr, item: _T) -> str | None:
494 """
495 Add a multi-modal item to the current prompt and returns the
496 placeholder string to use, if any.
@@ -430,99 +504,80 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
504 self.mm_processor.validate_num_items(input_modality, num_items)
505
506 self._items_by_modality[modality].append(item)
433 - self._uuids_by_modality[modality].append(uuid)
507
508 return self.model_cls.get_placeholder_str(modality, num_items)
509
437 - def all_mm_uuids(self) -> MultiModalUUIDDict | None:
438 - if not self._items_by_modality:
439 - return None
440 -
441 - uuids_by_modality = dict(self._uuids_by_modality)
442 - if "image" in uuids_by_modality and "image_embeds" in uuids_by_modality:
443 - raise ValueError("Mixing raw image and embedding inputs is not allowed")
444 - if "audio" in uuids_by_modality and "audio_embeds" in uuids_by_modality:
445 - raise ValueError("Mixing raw audio and embedding inputs is not allowed")
446 -
447 - mm_uuids = {}
448 - if "image_embeds" in uuids_by_modality:
449 - mm_uuids["image"] = uuids_by_modality["image_embeds"]
450 - if "image" in uuids_by_modality:
451 - mm_uuids["image"] = uuids_by_modality["image"] # UUIDs of images
452 - if "audio_embeds" in uuids_by_modality:
453 - mm_uuids["audio"] = uuids_by_modality["audio_embeds"]
454 - if "audio" in uuids_by_modality:
455 - mm_uuids["audio"] = uuids_by_modality["audio"] # UUIDs of audios
456 - if "video" in uuids_by_modality:
457 - mm_uuids["video"] = uuids_by_modality["video"] # UUIDs of videos
458 -
459 - return mm_uuids
460 -
510 @abstractmethod
511 def create_parser(self) -> "BaseMultiModalContentParser":
512 raise NotImplementedError
513
514
466 -class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
467 - def all_mm_data(self) -> MultiModalDataDict | None:
515 +def _resolve_items(
516 + items_by_modality: dict[str, list[tuple[object, str | None]]],
517 + mm_processor: BaseMultiModalProcessor,
518 +) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
519 + if "image" in items_by_modality and "image_embeds" in items_by_modality:
520 + raise ValueError("Mixing raw image and embedding inputs is not allowed")
521 + if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
522 + raise ValueError("Mixing raw audio and embedding inputs is not allowed")
523 +
524 + mm_data = {}
525 + mm_uuids = {}
526 + if "image_embeds" in items_by_modality:
527 + mm_data["image"] = _get_embeds_data(
528 + "image",
529 + [data for data, uuid in items_by_modality["image_embeds"]],
530 + mm_processor,
531 + )
532 + mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image_embeds"]]
533 + if "image" in items_by_modality:
534 + mm_data["image"] = [data for data, uuid in items_by_modality["image"]]
535 + mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]]
536 + if "audio_embeds" in items_by_modality:
537 + mm_data["audio"] = _get_embeds_data(
538 + "audio",
539 + [data for data, uuid in items_by_modality["audio_embeds"]],
540 + mm_processor,
541 + )
542 + mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio_embeds"]]
543 + if "audio" in items_by_modality:
544 + mm_data["audio"] = [data for data, uuid in items_by_modality["audio"]]
545 + mm_uuids["audio"] = [uuid for data, uuid in items_by_modality["audio"]]
546 + if "video" in items_by_modality:
547 + mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
548 + mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
549 +
550 + return mm_data, mm_uuids
551 +
552 +
553 +class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]):
554 + def resolve_items(
555 + self,
556 + ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
557 if not self._items_by_modality:
469 - return None
470 -
471 - items_by_modality = dict(self._items_by_modality)
472 - if "image" in items_by_modality and "image_embeds" in items_by_modality:
473 - raise ValueError("Mixing raw image and embedding inputs is not allowed")
474 - if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
475 - raise ValueError("Mixing raw audio and embedding inputs is not allowed")
476 -
477 - mm_inputs = {}
478 - if "image_embeds" in items_by_modality:
479 - mm_inputs["image"] = _get_embeds_data(items_by_modality, "image")
480 - if "image" in items_by_modality:
481 - mm_inputs["image"] = items_by_modality["image"] # A list of images
482 - if "audio_embeds" in items_by_modality:
483 - mm_inputs["audio"] = _get_embeds_data(items_by_modality, "audio")
484 - if "audio" in items_by_modality:
485 - mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
486 - if "video" in items_by_modality:
487 - mm_inputs["video"] = items_by_modality["video"] # A list of videos
488 -
489 - return mm_inputs
558 + return None, None
559 +
560 + return _resolve_items(dict(self._items_by_modality), self.mm_processor)
561
562 def create_parser(self) -> "BaseMultiModalContentParser":
563 return MultiModalContentParser(self)
564
565
495 -class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
496 - async def all_mm_data(self) -> MultiModalDataDict | None:
566 +class AsyncMultiModalItemTracker(
567 + BaseMultiModalItemTracker[Awaitable[tuple[object, str | None]]]
568 +):
569 + async def resolve_items(
570 + self,
571 + ) -> tuple[MultiModalDataDict | None, MultiModalUUIDDict | None]:
572 if not self._items_by_modality:
498 - return None
573 + return None, None
574
500 - coros_by_modality = {
501 - modality: [item or asyncio.sleep(0) for item in items]
502 - for modality, items in self._items_by_modality.items()
503 - }
504 - items_by_modality: dict[str, list[object | None]] = {
575 + resolved_items_by_modality = {
576 modality: await asyncio.gather(*coros)
506 - for modality, coros in coros_by_modality.items()
577 + for modality, coros in self._items_by_modality.items()
578 }
508 - if "image" in items_by_modality and "image_embeds" in items_by_modality:
509 - raise ValueError("Mixing raw image and embedding inputs is not allowed")
510 - if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
511 - raise ValueError("Mixing raw audio and embedding inputs is not allowed")
512 -
513 - mm_inputs = {}
514 - if "image_embeds" in items_by_modality:
515 - mm_inputs["image"] = _get_embeds_data(items_by_modality, "image")
516 - if "image" in items_by_modality:
517 - mm_inputs["image"] = items_by_modality["image"] # A list of images
518 - if "audio_embeds" in items_by_modality:
519 - mm_inputs["audio"] = _get_embeds_data(items_by_modality, "audio")
520 - if "audio" in items_by_modality:
521 - mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
522 - if "video" in items_by_modality:
523 - mm_inputs["video"] = items_by_modality["video"] # A list of videos
524 -
525 - return mm_inputs
579 +
580 + return _resolve_items(resolved_items_by_modality, self.mm_processor)
581
582 def create_parser(self) -> "BaseMultiModalContentParser":
583 return AsyncMultiModalContentParser(self)
@@ -611,7 +666,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
666 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
667 image = self._connector.fetch_image(image_url) if image_url else None
668
614 - placeholder = self._tracker.add("image", image, uuid)
669 + placeholder = self._tracker.add("image", (image, uuid))
670 self._add_placeholder("image", placeholder)
671
672 def parse_image_embeds(
@@ -630,14 +685,14 @@ class MultiModalContentParser(BaseMultiModalContentParser):
685 k: self._connector.fetch_image_embedding(v)
686 for k, v in image_embeds.items()
687 }
633 - placeholder = self._tracker.add("image_embeds", embeds, uuid)
688 + placeholder = self._tracker.add("image_embeds", (embeds, uuid))
689
690 if isinstance(image_embeds, str):
691 embedding = self._connector.fetch_image_embedding(image_embeds)
637 - placeholder = self._tracker.add("image_embeds", embedding, uuid)
692 + placeholder = self._tracker.add("image_embeds", (embedding, uuid))
693
694 if image_embeds is None:
640 - placeholder = self._tracker.add("image_embeds", None, uuid)
695 + placeholder = self._tracker.add("image_embeds", (None, uuid))
696
697 self._add_placeholder("image", placeholder)
698
@@ -657,25 +712,25 @@ class MultiModalContentParser(BaseMultiModalContentParser):
712 k: self._connector.fetch_audio_embedding(v)
713 for k, v in audio_embeds.items()
714 }
660 - placeholder = self._tracker.add("audio_embeds", embeds, uuid)
715 + placeholder = self._tracker.add("audio_embeds", (embeds, uuid))
716 elif isinstance(audio_embeds, str):
717 embedding = self._connector.fetch_audio_embedding(audio_embeds)
663 - placeholder = self._tracker.add("audio_embeds", embedding, uuid)
718 + placeholder = self._tracker.add("audio_embeds", (embedding, uuid))
719 else:
665 - placeholder = self._tracker.add("audio_embeds", None, uuid)
720 + placeholder = self._tracker.add("audio_embeds", (None, uuid))
721
722 self._add_placeholder("audio", placeholder)
723
724 def parse_image_pil(
725 self, image_pil: Image.Image | None, uuid: str | None = None
726 ) -> None:
672 - placeholder = self._tracker.add("image", image_pil, uuid)
727 + placeholder = self._tracker.add("image", (image_pil, uuid))
728 self._add_placeholder("image", placeholder)
729
730 def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
731 audio = self._connector.fetch_audio(audio_url) if audio_url else None
732
678 - placeholder = self._tracker.add("audio", audio, uuid)
733 + placeholder = self._tracker.add("audio", (audio, uuid))
734 self._add_placeholder("audio", placeholder)
735
736 def parse_input_audio(
@@ -697,7 +752,7 @@ class MultiModalContentParser(BaseMultiModalContentParser):
752 def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
753 video = self._connector.fetch_video(video_url=video_url) if video_url else None
754
700 - placeholder = self._tracker.add("video", video, uuid)
755 + placeholder = self._tracker.add("video", (video, uuid))
756 self._add_placeholder("video", placeholder)
757
758
@@ -719,10 +774,16 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
774 def model_config(self) -> ModelConfig:
775 return self._tracker.model_config
776
777 + async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None):
778 + image = (
779 + await self._connector.fetch_image_async(image_url) if image_url else None
780 + )
781 + return image, uuid
782 +
783 def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
723 - image_coro = self._connector.fetch_image_async(image_url) if image_url else None
784 + coro = self._image_with_uuid_async(image_url, uuid)
785
725 - placeholder = self._tracker.add("image", image_coro, uuid)
786 + placeholder = self._tracker.add("image", coro)
787 self._add_placeholder("image", placeholder)
788
789 def parse_image_embeds(
@@ -736,23 +797,25 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
797 "You must set `--enable-mm-embeds` to input `image_embeds`"
798 )
799
739 - future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
800 + future = asyncio.Future[
801 + tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
802 + ]()
803
804 if isinstance(image_embeds, dict):
805 embeds = {
806 k: self._connector.fetch_image_embedding(v)
807 for k, v in image_embeds.items()
808 }
746 - future.set_result(embeds)
809 + future.set_result((embeds, uuid))
810
811 if isinstance(image_embeds, str):
812 embedding = self._connector.fetch_image_embedding(image_embeds)
750 - future.set_result(embedding)
813 + future.set_result((embedding, uuid))
814
815 if image_embeds is None:
753 - future.set_result(None)
816 + future.set_result((None, uuid))
817
755 - placeholder = self._tracker.add("image_embeds", future, uuid)
818 + placeholder = self._tracker.add("image_embeds", future)
819 self._add_placeholder("image", placeholder)
820
821 def parse_audio_embeds(
@@ -766,72 +829,51 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
829 "You must set `--enable-mm-embeds` to input `audio_embeds`"
830 )
831
769 - logger.info(
770 - "🎵 Parsing audio_embeds: type=%s, uuid=%s, is_dict=%s, "
771 - "is_str=%s, is_none=%s",
772 - type(audio_embeds).__name__,
773 - uuid,
774 - isinstance(audio_embeds, dict),
775 - isinstance(audio_embeds, str),
776 - audio_embeds is None,
777 - )
778 -
779 - future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
832 + future = asyncio.Future[
833 + tuple[torch.Tensor | dict[str, torch.Tensor] | None, str | None]
834 + ]()
835
836 if isinstance(audio_embeds, dict):
782 - logger.info(
783 - "🎵 Processing dict audio_embeds with %d entries",
784 - len(audio_embeds),
785 - )
837 embeds = {
838 k: self._connector.fetch_audio_embedding(v)
839 for k, v in audio_embeds.items()
840 }
790 - future.set_result(embeds)
791 - logger.info(
792 - "🎵 Successfully loaded %d audio embeddings from dict",
793 - len(embeds),
794 - )
841 + future.set_result((embeds, uuid))
842
843 if isinstance(audio_embeds, str):
797 - base64_size = len(audio_embeds)
798 - logger.info(
799 - "🎵 Processing base64 audio_embeds: %d chars (%.2f KB)",
800 - base64_size,
801 - base64_size / 1024,
802 - )
844 embedding = self._connector.fetch_audio_embedding(audio_embeds)
804 - future.set_result(embedding)
805 - logger.info(
806 - "🎵 Successfully loaded audio embedding tensor: shape=%s, dtype=%s",
807 - embedding.shape,
808 - embedding.dtype,
809 - )
845 + future.set_result((embedding, uuid))
846
847 if audio_embeds is None:
812 - logger.info("🎵 Audio embeds is None (UUID-only reference)")
813 - future.set_result(None)
848 + future.set_result((None, uuid))
849
815 - placeholder = self._tracker.add("audio_embeds", future, uuid)
850 + placeholder = self._tracker.add("audio_embeds", future)
851 self._add_placeholder("audio", placeholder)
817 - logger.info("🎵 Added audio_embeds placeholder with uuid=%s", uuid)
852
853 def parse_image_pil(
820 - self, image_pil: Image.Image | None, uuid: str | None = None
854 + self,
855 + image_pil: Image.Image | None,
856 + uuid: str | None = None,
857 ) -> None:
822 - future: asyncio.Future[Image.Image | None] = asyncio.Future()
858 + future = asyncio.Future[tuple[Image.Image | None, str | None]]()
859 if image_pil:
824 - future.set_result(image_pil)
860 + future.set_result((image_pil, uuid))
861 else:
826 - future.set_result(None)
862 + future.set_result((None, uuid))
863
828 - placeholder = self._tracker.add("image", future, uuid)
864 + placeholder = self._tracker.add("image", future)
865 self._add_placeholder("image", placeholder)
866
867 + async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None):
868 + audio = (
869 + await self._connector.fetch_audio_async(audio_url) if audio_url else None
870 + )
871 + return audio, uuid
872 +
873 def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
832 - audio_coro = self._connector.fetch_audio_async(audio_url) if audio_url else None
874 + coro = self._audio_with_uuid_async(audio_url, uuid)
875
834 - placeholder = self._tracker.add("audio", audio_coro, uuid)
876 + placeholder = self._tracker.add("audio", coro)
877 self._add_placeholder("audio", placeholder)
878
879 def parse_input_audio(
@@ -850,14 +892,16 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
892
893 return self.parse_audio(audio_url, uuid)
894
853 - def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
895 + async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None):
896 video = (
855 - self._connector.fetch_video_async(video_url=video_url)
856 - if video_url
857 - else None
897 + await self._connector.fetch_video_async(video_url) if video_url else None
898 )
899 + return video, uuid
900 +
901 + def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
902 + coro = self._video_with_uuid_async(video_url, uuid)
903
860 - placeholder = self._tracker.add("video", video, uuid)
904 + placeholder = self._tracker.add("video", coro)
905 self._add_placeholder("video", placeholder)
906
907
@@ -1380,7 +1424,9 @@ def parse_chat_messages(
1424
1425 _postprocess_messages(conversation)
1426
1383 - return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1427 + mm_data, mm_uuids = mm_tracker.resolve_items()
1428 +
1429 + return conversation, mm_data, mm_uuids
1430
1431
1432 async def parse_chat_messages_async(
@@ -1411,7 +1457,9 @@ async def parse_chat_messages_async(
1457
1458 _postprocess_messages(conversation)
1459
1414 - return conversation, await mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1460 + mm_data, mm_uuids = await mm_tracker.resolve_items()
1461 +
1462 + return conversation, mm_data, mm_uuids
1463
1464
1465 def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):
vllm/entrypoints/llm.py
-52
@@ -1574,10 +1574,6 @@ class LLM:
1574
1575 try:
1576 for i, prompt in enumerate(it):
1577 - if isinstance(prompt, dict):
1578 - self._validate_mm_data_and_uuids(
1579 - prompt.get("multi_modal_data"), prompt.get("multi_modal_uuids")
1580 - )
1577 request_id = self._add_request(
1578 prompt,
1579 params[i] if isinstance(params, Sequence) else params,
@@ -1593,54 +1589,6 @@ class LLM:
1589 self.llm_engine.abort_request(added_request_ids, internal=True)
1590 raise e
1591
1596 - def _validate_mm_data_and_uuids(
1597 - self,
1598 - multi_modal_data: Any | None, # MultiModalDataDict
1599 - multi_modal_uuids: Any | None, # MultiModalUUIDDict
1600 - ):
1601 - """
1602 - Validate that if any multi-modal data is skipped (i.e. None),
1603 - then its corresponding UUID must be set.
1604 - """
1605 - if multi_modal_data is None:
1606 - return
1607 -
1608 - for modality, data in multi_modal_data.items():
1609 - if isinstance(data, list):
1610 - for i, d in enumerate(data):
1611 - if d is None:
1612 - if (
1613 - multi_modal_uuids is None
1614 - or modality not in multi_modal_uuids
1615 - or multi_modal_uuids[ # noqa: E501
1616 - modality
1617 - ]
1618 - is None
1619 - ):
1620 - raise ValueError(
1621 - f"Multi-modal data for {modality} is None "
1622 - f"but UUID is not provided"
1623 - )
1624 - else:
1625 - if (
1626 - len(multi_modal_uuids[modality]) <= i
1627 - or multi_modal_uuids[modality][i] is None
1628 - ):
1629 - raise ValueError(
1630 - f"Multi-modal data for {modality} is None "
1631 - f"but UUID is not provided"
1632 - )
1633 - else:
1634 - if data is None and (
1635 - multi_modal_uuids is None
1636 - or modality not in multi_modal_uuids
1637 - or multi_modal_uuids[modality] is None
1638 - ):
1639 - raise ValueError(
1640 - f"Multi-modal data for {modality} is None"
1641 - f" but UUID is not provided"
1642 - )
1643 -
1592 def _process_inputs(
1593 self,
1594 request_id: str,
vllm/entrypoints/pooling/score/utils.py
+8 -4
@@ -19,7 +19,7 @@ from vllm.entrypoints.chat_utils import (
19 )
20 from vllm.inputs import TokensPrompt
21 from vllm.model_executor.models.interfaces import supports_score_template
22 -from vllm.multimodal.inputs import MultiModalDataDict
22 +from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
23 from vllm.outputs import PoolingRequestOutput
24 from vllm.renderers.hf import safe_apply_chat_template
25 from vllm.tokenizers import TokenizerLike
@@ -95,7 +95,7 @@ def parse_score_data(
95 data_1: str | ScoreContentPartParam,
96 data_2: str | ScoreContentPartParam,
97 model_config: ModelConfig,
98 -) -> tuple[str, str, MultiModalDataDict | None]:
98 +) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
99 mm_tracker = MultiModalItemTracker(model_config)
100
101 content_1 = _parse_score_content(data_1, mm_tracker)
@@ -109,8 +109,9 @@ def parse_score_data(
109
110 prompt_1 = ensure_str(content_1)
111 prompt_2 = ensure_str(content_2)
112 + mm_items, mm_uuids = mm_tracker.resolve_items()
113
113 - return prompt_1, prompt_2, mm_tracker.all_mm_data()
114 + return prompt_1, prompt_2, mm_items, mm_uuids
115
116
117 def _parse_score_content(
@@ -187,7 +188,7 @@ def get_score_prompt(
188 data_2: str | ScoreContentPartParam,
189 score_template: str | None = None,
190 ) -> tuple[str, TokensPrompt]:
190 - prompt_1, prompt_2, mm_data = parse_score_data(
191 + prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
192 data_1,
193 data_2,
194 model_config,
@@ -248,6 +249,9 @@ def get_score_prompt(
249
250 if mm_data is not None:
251 engine_prompt["multi_modal_data"] = mm_data
252 + if mm_uuids is not None:
253 + engine_prompt["multi_modal_uuids"] = mm_uuids
254 +
255 return full_prompt, engine_prompt
256
257
vllm/model_executor/models/qwen3_vl.py
+8 -23
@@ -113,7 +113,11 @@ from .qwen2_5_vl import (
113 Qwen2_5_VLVideoInputs,
114 Qwen2_5_VLVideoPixelInputs,
115 )
116 -from .qwen2_vl import Qwen2VLMultiModalDataParser, Qwen2VLProcessingInfo
116 +from .qwen2_vl import (
117 + Qwen2VLMultiModalDataParser,
118 + Qwen2VLProcessingInfo,
119 + _create_qwen2vl_field_factory,
120 +)
121 from .qwen3 import Qwen3ForCausalLM, Qwen3Model
122 from .utils import (
123 AutoWeightsLoader,
@@ -985,28 +989,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
989 hf_inputs: BatchFeature,
990 hf_processor_mm_kwargs: Mapping[str, object],
991 ) -> Mapping[str, MultiModalFieldConfig]:
988 - image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
989 - image_grid_sizes = image_grid_thw.prod(-1)
990 -
991 - video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
992 - video_grid_sizes = video_grid_thw.prod(-1)
993 -
994 - return dict(
995 - pixel_values=MultiModalFieldConfig.flat_from_sizes(
996 - "image", image_grid_sizes
997 - ),
998 - image_embeds=MultiModalFieldConfig.flat_from_sizes(
999 - "image", image_grid_sizes
1000 - ),
1001 - image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
1002 - pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
1003 - "video", video_grid_sizes
1004 - ),
1005 - video_embeds=MultiModalFieldConfig.flat_from_sizes(
1006 - "video", video_grid_sizes
1007 - ),
1008 - video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
1009 - )
992 + return _create_qwen2vl_field_factory(
993 + self.info.get_hf_config().vision_config.spatial_merge_size
994 + )(hf_inputs)
995
996 def _get_prompt_updates(
997 self,
vllm/model_executor/models/terratorch.py
+40 -40
@@ -18,7 +18,7 @@
18 """Wrapper around `Terratorch` models"""
19
20 from collections import OrderedDict
21 -from collections.abc import Callable, Iterable, Mapping, Sequence
21 +from collections.abc import Iterable, Mapping, Sequence
22 from typing import Any
23
24 import torch
@@ -62,6 +62,7 @@ from vllm.multimodal.processing import (
62 PromptUpdate,
63 )
64 from vllm.sequence import IntermediateTensors
65 +from vllm.utils import length_from_prompt_token_ids_or_embeds
66
67 from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal
68 from .interfaces_base import attn_type
@@ -69,28 +70,21 @@ from .interfaces_base import attn_type
70 logger = init_logger(__name__)
71
72
72 -def _terratorch_field_names(pretrained_cfg: dict):
73 - input_definition = InputDefinition(**pretrained_cfg["input"])
73 +def _terratorch_field_names(input_definition: InputDefinition):
74 return set(input_definition.data.keys())
75
76
77 -def _terratorch_field_factory(
78 - pretrained_cfg: dict,
79 -) -> Callable[
80 - [Mapping[str, torch.Tensor]],
81 - Mapping[str, MultiModalFieldConfig],
82 -]:
83 - def _terratorch_field_config(hf_inputs: Mapping[str, torch.Tensor]):
84 - input_definition = InputDefinition(**pretrained_cfg["input"])
85 - fields = {}
86 - for input_name, input in input_definition.data.items():
77 +def _terratorch_field_factory(input_definition: InputDefinition):
78 + def _terratorch_field_config(
79 + hf_inputs: Mapping[str, torch.Tensor],
80 + ) -> Mapping[str, MultiModalFieldConfig]:
81 + fields = dict[str, MultiModalFieldConfig]()
82 + for name, input in input_definition.data.items():
83 + modality = "image"
84 if input.type == InputTypeEnum.tensor:
88 - fields[input_name] = "image"
85 + fields[name] = MultiModalFieldConfig.shared(modality, batch_size=1)
86
90 - return {
91 - field_name: MultiModalFieldConfig.batched(modality=field_modality)
92 - for field_name, field_modality in fields.items()
93 - }
87 + return fields
88
89 return _terratorch_field_config
90
@@ -130,26 +124,31 @@ class TerratorchInputBuilder(BaseDummyInputsBuilder[TerratorchProcessingInfo]):
124
125
126 class TerratorchMultiModalDataParser(MultiModalDataParser):
133 - def __init__(self, pretrained_cfg: dict, *args, **kwargs):
134 - self._pretrained_cfg = pretrained_cfg
127 + def __init__(self, input_definition: InputDefinition, *args, **kwargs):
128 super().__init__(*args, **kwargs)
129
130 + self.input_definition = input_definition
131 +
132 def _parse_image_data(
133 self,
134 data: dict[str, torch.Tensor] | ModalityData[ImageItem],
135 ) -> ModalityDataItems[Any, Any] | None:
136 if isinstance(data, dict):
142 - terratorch_fields = _terratorch_field_names(self._pretrained_cfg)
143 -
137 return DictEmbeddingItems(
138 data,
139 modality="image",
147 - required_fields=terratorch_fields,
148 - fields_factory=_terratorch_field_factory(self._pretrained_cfg),
140 + required_fields=_terratorch_field_names(self.input_definition),
141 + fields_factory=_terratorch_field_factory(self.input_definition),
142 )
143
144 return super()._parse_image_data(data)
145
146 + def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
147 + if "image" not in mm_data:
148 + mm_data = {"image": mm_data}
149 +
150 + return super().parse_mm_data(mm_data)
151 +
152
153 class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
154 def __init__(
@@ -159,18 +158,20 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
158 *,
159 cache: MultiModalProcessorOnlyCache | None = None,
160 ) -> None:
162 - self.pretrained_cfg = info.get_hf_config().to_dict()["pretrained_cfg"]
161 + pretrained_cfg = info.get_hf_config().to_dict()["pretrained_cfg"]
162 + self._input_definition = InputDefinition(**pretrained_cfg["input"])
163 +
164 super().__init__(info=info, dummy_inputs=dummy_inputs, cache=cache)
165
166 def _get_data_parser(self) -> MultiModalDataParser:
166 - return TerratorchMultiModalDataParser(pretrained_cfg=self.pretrained_cfg)
167 + return TerratorchMultiModalDataParser(self._input_definition)
168
169 def _get_mm_fields_config(
170 self,
171 hf_inputs: BatchFeature,
172 hf_processor_mm_kwargs: Mapping[str, object],
173 ) -> Mapping[str, MultiModalFieldConfig]:
173 - return _terratorch_field_factory(self.pretrained_cfg)(hf_inputs)
174 + return _terratorch_field_factory(self._input_definition)(hf_inputs)
175
176 def _get_prompt_updates(
177 self,
@@ -188,23 +189,16 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
189 tokenization_kwargs: Mapping[str, object] | None = None,
190 mm_uuids: MultiModalUUIDDict | None = None,
191 ) -> MultiModalInputs:
191 - if "image" in mm_data:
192 - image_data = mm_data["image"]
193 - image_data = {k: v.unsqueeze(0) for k, v in image_data.items()}
194 - else:
195 - image_data = mm_data
196 - image_data = {k: v.unsqueeze(0) for k, v in image_data.items()}
197 -
198 - mm_data = {"image": image_data}
199 -
192 mm_items = self._to_mm_items(mm_data)
193 tokenization_kwargs = tokenization_kwargs or {}
194 mm_hashes = self._hash_mm_items(
195 mm_items, hf_processor_mm_kwargs, tokenization_kwargs, mm_uuids=mm_uuids
196 )
205 - mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
197
207 - mm_processed_data = BatchFeature(image_data)
198 + mm_processed_data = BatchFeature(
199 + mm_data.get("image", mm_data), tensor_type="pt"
200 + )
201 + mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
202
203 mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
204 mm_processed_data,
@@ -272,9 +266,15 @@ class Terratorch(nn.Module, IsAttentionFree, SupportsMultiModal):
266 inputs_embeds: torch.Tensor | None = None,
267 **kwargs: object,
268 ):
275 - model_output = self.inference_runner.forward(**kwargs)
269 + input_len = length_from_prompt_token_ids_or_embeds(input_ids, inputs_embeds)
270 +
271 + batched_kwargs = {k: v.unsqueeze(0) for k, v in kwargs.items()}
272 + model_output = self.inference_runner.forward(**batched_kwargs).output
273
277 - return model_output.output
274 + # The leading dimension of hidden states needs to equal input length
275 + return model_output.expand(
276 + input_len, *(-1 for _ in range(model_output.ndim - 1))
277 + )
278
279 def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
280 params_list = []
vllm/utils/__init__.py
+1 -1
@@ -13,7 +13,7 @@ def random_uuid() -> str:
13
14
15 def length_from_prompt_token_ids_or_embeds(
16 - prompt_token_ids: list[int] | None,
16 + prompt_token_ids: list[int] | torch.Tensor | None,
17 prompt_embeds: torch.Tensor | None,
18 ) -> int:
19 """Calculate the request length (in number of tokens) give either
vllm/v1/engine/input_processor.py
+103 -75
@@ -8,14 +8,24 @@ from typing import Any, Literal, cast
8
9 from vllm.config import VllmConfig
10 from vllm.exceptions import VLLMValidationError
11 -from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs
12 -from vllm.inputs.parse import split_enc_dec_inputs
11 +from vllm.inputs import (
12 + ProcessorInputs,
13 + PromptType,
14 + SingletonInputs,
15 + SingletonPrompt,
16 + TextPrompt,
17 +)
18 +from vllm.inputs.parse import is_explicit_encoder_decoder_prompt, split_enc_dec_inputs
19 from vllm.inputs.preprocess import InputPreprocessor
20 from vllm.logger import init_logger
21 from vllm.lora.request import LoRARequest
22 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
17 -from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict
18 -from vllm.multimodal.parse import MultiModalDataParser
23 +from vllm.multimodal.inputs import (
24 + MultiModalDataDict,
25 + MultiModalFeatureSpec,
26 + MultiModalUUIDDict,
27 +)
28 +from vllm.multimodal.parse import ModalityDataItems, MultiModalDataItems
29 from vllm.multimodal.processing.context import set_request_id
30 from vllm.multimodal.utils import argsort_mm_positions
31 from vllm.pooling_params import PoolingParams
@@ -188,7 +198,66 @@ class InputProcessor:
198 self._validate_sampling_params(params)
199 self._validate_supported_sampling_params(params)
200
191 - def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:
201 + def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
202 + mm_processor = self.input_preprocessor._get_mm_processor()
203 + return mm_processor.data_parser.parse_mm_data(mm_data)
204 +
205 + def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None:
206 + if isinstance(prompt, str):
207 + prompt = TextPrompt(prompt=prompt)
208 +
209 + mm_data = cast(MultiModalDataDict, prompt.get("multi_modal_data") or {})
210 + mm_uuids = cast(MultiModalUUIDDict, prompt.get("multi_modal_uuids") or {})
211 + if not mm_data and not mm_uuids:
212 + return
213 +
214 + mm_data_parsed = self._parse_mm_items(
215 + {k: v for k, v in mm_data.items() if v is not None}
216 + )
217 + mm_uuids_parsed = {
218 + k: [v] if isinstance(v, str) else v
219 + for k, v in mm_uuids.items()
220 + if v is not None
221 + }
222 +
223 + # NOTE: Include the keys corresponding to `None`
224 + modalities = mm_data.keys() | mm_uuids.keys()
225 +
226 + for modality in modalities:
227 + data_items = cast(
228 + ModalityDataItems | list[Any], mm_data_parsed.get(modality, [])
229 + )
230 + uuid_items = cast(list[str | None], mm_uuids_parsed.get(modality, []))
231 +
232 + if len(data_items) > 0:
233 + if len(uuid_items) > 0 and len(data_items) != len(uuid_items):
234 + raise ValueError(
235 + f"If given, multi_modal_uuids[{modality!r}] must have "
236 + f"same length as multi_modal_data[{modality!r}], but "
237 + f"got {len(uuid_items)} vs {len(data_items)}."
238 + )
239 +
240 + for i, item in enumerate(data_items):
241 + if item is None:
242 + if not uuid_items:
243 + raise ValueError(
244 + f"multi_modal_data[{modality!r}][{i}] is empty but "
245 + f"multi_modal_uuids[{modality!r}] is missing."
246 + )
247 +
248 + if uuid_items[i] is None:
249 + raise ValueError(
250 + f"multi_modal_data[{modality!r}][{i}] is empty but "
251 + f"multi_modal_uuids[{modality!r}][{i}] is missing."
252 + )
253 + else:
254 + if len(uuid_items) == 0:
255 + raise ValueError(
256 + f"multi_modal_data[{modality!r}] is empty but "
257 + f"multi_modal_uuids[{modality!r}] is missing."
258 + )
259 +
260 + def _validate_mm_uuids(self, prompt: PromptType) -> None:
261 """
262 Validate that user-provided multi_modal_uuids align with
263 multi_modal_data in the incoming request prompt(s).
@@ -196,55 +265,13 @@ class InputProcessor:
265 auto-hashed downstream.
266 """
267
199 - def _validate_single_prompt(single_prompt: dict | str) -> None:
200 - if not isinstance(single_prompt, dict):
201 - return
202 -
203 - mm_data = single_prompt.get("multi_modal_data")
204 - mm_uuids = single_prompt.get("multi_modal_uuids")
205 - if not mm_data or not mm_uuids:
206 - return
207 -
208 - import torch
209 -
210 - def _get_len(items: object):
211 - if isinstance(items, dict): # Embedding inputs
212 - return _get_len(next(iter(items.values()))) if items else 1
213 -
214 - if isinstance(items, list):
215 - return len(items)
216 - if isinstance(items, torch.Tensor):
217 - # To keep backwards compatibility for single item embedding input
218 - return 1 if getattr(items, "_is_single_item", False) else len(items)
219 -
220 - return 1
221 -
222 - for modality, items in mm_data.items():
223 - if modality in mm_uuids:
224 - data_len = _get_len(items)
225 - uuid_len = _get_len(mm_uuids[modality])
226 - if uuid_len != data_len:
227 - raise ValueError(
228 - f"multi_modal_uuids for modality {modality!r} "
229 - "must have same length as data: got "
230 - f"{uuid_len} uuids vs {data_len} items."
231 - )
232 - else:
233 - raise ValueError(
234 - f"multi_modal_uuids for modality {modality!r} must "
235 - "be provided if multi_modal_data is provided."
236 - )
268 + if is_explicit_encoder_decoder_prompt(prompt):
269 + self._validate_singleton_mm_uuids(prompt["encoder_prompt"])
270
238 - # Handle explicit encoder/decoder prompts or singleton prompt
239 - if isinstance(prompt, dict) and "encoder_prompt" in prompt:
240 - enc = prompt.get("encoder_prompt")
241 - dec = prompt.get("decoder_prompt")
242 - if enc is not None:
243 - _validate_single_prompt(cast(dict | str, enc))
244 - if dec is not None:
245 - _validate_single_prompt(cast(dict | str, dec))
271 + if (dec_prompt := prompt["decoder_prompt"]) is not None:
272 + self._validate_singleton_mm_uuids(dec_prompt)
273 else:
247 - _validate_single_prompt(prompt) # type: ignore[arg-type]
274 + self._validate_singleton_mm_uuids(prompt)
275
276 def _validate_lora(self, lora_request: LoRARequest | None) -> None:
277 if lora_request is None:
@@ -379,6 +406,20 @@ class InputProcessor:
406 # roundtrip serialization/deserialization won't fail.
407 params.structured_outputs.__post_init__()
408
409 + def _extract_singleton_mm_data(
410 + self, prompt: SingletonPrompt
411 + ) -> MultiModalDataDict | None:
412 + if isinstance(prompt, str):
413 + return None
414 +
415 + return prompt.get("multi_modal_data") # type: ignore[return-value]
416 +
417 + def _extract_mm_data(self, prompt: PromptType) -> MultiModalDataDict | None:
418 + if is_explicit_encoder_decoder_prompt(prompt):
419 + return self._extract_singleton_mm_data(prompt["encoder_prompt"])
420 + else:
421 + return self._extract_singleton_mm_data(prompt)
422 +
423 def _maybe_build_mm_uuids(
424 self,
425 request_id: str,
@@ -391,31 +432,18 @@ class InputProcessor:
432 Returns a dictionary of modality -> list[str] of overrides, or None if
433 disabled or no multimodal data is present.
434 """
394 -
395 - def _extract_mm_data(p: PromptType):
396 - if isinstance(p, dict) and "encoder_prompt" in p:
397 - enc = p.get("encoder_prompt")
398 - if isinstance(enc, dict):
399 - return enc.get("multi_modal_data")
400 - return None
401 - if isinstance(p, dict):
402 - return p.get("multi_modal_data")
403 - return None
404 -
405 - mm_data = _extract_mm_data(prompt)
435 + mm_data = self._extract_mm_data(prompt)
436 if not mm_data:
437 return None
438
409 - mm_uuids: dict[str, list[str | None] | str] = {}
410 - for modality, data in mm_data.items():
411 - # Hash each item for embedding inputs.
412 - n = (
413 - len(data)
414 - if isinstance(data, list) or MultiModalDataParser.is_embeddings(data)
415 - else 1
416 - )
417 - mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]
418 - return mm_uuids
439 + mm_items = self._parse_mm_items(
440 + {k: v for k, v in mm_data.items() if v is not None}
441 + )
442 +
443 + return {
444 + modality: [f"{request_id}-{modality}-{i}" for i in range(data_count)]
445 + for modality, data_count in mm_items.get_all_counts().items()
446 + }
447
448 def _get_mm_identifier(
449 self,
@@ -494,7 +522,7 @@ class InputProcessor:
522 else:
523 # Otherwise, use user-provided uuids as multimodal hash overrides
524 # if provided.
497 - self._validate_multi_modal_uuids(prompt)
525 + self._validate_mm_uuids(prompt)
526 if isinstance(prompt, dict):
527 mm_uuids = cast(
528 MultiModalUUIDDict | None, prompt.get("multi_modal_uuids")