@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7f2ea7074e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend][Multimodal] Allow skipping media data when UUIDs are provided. (#23950)

Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Signed-off-by: Roger Wang <hey@rogerw.me> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.me>

Chenheli Hua committed Sep 12, 2025 at 19:16 UTC 7f2ea7074e4c2ee75278623a02007fc1b6bb9639
9 files changed +970 -96
docs/features/multimodal_inputs.md
+59
@@ -45,6 +45,32 @@ When using multi-modal inputs, vLLM normally hashes each media item by content t
45 print(o.outputs[0].text)
46 ```
47
48 +Using UUIDs, you can also skip sending media data entirely if you expect cache hits for respective items. Note that the request will fail if the skipped media doesn't have a corresponding UUID, or if the UUID fails to hit the cache.
49 +
50 +??? code
51 +
52 + ```python
53 + from vllm import LLM
54 + from PIL import Image
55 +
56 + # Qwen2.5-VL example with two images
57 + llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct")
58 +
59 + prompt = "USER: <image><image>\nDescribe the differences.\nASSISTANT:"
60 + img_b = Image.open("/path/to/b.jpg")
61 +
62 + outputs = llm.generate({
63 + "prompt": prompt,
64 + "multi_modal_data": {"image": [None, img_b]},
65 + # Since img_a is expected to be cached, we can skip sending the actual
66 + # image entirely.
67 + "multi_modal_uuids": {"image": ["sku-1234-a", None]},
68 + })
69 +
70 + for o in outputs:
71 + print(o.outputs[0].text)
72 + ```
73 +
74 !!! warning
75 If both multimodal processor caching and prefix caching are disabled, user-provided `multi_modal_uuids` are ignored.
76
@@ -755,6 +781,39 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
781 )
782 ```
783
784 +For Online Serving, you can also skip sending media if you expect cache hits with provided UUIDs. You can do so by sending media like this:
785 +
786 + ```python
787 + # Image/video/audio URL:
788 + {
789 + "type": "image_url",
790 + "image_url": None,
791 + "uuid": image_uuid,
792 + },
793 +
794 + # image_embeds
795 + {
796 + "type": "image_embeds",
797 + "image_embeds": None,
798 + "uuid": image_uuid
799 + },
800 +
801 + # input_audio:
802 + {
803 + "type": "input_audio",
804 + "input_audio": None,
805 + "uuid": audio_uuid
806 + },
807 +
808 + # PIL Image:
809 + {
810 + "type": "image_pil",
811 + "image_pil": None
812 + "uuid": image_uuid
813 + }
814 +
815 + ```
816 +
817 !!! note
818 Only one message can contain `{"type": "image_embeds"}`.
819 If used with a model that requires additional parameters, you must also provide a tensor for each of them, e.g. `image_grid_thw`, `image_sizes`, etc.
examples/offline_inference/vision_language.py
+71 -10
@@ -1764,6 +1764,7 @@ def apply_image_repeat(
1764 probs = [1.0 - image_repeat_prob, image_repeat_prob]
1765
1766 inputs = []
1767 + inputs_with_empty_media = []
1768 cur_image = data
1769 for i in range(num_prompts):
1770 if image_repeat_prob is not None:
@@ -1774,14 +1775,25 @@ def apply_image_repeat(
1775 new_val = (i // 256 // 256, i // 256, i % 256)
1776 cur_image.putpixel((0, 0), new_val)
1777
1778 + uuid = "uuid_{}".format(i)
1779 +
1780 inputs.append(
1781 {
1782 "prompt": prompts[i % len(prompts)],
1783 "multi_modal_data": {modality: cur_image},
1784 + "multi_modal_uuids": {modality: uuid},
1785 + }
1786 + )
1787 +
1788 + inputs_with_empty_media.append(
1789 + {
1790 + "prompt": prompts[i % len(prompts)],
1791 + "multi_modal_data": {modality: None},
1792 + "multi_modal_uuids": {modality: uuid},
1793 }
1794 )
1795
1784 - return inputs
1796 + return inputs, inputs_with_empty_media
1797
1798
1799 @contextmanager
@@ -1860,6 +1872,13 @@ def parse_args():
1872 help="If True, then use different prompt (with the same multi-modal "
1873 "data) for each request.",
1874 )
1875 +
1876 + parser.add_argument(
1877 + "--verify-mm-cache-hit-with-uuids",
1878 + action="store_true",
1879 + help="If True, will send all requests in a second batch with empty mm "
1880 + "data to verify cache hits with UUIDs.",
1881 + )
1882 return parser.parse_args()
1883
1884
@@ -1903,26 +1922,48 @@ def main(args):
1922 assert args.num_prompts > 0
1923 if args.num_prompts == 1:
1924 # Single inference
1925 + uuid = "uuid_0"
1926 inputs = {
1927 "prompt": prompts[0],
1928 "multi_modal_data": {modality: data},
1929 + "multi_modal_uuids": {modality: uuid},
1930 + }
1931 + inputs_with_empty_media = {
1932 + "prompt": prompts[0],
1933 + "multi_modal_data": {modality: None},
1934 + "multi_modal_uuids": {modality: uuid},
1935 }
1936 else:
1937 # Batch inference
1938 if args.image_repeat_prob is not None:
1939 # Repeat images with specified probability of "image_repeat_prob"
1914 - inputs = apply_image_repeat(
1915 - args.image_repeat_prob, args.num_prompts, data, prompts, modality
1940 + inputs, inputs_with_empty_media = apply_image_repeat(
1941 + args.image_repeat_prob,
1942 + args.num_prompts,
1943 + data,
1944 + prompts,
1945 + modality,
1946 )
1947 else:
1948 # Use the same image for all prompts
1919 - inputs = [
1920 - {
1921 - "prompt": prompts[i % len(prompts)],
1922 - "multi_modal_data": {modality: data},
1923 - }
1924 - for i in range(args.num_prompts)
1925 - ]
1949 + inputs = []
1950 + inputs_with_empty_media = []
1951 + for i in range(args.num_prompts):
1952 + uuid = "uuid_{}".format(i)
1953 + inputs.append(
1954 + {
1955 + "prompt": prompts[i % len(prompts)],
1956 + "multi_modal_data": {modality: data},
1957 + "multi_modal_uuids": {modality: uuid},
1958 + }
1959 + )
1960 + inputs_with_empty_media.append(
1961 + {
1962 + "prompt": prompts[i % len(prompts)],
1963 + "multi_modal_data": {modality: None},
1964 + "multi_modal_uuids": {modality: uuid},
1965 + }
1966 + )
1967
1968 # Add LoRA request if applicable
1969 lora_request = (
@@ -1942,6 +1983,26 @@ def main(args):
1983 print(generated_text)
1984 print("-" * 50)
1985
1986 + if args.verify_mm_cache_hit_with_uuids:
1987 + try:
1988 + # Verify cache hits with UUIDs
1989 + print(
1990 + "Sending a second batch of requests with empty media"
1991 + " and matching UUIDs."
1992 + )
1993 + outputs = llm.generate(
1994 + inputs_with_empty_media,
1995 + sampling_params=sampling_params,
1996 + lora_request=lora_request,
1997 + )
1998 + print("-" * 50)
1999 + for o in outputs:
2000 + generated_text = o.outputs[0].text
2001 + print(generated_text)
2002 + print("-" * 50)
2003 + except Exception as e:
2004 + print(f"Failed to verify cache hits with UUIDs. Error: {e}")
2005 +
2006
2007 if __name__ == "__main__":
2008 args = parse_args()
tests/entrypoints/openai/test_vision.py
+65
@@ -522,6 +522,71 @@ async def test_completions_with_image_with_uuid(
522 assert isinstance(chat_completion.choices[0].message.content, str)
523 assert len(chat_completion.choices[0].message.content) > 0
524
525 + # Second request, with empty image but the same uuid.
526 + chat_completion_with_empty_image = await client.chat.completions.create(
527 + messages=[
528 + {
529 + "role": "system",
530 + "content": "You are a helpful assistant."
531 + },
532 + {
533 + "role":
534 + "user",
535 + "content": [
536 + {
537 + "type": "text",
538 + "text": "Describe this image.",
539 + },
540 + {
541 + "type": "image_url",
542 + "image_url": {},
543 + "uuid": image_url
544 + },
545 + ],
546 + },
547 + ],
548 + model=model_name,
549 + )
550 + assert chat_completion_with_empty_image.choices[
551 + 0].message.content is not None
552 + assert isinstance(
553 + chat_completion_with_empty_image.choices[0].message.content, str)
554 + assert len(
555 + chat_completion_with_empty_image.choices[0].message.content) > 0
556 +
557 +
558 +@pytest.mark.asyncio
559 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
560 +async def test_completions_with_empty_image_with_uuid_without_cache_hit(
561 + client: openai.AsyncOpenAI,
562 + model_name: str,
563 +):
564 + with pytest.raises(openai.BadRequestError):
565 + _ = await client.chat.completions.create(
566 + messages=[
567 + {
568 + "role": "system",
569 + "content": "You are a helpful assistant."
570 + },
571 + {
572 + "role":
573 + "user",
574 + "content": [
575 + {
576 + "type": "text",
577 + "text": "Describe this image.",
578 + },
579 + {
580 + "type": "image_url",
581 + "image_url": {},
582 + "uuid": "uuid_not_previously_seen"
583 + },
584 + ],
585 + },
586 + ],
587 + model=model_name,
588 + )
589 +
590
591 @pytest.mark.asyncio
592 @pytest.mark.parametrize("model_name", [MODEL_NAME])
tests/entrypoints/test_chat_utils.py
+546 -2
@@ -79,6 +79,28 @@ def phi3v_tokenizer():
79 )
80
81
82 +@pytest.fixture(scope="function")
83 +def qwen2_audio_model_config():
84 + return ModelConfig(
85 + QWEN2AUDIO_MODEL_ID,
86 + runner="generate",
87 + trust_remote_code=True,
88 + limit_mm_per_prompt={
89 + "audio": 1,
90 + },
91 + )
92 +
93 +
94 +@pytest.fixture(scope="module")
95 +def qwen2_audio_tokenizer():
96 + return TokenizerGroup(
97 + tokenizer_id=QWEN2AUDIO_MODEL_ID,
98 + enable_lora=False,
99 + max_num_seqs=5,
100 + max_input_length=None,
101 + )
102 +
103 +
104 @pytest.fixture(scope="function")
105 def qwen25omni_model_config_mm_interleaved():
106 return ModelConfig(
@@ -169,6 +191,7 @@ def audio_url():
191 def _assert_mm_data_is_image_input(
192 mm_data: Optional[MultiModalDataDict],
193 image_count: int,
194 + skipped_image_indices: Optional[list] = None,
195 ) -> None:
196 assert mm_data is not None
197 assert set(mm_data.keys()) == {"image"}
@@ -177,6 +200,9 @@ def _assert_mm_data_is_image_input(
200 assert image_data is not None
201
202 assert isinstance(image_data, list) and len(image_data) == image_count
203 + if skipped_image_indices is not None:
204 + for i in skipped_image_indices:
205 + assert image_data[i] is None
206
207
208 def _assert_mm_uuids(
@@ -205,8 +231,10 @@ MultiModalDataCounts = Mapping[ModalityType, int]
231
232
233 def _assert_mm_data_inputs(
208 - mm_data: Optional[MultiModalDataDict],
209 - data_count: MultiModalDataCounts,
234 + mm_data: Optional[MultiModalDataDict],
235 + data_count: MultiModalDataCounts,
236 + skipped_media_indices: Optional[dict[
237 + str, list]] = None, # modality -> list[int]
238 ) -> None:
239 assert mm_data is not None
240 assert set(data_count.keys()) == (set(mm_data.keys()))
@@ -216,6 +244,13 @@ def _assert_mm_data_inputs(
244 assert modality_data is not None
245 assert isinstance(modality_data, list) and len(modality_data) == n
246
247 + if skipped_media_indices is not None:
248 + skipped_media_indices_for_modality = skipped_media_indices.get(
249 + modality)
250 + assert skipped_media_indices_for_modality is not None
251 + for i in skipped_media_indices_for_modality:
252 + assert modality_data[i] is None
253 +
254
255 def test_parse_chat_messages_single_image(
256 phi3v_model_config,
@@ -289,6 +324,41 @@ def test_parse_chat_messages_single_image_with_uuid(
324 _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
325
326
327 +def test_parse_chat_messages_single_empty_image_with_uuid(
328 + phi3v_model_config,
329 + phi3v_tokenizer,
330 + image_url,
331 +):
332 + image_uuid = str(hash(image_url))
333 + conversation, mm_data, mm_uuids = parse_chat_messages(
334 + [{
335 + "role":
336 + "user",
337 + "content": [
338 + {
339 + "type": "image_url",
340 + "image_url": None,
341 + "uuid": image_uuid,
342 + },
343 + {
344 + "type": "text",
345 + "text": "What's in the image?"
346 + },
347 + ],
348 + }],
349 + phi3v_model_config,
350 + phi3v_tokenizer,
351 + content_format="string",
352 + )
353 +
354 + assert conversation == [{
355 + "role": "user",
356 + "content": "<|image_1|>\nWhat's in the image?"
357 + }]
358 + _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
359 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
360 +
361 +
362 def test_parse_chat_messages_single_image_with_bad_uuid_format(
363 phi3v_model_config,
364 phi3v_tokenizer,
@@ -375,6 +445,96 @@ def test_parse_chat_messages_multiple_images_with_uuids(
445 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
446
447
448 +def test_parse_chat_messages_multiple_empty_images_with_uuids(
449 + phi3v_model_config,
450 + phi3v_tokenizer,
451 + image_url,
452 +):
453 + image_uuid1 = "my_uuid_1"
454 + image_uuid2 = "my_uuid_2"
455 +
456 + conversation, mm_data, mm_uuids = parse_chat_messages(
457 + [{
458 + "role":
459 + "user",
460 + "content": [
461 + {
462 + "type": "image_url",
463 + "image_url": None,
464 + "uuid": image_uuid1,
465 + },
466 + {
467 + "type": "image_url",
468 + "image_url": None,
469 + "uuid": image_uuid2,
470 + },
471 + {
472 + "type": "text",
473 + "text": "What's in the image?"
474 + },
475 + ],
476 + }],
477 + phi3v_model_config,
478 + phi3v_tokenizer,
479 + content_format="string",
480 + )
481 +
482 + assert conversation == [{
483 + "role":
484 + "user",
485 + "content":
486 + "<|image_1|>\n<|image_2|>\nWhat's in the image?",
487 + }]
488 + _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1])
489 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
490 +
491 +
492 +def test_parse_chat_messages_mixed_empty_images_with_uuids(
493 + phi3v_model_config,
494 + phi3v_tokenizer,
495 + image_url,
496 +):
497 + image_uuid1 = "my_uuid_1"
498 + image_uuid2 = "my_uuid_2"
499 +
500 + conversation, mm_data, mm_uuids = parse_chat_messages(
501 + [{
502 + "role":
503 + "user",
504 + "content": [
505 + {
506 + "type": "image_url",
507 + "image_url": {
508 + "url": image_url,
509 + },
510 + "uuid": image_uuid1,
511 + },
512 + {
513 + "type": "image_url",
514 + "image_url": None,
515 + "uuid": image_uuid2,
516 + },
517 + {
518 + "type": "text",
519 + "text": "What's in the image?"
520 + },
521 + ],
522 + }],
523 + phi3v_model_config,
524 + phi3v_tokenizer,
525 + content_format="string",
526 + )
527 +
528 + assert conversation == [{
529 + "role":
530 + "user",
531 + "content":
532 + "<|image_1|>\n<|image_2|>\nWhat's in the image?",
533 + }]
534 + _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[1])
535 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
536 +
537 +
538 @pytest.mark.asyncio
539 async def test_parse_chat_messages_single_image_with_uuid_async(
540 phi3v_model_config,
@@ -413,6 +573,44 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
573 _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
574
575
576 +@pytest.mark.asyncio
577 +async def test_parse_chat_messages_empty_image_with_uuid_async(
578 + phi3v_model_config,
579 + phi3v_tokenizer,
580 + image_url,
581 +):
582 + image_uuid = str(hash(image_url))
583 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
584 + [{
585 + "role":
586 + "user",
587 + "content": [
588 + {
589 + "type": "image_url",
590 + "image_url": None,
591 + "uuid": image_uuid,
592 + },
593 + {
594 + "type": "text",
595 + "text": "What's in the image?"
596 + },
597 + ],
598 + }],
599 + phi3v_model_config,
600 + phi3v_tokenizer,
601 + content_format="string",
602 + )
603 +
604 + assert conversation == [{
605 + "role": "user",
606 + "content": "<|image_1|>\nWhat's in the image?"
607 + }]
608 + _assert_mm_data_is_image_input(await mm_future,
609 + 1,
610 + skipped_image_indices=[0])
611 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
612 +
613 +
614 @pytest.mark.asyncio
615 async def test_parse_chat_messages_multiple_images_with_uuids_async(
616 phi3v_model_config,
@@ -460,6 +658,53 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
658 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
659
660
661 +@pytest.mark.asyncio
662 +async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
663 + phi3v_model_config,
664 + phi3v_tokenizer,
665 + image_url,
666 +):
667 + image_uuid1 = "my_uuid_1"
668 + image_uuid2 = "my_uuid_2"
669 +
670 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
671 + [{
672 + "role":
673 + "user",
674 + "content": [
675 + {
676 + "type": "image_url",
677 + "image_url": None,
678 + "uuid": image_uuid1,
679 + },
680 + {
681 + "type": "image_pil",
682 + "image_pil": None,
683 + "uuid": image_uuid2,
684 + },
685 + {
686 + "type": "text",
687 + "text": "What's in these images?"
688 + },
689 + ],
690 + }],
691 + phi3v_model_config,
692 + phi3v_tokenizer,
693 + content_format="string",
694 + )
695 +
696 + assert conversation == [{
697 + "role":
698 + "user",
699 + "content":
700 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",
701 + }]
702 + _assert_mm_data_is_image_input(await mm_future,
703 + 2,
704 + skipped_image_indices=[0, 1])
705 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
706 +
707 +
708 @pytest.mark.asyncio
709 async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
710 phi3v_model_config,
@@ -653,6 +898,114 @@ def test_parse_chat_messages_multiple_images(
898 _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
899
900
901 +def test_parse_chat_messages_empty_pil_image_with_uuid(
902 + phi3v_model_config,
903 + phi3v_tokenizer,
904 +):
905 + uuid = "abcd"
906 + conversation, mm_data, mm_uuids = parse_chat_messages(
907 + [{
908 + "role":
909 + "user",
910 + "content": [
911 + {
912 + "type": "image_pil",
913 + "image_pil": None,
914 + "uuid": uuid
915 + },
916 + {
917 + "type": "text",
918 + "text": "What's in this image?"
919 + },
920 + ],
921 + }],
922 + phi3v_model_config,
923 + phi3v_tokenizer,
924 + content_format="string",
925 + )
926 +
927 + assert conversation == [{
928 + "role": "user",
929 + "content": "<|image_1|>\nWhat's in this image?",
930 + }]
931 + _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0])
932 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])
933 +
934 +
935 +def test_parse_chat_messages_empty_image_embeds_with_uuid(
936 + phi3v_model_config,
937 + phi3v_tokenizer,
938 +):
939 + uuid = "abcd"
940 + conversation, mm_data, mm_uuids = parse_chat_messages(
941 + [{
942 + "role":
943 + "user",
944 + "content": [
945 + {
946 + "type": "image_embeds",
947 + "image_embeds": None,
948 + "uuid": uuid
949 + },
950 + {
951 + "type": "text",
952 + "text": "What's in this image?"
953 + },
954 + ],
955 + }],
956 + phi3v_model_config,
957 + phi3v_tokenizer,
958 + content_format="string",
959 + )
960 +
961 + assert conversation == [{
962 + "role": "user",
963 + "content": "<|image_1|>\nWhat's in this image?",
964 + }]
965 + assert mm_data is not None
966 + assert "image" in mm_data
967 + assert mm_data["image"] is None
968 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])
969 +
970 +
971 +@pytest.mark.asyncio
972 +async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
973 + phi3v_model_config,
974 + phi3v_tokenizer,
975 +):
976 + uuid = "abcd"
977 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
978 + [{
979 + "role":
980 + "user",
981 + "content": [
982 + {
983 + "type": "image_embeds",
984 + "image_embeds": None,
985 + "uuid": uuid
986 + },
987 + {
988 + "type": "text",
989 + "text": "What's in this image?"
990 + },
991 + ],
992 + }],
993 + phi3v_model_config,
994 + phi3v_tokenizer,
995 + content_format="string",
996 + )
997 +
998 + assert conversation == [{
999 + "role": "user",
1000 + "content": "<|image_1|>\nWhat's in this image?",
1001 + }]
1002 + mm_data = await mm_future
1003 + assert mm_data is not None
1004 + assert "image" in mm_data
1005 + assert mm_data["image"] is None
1006 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[uuid])
1007 +
1008 +
1009 @pytest.mark.asyncio
1010 async def test_parse_chat_messages_multiple_images_async(
1011 phi3v_model_config,
@@ -1636,6 +1989,118 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
1989 expected_uuids=["audio_123"])
1990
1991
1992 +def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave( # noqa: E501
1993 + qwen25omni_model_config_mm_interleaved,
1994 + qwen25omni_tokenizer,
1995 + image_url,
1996 + video_url,
1997 + audio_url,
1998 +):
1999 + conversation, mm_data, mm_uuids = parse_chat_messages(
2000 + [
2001 + {
2002 + "role":
2003 + "user",
2004 + "content": [
2005 + {
2006 + "type": "text",
2007 + "text": "What's on this image?"
2008 + },
2009 + {
2010 + "type": "image_url",
2011 + "image_url": None,
2012 + "uuid": "image_123",
2013 + },
2014 + {
2015 + "type": "text",
2016 + "text": "Now listen to this audio"
2017 + },
2018 + {
2019 + "type": "audio_url",
2020 + "audio_url": None,
2021 + "uuid": "audio_123",
2022 + },
2023 + ],
2024 + },
2025 + {
2026 + "role": "assistant",
2027 + "content": "Some stuff."
2028 + },
2029 + {
2030 + "role":
2031 + "user",
2032 + "content": [
2033 + {
2034 + "type": "text",
2035 + "text": "What's on this image?"
2036 + },
2037 + {
2038 + "type": "image_url",
2039 + "image_url": None,
2040 + "uuid": "image_123",
2041 + },
2042 + {
2043 + "type": "text",
2044 + "text": "And what's in the video?"
2045 + },
2046 + {
2047 + "type": "video_url",
2048 + "video_url": None,
2049 + "uuid": "video_123",
2050 + },
2051 + ],
2052 + },
2053 + ],
2054 + qwen25omni_model_config_mm_interleaved,
2055 + qwen25omni_tokenizer,
2056 + content_format="string",
2057 + )
2058 +
2059 + assert conversation == [
2060 + {
2061 + "role":
2062 + "user",
2063 + "content":
2064 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
2065 + "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
2066 + },
2067 + {
2068 + "role": "assistant",
2069 + "content": "Some stuff."
2070 + },
2071 + {
2072 + "role":
2073 + "user",
2074 + "content":
2075 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
2076 + "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
2077 + },
2078 + ]
2079 +
2080 + _assert_mm_data_inputs(mm_data, {
2081 + "image": 2,
2082 + "video": 1,
2083 + "audio": 1
2084 + },
2085 + skipped_media_indices={
2086 + "image": [0, 1],
2087 + "video": [0],
2088 + "audio": [0]
2089 + })
2090 + _assert_mm_uuids(mm_uuids,
2091 + 2,
2092 + modality="image",
2093 + expected_uuids=["image_123", "image_123"])
2094 + _assert_mm_uuids(mm_uuids,
2095 + 1,
2096 + modality="video",
2097 + expected_uuids=["video_123"])
2098 + _assert_mm_uuids(mm_uuids,
2099 + 1,
2100 + modality="audio",
2101 + expected_uuids=["audio_123"])
2102 +
2103 +
2104 def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501
2105 qwen25omni_model_config_mm_interleaved,
2106 qwen25omni_tokenizer,
@@ -2355,3 +2820,82 @@ def test_apply_mistral_chat_template_thinking_chunk():
2820 r"[INST]Thanks, what is 3+3?[/INST]")
2821
2822 assert string_tokens == expected_tokens
2823 +
2824 +
2825 +def test_parse_chat_messages_single_empty_audio_with_uuid(
2826 + qwen2_audio_model_config,
2827 + qwen2_audio_tokenizer,
2828 +):
2829 + audio_uuid = "abcd"
2830 + conversation, mm_data, mm_uuids = parse_chat_messages(
2831 + [{
2832 + "role":
2833 + "user",
2834 + "content": [
2835 + {
2836 + "type": "input_audio",
2837 + "input_audio": {},
2838 + "uuid": audio_uuid,
2839 + },
2840 + {
2841 + "type": "text",
2842 + "text": "What does the audio say?"
2843 + },
2844 + ],
2845 + }],
2846 + qwen2_audio_model_config,
2847 + qwen2_audio_tokenizer,
2848 + content_format="string",
2849 + )
2850 +
2851 + assert conversation == [{
2852 + "role":
2853 + "user",
2854 + "content":
2855 + "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
2856 + }]
2857 + _assert_mm_data_inputs(mm_data, {"audio": 1})
2858 + _assert_mm_uuids(mm_uuids,
2859 + 1,
2860 + modality="audio",
2861 + expected_uuids=[audio_uuid])
2862 +
2863 +
2864 +@pytest.mark.asyncio
2865 +async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2866 + qwen2_audio_model_config,
2867 + qwen2_audio_tokenizer,
2868 +):
2869 + audio_uuid = "abcd"
2870 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
2871 + [{
2872 + "role":
2873 + "user",
2874 + "content": [
2875 + {
2876 + "type": "input_audio",
2877 + "input_audio": {},
2878 + "uuid": audio_uuid,
2879 + },
2880 + {
2881 + "type": "text",
2882 + "text": "What does the audio say?"
2883 + },
2884 + ],
2885 + }],
2886 + qwen2_audio_model_config,
2887 + qwen2_audio_tokenizer,
2888 + content_format="string",
2889 + )
2890 +
2891 + assert conversation == [{
2892 + "role":
2893 + "user",
2894 + "content":
2895 + "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\nWhat does the audio say?"
2896 + }]
2897 + _assert_mm_data_inputs(await mm_future, {"audio": 1})
2898 + _assert_mm_uuids(mm_uuids,
2899 + 1,
2900 + modality="audio",
2901 + expected_uuids=[audio_uuid])
vllm/entrypoints/chat_utils.py
+155 -73
@@ -73,15 +73,10 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
73
74 type: Required[Literal["audio_url"]]
75 """The type of the content part."""
76 - uuid: Optional[str]
77 - """
78 - User-provided UUID of a media. User must guarantee that it is properly
79 - generated and unique for different medias.
80 - """
76
77
78 class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
84 - image_embeds: Required[Union[str, dict[str, str]]]
79 + image_embeds: Optional[Union[str, dict[str, str]]]
80 """
81 The image embeddings. It can be either:
82 - A single base64 string.
@@ -108,11 +103,6 @@ class ChatCompletionContentPartVideoParam(TypedDict, total=False):
103
104 type: Required[Literal["video_url"]]
105 """The type of the content part."""
111 - uuid: Optional[str]
112 - """
113 - User-provided UUID of a media. User must guarantee that it is properly
114 - generated and unique for different medias.
115 - """
106
107
108 class PILImage(BaseModel):
@@ -133,7 +123,7 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
123 }
124 """
125
136 - image_pil: Required[PILImage]
126 + image_pil: Optional[PILImage]
127 uuid: Optional[str]
128 """
129 User-provided UUID of a media. User must guarantee that it is properly
@@ -151,7 +141,7 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
141 }
142 """
143
154 - image_url: Required[str]
144 + image_url: Optional[str]
145 uuid: Optional[str]
146 """
147 User-provided UUID of a media. User must guarantee that it is properly
@@ -168,7 +158,7 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
158 }
159 """
160
171 - audio_url: Required[str]
161 + audio_url: Optional[str]
162
163
164 class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
@@ -180,7 +170,7 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
170 }
171 """
172
183 - video_url: Required[str]
173 + video_url: Optional[str]
174 uuid: Optional[str]
175 """
176 User-provided UUID of a media. User must guarantee that it is properly
@@ -597,7 +587,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
587 self._model_config = model_config
588 self._tokenizer = tokenizer
589
600 - self._items_by_modality = defaultdict[str, list[_T]](list)
590 + self._items_by_modality = defaultdict[str, list[Optional[_T]]](list)
591 self._uuids_by_modality = defaultdict[str, list[Optional[str]]](list)
592
593 @property
@@ -624,14 +614,17 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
614 return self.mm_registry.create_processor(self.model_config)
615
616 def add(
627 - self, modality: ModalityStr, item: _T, uuid: Optional[str] = None
617 + self,
618 + modality: ModalityStr,
619 + item: Optional[_T],
620 + uuid: Optional[str] = None,
621 ) -> Optional[str]:
622 """
623 Add a multi-modal item to the current prompt and returns the
624 placeholder string to use, if any.
625
626 An optional uuid can be added which serves as a unique identifier of the
634 - media.
627 + media.
628 """
629 input_modality = modality.replace("_embeds", "")
630 num_items = len(self._items_by_modality[modality]) + 1
@@ -708,10 +701,15 @@ class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
701 if not self._items_by_modality:
702 return None
703 mm_inputs = {}
711 - items_by_modality = {
712 - modality: await asyncio.gather(*items)
713 - for modality, items in self._items_by_modality.items()
714 - }
704 + items_by_modality = {}
705 + for modality, items in self._items_by_modality.items():
706 + coros = []
707 + for item in items:
708 + if item is not None:
709 + coros.append(item)
710 + else:
711 + coros.append(asyncio.sleep(0))
712 + items_by_modality[modality] = await asyncio.gather(*coros)
713
714 if "image" in items_by_modality and "image_embeds" in items_by_modality:
715 raise ValueError(
@@ -760,35 +758,40 @@ class BaseMultiModalContentParser(ABC):
758 return dict(self._placeholder_storage)
759
760 @abstractmethod
763 - def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
761 + def parse_image(
762 + self, image_url: Optional[str], uuid: Optional[str] = None) -> None:
763 raise NotImplementedError
764
765 @abstractmethod
766 def parse_image_embeds(
767 self,
769 - image_embeds: Union[str, dict[str, str]],
768 + image_embeds: Union[str, dict[str, str], None],
769 uuid: Optional[str] = None,
770 ) -> None:
771 raise NotImplementedError
772
773 @abstractmethod
774 def parse_image_pil(
776 - self, image_pil: Image.Image, uuid: Optional[str] = None
775 + self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
776 ) -> None:
777 raise NotImplementedError
778
779 @abstractmethod
781 - def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
780 + def parse_audio(
781 + self, audio_url: Optional[str], uuid: Optional[str] = None
782 + ) -> None:
783 raise NotImplementedError
784
785 @abstractmethod
786 def parse_input_audio(
786 - self, input_audio: InputAudio, uuid: Optional[str] = None
787 + self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
788 ) -> None:
789 raise NotImplementedError
790
791 @abstractmethod
791 - def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
792 + def parse_video(
793 + self, video_url: Optional[str], uuid: Optional[str] = None
794 + ) -> None:
795 raise NotImplementedError
796
797
@@ -803,15 +806,17 @@ class MultiModalContentParser(BaseMultiModalContentParser):
806 allowed_local_media_path=tracker.allowed_local_media_path,
807 )
808
806 - def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
807 - image = self._connector.fetch_image(image_url)
809 + def parse_image(
810 + self, image_url: Optional[str], uuid: Optional[str] = None
811 + ) -> None:
812 + image = self._connector.fetch_image(image_url) if image_url else None
813
814 placeholder = self._tracker.add("image", image, uuid)
815 self._add_placeholder("image", placeholder)
816
817 def parse_image_embeds(
818 self,
814 - image_embeds: Union[str, dict[str, str]],
819 + image_embeds: Union[str, dict[str, str], None],
820 uuid: Optional[str] = None,
821 ) -> None:
822 if isinstance(image_embeds, dict):
@@ -825,31 +830,49 @@ class MultiModalContentParser(BaseMultiModalContentParser):
830 embedding = self._connector.fetch_image_embedding(image_embeds)
831 placeholder = self._tracker.add("image_embeds", embedding, uuid)
832
833 + if image_embeds is None:
834 + placeholder = self._tracker.add("image_embeds", None, uuid)
835 +
836 self._add_placeholder("image", placeholder)
837
838 def parse_image_pil(
831 - self, image_pil: Image.Image, uuid: Optional[str] = None
839 + self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
840 ) -> None:
841 placeholder = self._tracker.add("image", image_pil, uuid)
842 self._add_placeholder("image", placeholder)
843
836 - def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
837 - audio = self._connector.fetch_audio(audio_url)
844 + def parse_audio(
845 + self, audio_url: Optional[str], uuid: Optional[str] = None
846 + ) -> None:
847 + audio = self._connector.fetch_audio(audio_url) if audio_url else None
848
849 placeholder = self._tracker.add("audio", audio, uuid)
850 self._add_placeholder("audio", placeholder)
851
852 def parse_input_audio(
843 - self, input_audio: InputAudio, uuid: Optional[str] = None
853 + self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
854 ) -> None:
845 - audio_data = input_audio.get("data", "")
846 - audio_format = input_audio.get("format", "")
847 - audio_url = f"data:audio/{audio_format};base64,{audio_data}"
855 + if input_audio:
856 + audio_data = input_audio.get("data", "")
857 + audio_format = input_audio.get("format", "")
858 + if audio_data:
859 + audio_url = f"data:audio/{audio_format};base64,{audio_data}"
860 + else:
861 + # If a UUID is provided, audio data may be empty.
862 + audio_url = None
863 + else:
864 + audio_url = None
865
866 return self.parse_audio(audio_url, uuid)
867
851 - def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
852 - video = self._connector.fetch_video(video_url=video_url)
868 + def parse_video(
869 + self, video_url: Optional[str], uuid: Optional[str] = None
870 + ) -> None:
871 + video = (
872 + self._connector.fetch_video(video_url=video_url)
873 + if video_url
874 + else None
875 + )
876
877 placeholder = self._tracker.add("video", video, uuid)
878 self._add_placeholder("video", placeholder)
@@ -865,18 +888,24 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
888 allowed_local_media_path=tracker.allowed_local_media_path,
889 )
890
868 - def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
869 - image_coro = self._connector.fetch_image_async(image_url)
891 + def parse_image(
892 + self, image_url: Optional[str], uuid: Optional[str] = None
893 + ) -> None:
894 + image_coro = (
895 + self._connector.fetch_image_async(image_url) if image_url else None
896 + )
897
898 placeholder = self._tracker.add("image", image_coro, uuid)
899 self._add_placeholder("image", placeholder)
900
901 def parse_image_embeds(
902 self,
876 - image_embeds: Union[str, dict[str, str]],
903 + image_embeds: Union[str, dict[str, str], None],
904 uuid: Optional[str] = None,
905 ) -> None:
879 - future: asyncio.Future[Union[str, dict[str, str]]] = asyncio.Future()
906 + future: asyncio.Future[Union[str, dict[str, str], None]] = (
907 + asyncio.Future()
908 + )
909
910 if isinstance(image_embeds, dict):
911 embeds = {
@@ -889,35 +918,58 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
918 embedding = self._connector.fetch_image_embedding(image_embeds)
919 future.set_result(embedding)
920
921 + if image_embeds is None:
922 + future.set_result(None)
923 +
924 placeholder = self._tracker.add("image_embeds", future, uuid)
925 self._add_placeholder("image", placeholder)
926
927 def parse_image_pil(
896 - self, image_pil: Image.Image, uuid: Optional[str] = None
928 + self, image_pil: Optional[Image.Image], uuid: Optional[str] = None
929 ) -> None:
898 - future: asyncio.Future[Image.Image] = asyncio.Future()
899 - future.set_result(image_pil)
930 + future: asyncio.Future[Optional[Image.Image]] = asyncio.Future()
931 + if image_pil:
932 + future.set_result(image_pil)
933 + else:
934 + future.set_result(None)
935
936 placeholder = self._tracker.add("image", future, uuid)
937 self._add_placeholder("image", placeholder)
938
904 - def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
905 - audio_coro = self._connector.fetch_audio_async(audio_url)
939 + def parse_audio(
940 + self, audio_url: Optional[str], uuid: Optional[str] = None
941 + ) -> None:
942 + audio_coro = (
943 + self._connector.fetch_audio_async(audio_url) if audio_url else None
944 + )
945
946 placeholder = self._tracker.add("audio", audio_coro, uuid)
947 self._add_placeholder("audio", placeholder)
948
949 def parse_input_audio(
911 - self, input_audio: InputAudio, uuid: Optional[str] = None
950 + self, input_audio: Optional[InputAudio], uuid: Optional[str] = None
951 ) -> None:
913 - audio_data = input_audio.get("data", "")
914 - audio_format = input_audio.get("format", "")
915 - audio_url = f"data:audio/{audio_format};base64,{audio_data}"
952 + if input_audio:
953 + audio_data = input_audio.get("data", "")
954 + audio_format = input_audio.get("format", "")
955 + if audio_data:
956 + audio_url = f"data:audio/{audio_format};base64,{audio_data}"
957 + else:
958 + # If a UUID is provided, audio data may be empty.
959 + audio_url = None
960 + else:
961 + audio_url = None
962
963 return self.parse_audio(audio_url, uuid)
964
919 - def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
920 - video = self._connector.fetch_video_async(video_url=video_url)
965 + def parse_video(
966 + self, video_url: Optional[str], uuid: Optional[str] = None
967 + ) -> None:
968 + video = (
969 + self._connector.fetch_video_async(video_url=video_url)
970 + if video_url
971 + else None
972 + )
973
974 placeholder = self._tracker.add("video", video, uuid)
975 self._add_placeholder("video", placeholder)
@@ -1130,8 +1182,9 @@ def _parse_chat_message_content_mm_part(
1182 part, dict
1183 ) # This is needed to avoid mypy errors: part.get() from str
1184 part_type = part.get("type", None)
1185 + uuid = part.get("uuid", None)
1186
1134 - if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
1187 + if isinstance(part_type, str) and part_type in MM_PARSER_MAP and uuid is None: # noqa: E501
1188 content = MM_PARSER_MAP[part_type](part)
1189
1190 # Special case for 'image_url.detail'
@@ -1146,25 +1199,54 @@ def _parse_chat_message_content_mm_part(
1199
1200 # Handle missing 'type' but provided direct URL fields.
1201 # 'type' is required field by pydantic
1149 - if part_type is None:
1150 - if part.get("image_url") is not None:
1202 + if part_type is None or uuid is not None:
1203 + if "image_url" in part:
1204 image_params = cast(
1205 CustomChatCompletionContentSimpleImageParam, part
1206 )
1154 - return "image_url", image_params.get("image_url", "")
1155 - if part.get("audio_url") is not None:
1207 + image_url = image_params.get("image_url", None)
1208 + if isinstance(image_url, dict):
1209 + # Can potentially happen if user provides a uuid
1210 + # with url as a dict of {"url": url}
1211 + image_url = image_url.get("url", None)
1212 + return "image_url", image_url
1213 + if "image_pil" in part:
1214 + # "image_pil" could be None if UUID is provided.
1215 + image_params = cast( # type: ignore
1216 + CustomChatCompletionContentPILImageParam, part
1217 + )
1218 + image_pil = image_params.get("image_pil", None)
1219 + return "image_pil", image_pil
1220 + if "image_embeds" in part:
1221 + # "image_embeds" could be None if UUID is provided.
1222 + image_params = cast( # type: ignore
1223 + ChatCompletionContentPartImageEmbedsParam, part
1224 + )
1225 + image_embeds = image_params.get("image_embeds", None)
1226 + return "image_embeds", image_embeds
1227 + if "audio_url" in part:
1228 audio_params = cast(
1229 CustomChatCompletionContentSimpleAudioParam, part
1230 )
1159 - return "audio_url", audio_params.get("audio_url", "")
1231 + audio_url = audio_params.get("audio_url", None)
1232 + if isinstance(audio_url, dict):
1233 + # Can potentially happen if user provides a uuid
1234 + # with url as a dict of {"url": url}
1235 + audio_url = audio_url.get("url", None)
1236 + return "audio_url", audio_url
1237 if part.get("input_audio") is not None:
1238 input_audio_params = cast(dict[str, str], part)
1239 return "input_audio", input_audio_params
1163 - if part.get("video_url") is not None:
1240 + if "video_url" in part:
1241 video_params = cast(
1242 CustomChatCompletionContentSimpleVideoParam, part
1243 )
1167 - return "video_url", video_params.get("video_url", "")
1244 + video_url = video_params.get("video_url", None)
1245 + if isinstance(video_url, dict):
1246 + # Can potentially happen if user provides a uuid
1247 + # with url as a dict of {"url": url}
1248 + video_url = video_url.get("url", None)
1249 + return "video_url", video_url
1250 # Raise an error if no 'type' or direct URL is found.
1251 raise ValueError("Missing 'type' field in multimodal part.")
1252
@@ -1173,15 +1255,9 @@ def _parse_chat_message_content_mm_part(
1255 return part_type, "unknown part_type content"
1256
1257
1176 -VALID_MESSAGE_CONTENT_MM_PART_TYPES = (
1258 +PART_TYPES_TO_SKIP_NONE_CONTENT = (
1259 "text",
1260 "refusal",
1179 - "image_url",
1180 - "image_embeds",
1181 - "image_pil",
1182 - "audio_url",
1183 - "input_audio",
1184 - "video_url",
1261 )
1262
1263
@@ -1242,7 +1318,7 @@ def _parse_chat_message_content_part(
1318 part_type, content = _parse_chat_message_content_mm_part(part)
1319 # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1320 # content is None, log a warning and skip
1245 - if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:
1321 + if part_type in PART_TYPES_TO_SKIP_NONE_CONTENT and content is None:
1322 logger.warning(
1323 "Skipping multimodal part '%s' (type: '%s') "
1324 "with empty / unparsable content.",
@@ -1266,7 +1342,10 @@ def _parse_chat_message_content_part(
1342
1343 modality = None
1344 if part_type == "image_pil":
1269 - image_content = cast(Image.Image, content)
1345 + if content is not None:
1346 + image_content = cast(Image.Image, content)
1347 + else:
1348 + image_content = None
1349 mm_parser.parse_image_pil(image_content, uuid)
1350 modality = "image"
1351 elif part_type in ("image_url", "input_image"):
@@ -1274,7 +1353,10 @@ def _parse_chat_message_content_part(
1353 mm_parser.parse_image(str_content, uuid)
1354 modality = "image"
1355 elif part_type == "image_embeds":
1277 - content = cast(Union[str, dict[str, str]], content)
1356 + if content is not None:
1357 + content = cast(Union[str, dict[str, str]], content)
1358 + else:
1359 + content = None
1360 mm_parser.parse_image_embeds(content, uuid)
1361 modality = "image"
1362 elif part_type == "audio_url":
vllm/entrypoints/llm.py
+40
@@ -1491,6 +1491,11 @@ class LLM:
1491
1492 for i, prompt in enumerate(it):
1493
1494 + if isinstance(prompt, dict):
1495 + self._validate_mm_data_and_uuids(
1496 + prompt.get("multi_modal_data"),
1497 + prompt.get("multi_modal_uuids"))
1498 +
1499 param = params[i] if isinstance(params, Sequence) else params
1500
1501 tokenization_kwargs: dict[str, Any] = {}
@@ -1507,6 +1512,41 @@ class LLM:
1512 priority=priority[i] if priority else 0,
1513 )
1514
1515 + def _validate_mm_data_and_uuids(
1516 + self,
1517 + multi_modal_data: Optional[Any], # MultiModalDataDict
1518 + multi_modal_uuids: Optional[Any], # MultiModalUUIDDict
1519 + ):
1520 + """
1521 + Validate that if any multi-modal data is skipped (i.e. None),
1522 + then its corresponding UUID must be set.
1523 + """
1524 + if multi_modal_data is None:
1525 + return
1526 +
1527 + for modality, data in multi_modal_data.items():
1528 + if isinstance(data, list):
1529 + for i, d in enumerate(data):
1530 + if d is None:
1531 + if multi_modal_uuids is None or modality not in multi_modal_uuids or multi_modal_uuids[ # noqa: E501
1532 + modality] is None:
1533 + raise ValueError(
1534 + f"Multi-modal data for {modality} is None "
1535 + f"but UUID is not provided")
1536 + else:
1537 + if len(
1538 + multi_modal_uuids[modality]
1539 + ) <= i or multi_modal_uuids[modality][i] is None:
1540 + raise ValueError(
1541 + f"Multi-modal data for {modality} is None "
1542 + f"but UUID is not provided")
1543 + else:
1544 + if data is None and (multi_modal_uuids is None
1545 + or modality not in multi_modal_uuids
1546 + or multi_modal_uuids[modality] is None):
1547 + raise ValueError(f"Multi-modal data for {modality} is None"
1548 + f" but UUID is not provided")
1549 +
1550 def _add_request(
1551 self,
1552 prompt: PromptType,
vllm/multimodal/inputs.py
+3 -2
@@ -85,9 +85,10 @@ which are treated as audio embeddings;
85 these are directly passed to the model without HF processing.
86 """
87
88 -ModalityData: TypeAlias = Union[_T, list[_T]]
88 +ModalityData: TypeAlias = Union[_T, list[Optional[_T]], None]
89 """
90 -Either a single data item, or a list of data items.
90 +Either a single data item, or a list of data items. Can only be None if UUID
91 +is provided.
92
93 The number of data items allowed per modality is restricted by
94 `--limit-mm-per-prompt`.
vllm/multimodal/parse.py
+19 -4
@@ -36,7 +36,7 @@ class ModalityDataItems(ABC, Generic[_T, _I]):
36 def __init__(self, data: _T, modality: str) -> None:
37 super().__init__()
38
39 - self.data = data
39 + self.data: _T = data
40 self.modality = modality
41
42 def __repr__(self) -> str:
@@ -177,7 +177,9 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],
177
178 class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
179
180 - def __init__(self, data: Sequence[HfAudioItem]) -> None:
180 + def __init__(self, data: Optional[Sequence[HfAudioItem]]) -> None:
181 + if data is None:
182 + data = [None]
183 super().__init__(data, "audio")
184
185 def get_audio_length(self, item_idx: int) -> int:
@@ -198,7 +200,9 @@ class ImageSize(NamedTuple):
200
201 class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
202
201 - def __init__(self, data: Sequence[HfImageItem]) -> None:
203 + def __init__(self, data: Optional[Sequence[HfImageItem]]) -> None:
204 + if data is None:
205 + data = [None]
206 super().__init__(data, "image")
207
208 def get_image_size(self, item_idx: int) -> ImageSize:
@@ -223,10 +227,12 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
227
228 def __init__(
229 self,
226 - data: Sequence[HfVideoItem],
230 + data: Optional[Sequence[HfVideoItem]],
231 metadata: Optional[Union[dict[str, Any],
232 list[Optional[dict[str, Any]]]]] = None,
233 ) -> None:
234 + if data is None:
235 + data = [None]
236 super().__init__(data, "video")
237 self.metadata = metadata
238
@@ -385,6 +391,9 @@ class MultiModalDataParser:
391 self,
392 data: ModalityData[AudioItem],
393 ) -> Optional[ModalityDataItems[Any, Any]]:
394 + if data is None:
395 + return AudioProcessorItems(None)
396 +
397 # also check single audio item with sampling rate
398 if self._is_empty(data) or (isinstance(data, tuple)
399 and self._is_empty(data[0])):
@@ -420,6 +429,9 @@ class MultiModalDataParser:
429 self,
430 data: ModalityData[ImageItem],
431 ) -> Optional[ModalityDataItems[Any, Any]]:
432 + if data is None:
433 + return ImageProcessorItems(None)
434 +
435 if self._is_empty(data):
436 return None
437
@@ -441,6 +453,9 @@ class MultiModalDataParser:
453 self,
454 data: ModalityData[VideoItem],
455 ) -> Optional[ModalityDataItems[Any, Any]]:
456 + if data is None:
457 + return VideoProcessorItems(None)
458 +
459 if self._is_empty(data):
460 return None
461
vllm/multimodal/processing.py
+12 -5
@@ -1075,7 +1075,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1075 [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1076 """
1077 mm_items = self.data_parser.parse_mm_data(mm_data)
1078 -
1078 for modality, items in mm_items.items():
1079 self.validate_num_items(modality, len(items))
1080
@@ -1436,10 +1435,18 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1435 ]
1436 for modality, items_is_cached in mm_is_cached.items()
1437 }
1439 - mm_missing_data = {
1440 - modality: [mm_data_items[modality][idx] for idx in idxs]
1441 - for modality, idxs in mm_missing_idxs.items()
1442 - }
1438 + mm_missing_data = {}
1439 + for modality, idxs in mm_missing_idxs.items():
1440 + missing_modality_data = []
1441 + for idx in idxs:
1442 + data = mm_data_items[modality][idx]
1443 + if data is None:
1444 + raise ValueError(
1445 + f"Cache miss for {modality} at index {idx} "
1446 + f"but data is not provided.")
1447 + else:
1448 + missing_modality_data.append(data)
1449 + mm_missing_data[modality] = missing_modality_data
1450
1451 return self._to_mm_items(mm_missing_data)
1452