@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 01dfb5e982
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Frontend] User-provided uuids for medias in chat. (RFC #22044) (#23449)

Signed-off-by: Roger Wang <hey@rogerw.io> Signed-off-by: Chenheli Hua <huachenheli@outlook.com> Signed-off-by: Roger Wang <hey@rogerw.me> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.me> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Chenheli Hua committed Sep 8, 2025 at 06:42 UTC 01dfb5e982f45fff33c715e8c248555b0361276e
8 files changed +1087 -87
docs/features/multimodal_inputs.md
+33 -9
@@ -215,19 +215,19 @@ When loading RGBA images (images with transparency), vLLM converts them to RGB f
215
216 ```python
217 from vllm import LLM
218 -
218 +
219 # Default white background (no configuration needed)
220 llm = LLM(model="llava-hf/llava-1.5-7b-hf")
221 -
221 +
222 # Custom black background for dark theme
223 llm = LLM(
224 model="llava-hf/llava-1.5-7b-hf",
225 media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}}
226 )
227 -
227 +
228 # Custom brand color background (e.g., blue)
229 llm = LLM(
230 - model="llava-hf/llava-1.5-7b-hf",
230 + model="llava-hf/llava-1.5-7b-hf",
231 media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}}
232 )
233 ```
@@ -388,7 +388,7 @@ For Qwen2-VL and MiniCPM-V, we accept additional parameters alongside the embedd
388
389 ## Online Serving
390
391 -Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions API](https://platform.openai.com/docs/api-reference/chat).
391 +Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions API](https://platform.openai.com/docs/api-reference/chat). Media inputs also support optional UUIDs users can provide to uniquely identify each media, which is used to cache the media results across requests.
392
393 !!! important
394 A chat template is **required** to use Chat Completions API.
@@ -438,7 +438,13 @@ Then, you can use the OpenAI client as follows:
438 # NOTE: The prompt formatting with the image token `<image>` is not needed
439 # since the prompt will be processed automatically by the API server.
440 {"type": "text", "text": "What’s in this image?"},
441 - {"type": "image_url", "image_url": {"url": image_url}},
441 + {
442 + "type": "image_url",
443 + "image_url": {
444 + url": image_url
445 + },
446 + "uuid": image_url # Optional
447 + },
448 ],
449 }],
450 )
@@ -454,8 +460,20 @@ Then, you can use the OpenAI client as follows:
460 "role": "user",
461 "content": [
462 {"type": "text", "text": "What are the animals in these images?"},
457 - {"type": "image_url", "image_url": {"url": image_url_duck}},
458 - {"type": "image_url", "image_url": {"url": image_url_lion}},
463 + {
464 + "type": "image_url",
465 + "image_url": {
466 + "url": image_url_duck
467 + },
468 + "uuid": image_url_duck # Optional
469 + },
470 + {
471 + "type": "image_url",
472 + "image_url": {
473 + "url": image_url_lion
474 + },
475 + "uuid": image_url_lion # Optional
476 + },
477 ],
478 }],
479 )
@@ -522,6 +540,7 @@ Then, you can use the OpenAI client as follows:
540 "video_url": {
541 "url": video_url
542 },
543 + "uuid": video_url # Optional
544 },
545 ],
546 }],
@@ -613,6 +632,7 @@ Then, you can use the OpenAI client as follows:
632 "data": audio_base64,
633 "format": "wav"
634 },
635 + "uuid": audio_url # Optional
636 },
637 ],
638 }],
@@ -642,6 +662,7 @@ Alternatively, you can pass `audio_url`, which is the audio counterpart of `imag
662 "audio_url": {
663 "url": audio_url
664 },
665 + "uuid": audio_url # Optional
666 },
667 ],
668 }],
@@ -695,7 +716,8 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
716 model = "llava-hf/llava-1.5-7b-hf"
717 embeds = {
718 "type": "image_embeds",
698 - "image_embeds": f"{base64_image_embedding}"
719 + "image_embeds": f"{base64_image_embedding}",
720 + "uuid": image_url # Optional
721 }
722
723 # Pass additional parameters (available to Qwen2-VL and MiniCPM-V)
@@ -706,6 +728,7 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
728 "image_embeds": f"{base64_image_embedding}" , # Required
729 "image_grid_thw": f"{base64_image_grid_thw}" # Required by Qwen/Qwen2-VL-2B-Instruct
730 },
731 + "uuid": image_url # Optional
732 }
733 model = "openbmb/MiniCPM-V-2_6"
734 embeds = {
@@ -714,6 +737,7 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
737 "image_embeds": f"{base64_image_embedding}" , # Required
738 "image_sizes": f"{base64_image_sizes}" # Required by openbmb/MiniCPM-V-2_6
739 },
740 + "uuid": image_url # Optional
741 }
742 chat_completion = client.chat.completions.create(
743 messages=[
tests/entrypoints/openai/test_vision.py
+129
@@ -436,3 +436,132 @@ async def test_multi_image_input(client: openai.AsyncOpenAI, model_name: str,
436 )
437 message = chat_completion.choices[0].message
438 assert message.content is not None and len(message.content) >= 0
439 +
440 +
441 +@pytest.mark.asyncio
442 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
443 +@pytest.mark.parametrize(
444 + "image_urls",
445 + [TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
446 + indirect=True)
447 +async def test_completions_with_image(
448 + client: openai.AsyncOpenAI,
449 + model_name: str,
450 + image_urls: list[str],
451 +):
452 + for image_url in image_urls:
453 + chat_completion = await client.chat.completions.create(
454 + messages=[
455 + {
456 + "role": "system",
457 + "content": "You are a helpful assistant."
458 + },
459 + {
460 + "role":
461 + "user",
462 + "content": [
463 + {
464 + "type": "text",
465 + "text": "Describe this image.",
466 + },
467 + {
468 + "type": "image_url",
469 + "image_url": {
470 + "url": image_url,
471 + }
472 + },
473 + ],
474 + },
475 + ],
476 + model=model_name,
477 + )
478 + assert chat_completion.choices[0].message.content is not None
479 + assert isinstance(chat_completion.choices[0].message.content, str)
480 + assert len(chat_completion.choices[0].message.content) > 0
481 +
482 +
483 +@pytest.mark.asyncio
484 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
485 +@pytest.mark.parametrize(
486 + "image_urls",
487 + [TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
488 + indirect=True)
489 +async def test_completions_with_image_with_uuid(
490 + client: openai.AsyncOpenAI,
491 + model_name: str,
492 + image_urls: list[str],
493 +):
494 + for image_url in image_urls:
495 + chat_completion = await client.chat.completions.create(
496 + messages=[
497 + {
498 + "role": "system",
499 + "content": "You are a helpful assistant."
500 + },
501 + {
502 + "role":
503 + "user",
504 + "content": [
505 + {
506 + "type": "text",
507 + "text": "Describe this image.",
508 + },
509 + {
510 + "type": "image_url",
511 + "image_url": {
512 + "url": image_url,
513 + },
514 + "uuid": image_url
515 + },
516 + ],
517 + },
518 + ],
519 + model=model_name,
520 + )
521 + assert chat_completion.choices[0].message.content is not None
522 + assert isinstance(chat_completion.choices[0].message.content, str)
523 + assert len(chat_completion.choices[0].message.content) > 0
524 +
525 +
526 +@pytest.mark.asyncio
527 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
528 +@pytest.mark.parametrize(
529 + "image_urls",
530 + [TEST_IMAGE_ASSETS[:i] for i in range(2, len(TEST_IMAGE_ASSETS))],
531 + indirect=True)
532 +async def test_completions_with_image_with_incorrect_uuid_format(
533 + client: openai.AsyncOpenAI,
534 + model_name: str,
535 + image_urls: list[str],
536 +):
537 + for image_url in image_urls:
538 + chat_completion = await client.chat.completions.create(
539 + messages=[
540 + {
541 + "role": "system",
542 + "content": "You are a helpful assistant."
543 + },
544 + {
545 + "role":
546 + "user",
547 + "content": [
548 + {
549 + "type": "text",
550 + "text": "Describe this image.",
551 + },
552 + {
553 + "type": "image_url",
554 + "image_url": {
555 + "url": image_url,
556 + "incorrect_uuid_key": image_url,
557 + },
558 + "also_incorrect_uuid_key": image_url,
559 + },
560 + ],
561 + },
562 + ],
563 + model=model_name,
564 + )
565 + assert chat_completion.choices[0].message.content is not None
566 + assert isinstance(chat_completion.choices[0].message.content, str)
567 + assert len(chat_completion.choices[0].message.content) > 0
tests/entrypoints/test_chat_utils.py
+740 -28
@@ -21,7 +21,7 @@ from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
21 resolve_chat_template_content_format,
22 resolve_hf_chat_template)
23 from vllm.entrypoints.llm import apply_hf_chat_template
24 -from vllm.multimodal import MultiModalDataDict
24 +from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
25 from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,
26 encode_video_base64)
27 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -179,6 +179,27 @@ def _assert_mm_data_is_image_input(
179 assert isinstance(image_data, list) and len(image_data) == image_count
180
181
182 +def _assert_mm_uuids(
183 + mm_uuids: Optional[MultiModalUUIDDict],
184 + media_count: int,
185 + expected_uuids: list[Optional[str]],
186 + modality: str = "image",
187 +) -> None:
188 + if len(expected_uuids) > 0:
189 + assert mm_uuids is not None
190 + assert modality in mm_uuids
191 +
192 + image_uuids = mm_uuids.get(modality)
193 + assert image_uuids is not None
194 +
195 + assert isinstance(image_uuids,
196 + list) and len(image_uuids) == media_count
197 +
198 + assert image_uuids == expected_uuids
199 + else:
200 + assert mm_uuids is None
201 +
202 +
203 ModalityType = Literal["image", "video", "audio"]
204 MultiModalDataCounts = Mapping[ModalityType, int]
205
@@ -201,7 +222,7 @@ def test_parse_chat_messages_single_image(
222 phi3v_tokenizer,
223 image_url,
224 ):
204 - conversation, mm_data = parse_chat_messages(
225 + conversation, mm_data, mm_uuids = parse_chat_messages(
226 [{
227 "role":
228 "user",
@@ -228,6 +249,260 @@ def test_parse_chat_messages_single_image(
249 "content": "<|image_1|>\nWhat's in the image?"
250 }]
251 _assert_mm_data_is_image_input(mm_data, 1)
252 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
253 +
254 +
255 +def test_parse_chat_messages_single_image_with_uuid(
256 + phi3v_model_config,
257 + phi3v_tokenizer,
258 + image_url,
259 +):
260 + image_uuid = str(hash(image_url))
261 + conversation, mm_data, mm_uuids = parse_chat_messages(
262 + [{
263 + "role":
264 + "user",
265 + "content": [
266 + {
267 + "type": "image_url",
268 + "image_url": {
269 + "url": image_url,
270 + },
271 + "uuid": image_uuid,
272 + },
273 + {
274 + "type": "text",
275 + "text": "What's in the image?"
276 + },
277 + ],
278 + }],
279 + phi3v_model_config,
280 + phi3v_tokenizer,
281 + content_format="string",
282 + )
283 +
284 + assert conversation == [{
285 + "role": "user",
286 + "content": "<|image_1|>\nWhat's in the image?"
287 + }]
288 + _assert_mm_data_is_image_input(mm_data, 1)
289 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
290 +
291 +
292 +def test_parse_chat_messages_single_image_with_bad_uuid_format(
293 + phi3v_model_config,
294 + phi3v_tokenizer,
295 + image_url,
296 +):
297 + image_uuid = str(hash(image_url))
298 + conversation, mm_data, mm_uuids = parse_chat_messages(
299 + [{
300 + "role":
301 + "user",
302 + "content": [
303 + {
304 + "type": "image_url",
305 + "image_url": {
306 + "url": image_url,
307 + "uuid": image_uuid,
308 + },
309 + "bad_uuid_key": image_uuid,
310 + },
311 + {
312 + "type": "text",
313 + "text": "What's in the image?"
314 + },
315 + ],
316 + }],
317 + phi3v_model_config,
318 + phi3v_tokenizer,
319 + content_format="string",
320 + )
321 +
322 + assert conversation == [{
323 + "role": "user",
324 + "content": "<|image_1|>\nWhat's in the image?"
325 + }]
326 + _assert_mm_data_is_image_input(mm_data, 1)
327 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
328 +
329 +
330 +def test_parse_chat_messages_multiple_images_with_uuids(
331 + phi3v_model_config,
332 + phi3v_tokenizer,
333 + image_url,
334 +):
335 + image_uuid1 = "my_uuid_1"
336 + image_uuid2 = "my_uuid_2"
337 +
338 + conversation, mm_data, mm_uuids = parse_chat_messages(
339 + [{
340 + "role":
341 + "user",
342 + "content": [
343 + {
344 + "type": "image_url",
345 + "image_url": {
346 + "url": image_url,
347 + },
348 + "uuid": image_uuid1,
349 + },
350 + {
351 + "type": "image_url",
352 + "image_url": {
353 + "url": image_url,
354 + },
355 + "uuid": image_uuid2,
356 + },
357 + {
358 + "type": "text",
359 + "text": "What's in the image?"
360 + },
361 + ],
362 + }],
363 + phi3v_model_config,
364 + phi3v_tokenizer,
365 + content_format="string",
366 + )
367 +
368 + assert conversation == [{
369 + "role":
370 + "user",
371 + "content":
372 + "<|image_1|>\n<|image_2|>\nWhat's in the image?",
373 + }]
374 + _assert_mm_data_is_image_input(mm_data, 2)
375 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
376 +
377 +
378 +@pytest.mark.asyncio
379 +async def test_parse_chat_messages_single_image_with_uuid_async(
380 + phi3v_model_config,
381 + phi3v_tokenizer,
382 + image_url,
383 +):
384 + image_uuid = str(hash(image_url))
385 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
386 + [{
387 + "role":
388 + "user",
389 + "content": [
390 + {
391 + "type": "image_url",
392 + "image_url": {
393 + "url": image_url
394 + },
395 + "uuid": image_uuid,
396 + },
397 + {
398 + "type": "text",
399 + "text": "What's in the image?"
400 + },
401 + ],
402 + }],
403 + phi3v_model_config,
404 + phi3v_tokenizer,
405 + content_format="string",
406 + )
407 +
408 + assert conversation == [{
409 + "role": "user",
410 + "content": "<|image_1|>\nWhat's in the image?"
411 + }]
412 + _assert_mm_data_is_image_input(await mm_future, 1)
413 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid])
414 +
415 +
416 +@pytest.mark.asyncio
417 +async def test_parse_chat_messages_multiple_images_with_uuids_async(
418 + phi3v_model_config,
419 + phi3v_tokenizer,
420 + image_url,
421 +):
422 + image_uuid1 = "my_uuid_1"
423 + image_uuid2 = "my_uuid_2"
424 +
425 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
426 + [{
427 + "role":
428 + "user",
429 + "content": [
430 + {
431 + "type": "image_url",
432 + "image_url": {
433 + "url": image_url
434 + },
435 + "uuid": image_uuid1,
436 + },
437 + {
438 + "type": "image_pil",
439 + "image_pil": ImageAsset("cherry_blossom").pil_image,
440 + "uuid": image_uuid2,
441 + },
442 + {
443 + "type": "text",
444 + "text": "What's in these images?"
445 + },
446 + ],
447 + }],
448 + phi3v_model_config,
449 + phi3v_tokenizer,
450 + content_format="string",
451 + )
452 +
453 + assert conversation == [{
454 + "role":
455 + "user",
456 + "content":
457 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",
458 + }]
459 + _assert_mm_data_is_image_input(await mm_future, 2)
460 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2])
461 +
462 +
463 +@pytest.mark.asyncio
464 +async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
465 + phi3v_model_config,
466 + phi3v_tokenizer,
467 + image_url,
468 +):
469 + image_uuid2 = "my_uuid_2"
470 +
471 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
472 + [{
473 + "role":
474 + "user",
475 + "content": [
476 + {
477 + "type": "image_url",
478 + "image_url": {
479 + "url": image_url
480 + },
481 + },
482 + {
483 + "type": "image_pil",
484 + "image_pil": ImageAsset("cherry_blossom").pil_image,
485 + "uuid": image_uuid2,
486 + },
487 + {
488 + "type": "text",
489 + "text": "What's in these images?"
490 + },
491 + ],
492 + }],
493 + phi3v_model_config,
494 + phi3v_tokenizer,
495 + content_format="string",
496 + )
497 +
498 + assert conversation == [{
499 + "role":
500 + "user",
501 + "content":
502 + "<|image_1|>\n<|image_2|>\nWhat's in these images?",
503 + }]
504 + _assert_mm_data_is_image_input(await mm_future, 2)
505 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2])
506
507
508 def test_parse_chat_messages_empty_system(
@@ -235,7 +510,7 @@ def test_parse_chat_messages_empty_system(
510 mistral_tokenizer,
511 ):
512 # Test string format
238 - conversation, _ = parse_chat_messages(
513 + conversation, _, _ = parse_chat_messages(
514 [
515 {
516 "role": "system",
@@ -265,7 +540,7 @@ def test_parse_chat_messages_empty_system(
540 ]
541
542 # Test openai format
268 - conversation, _ = parse_chat_messages(
543 + conversation, _, _ = parse_chat_messages(
544 [
545 {
546 "role": "system",
@@ -307,7 +582,7 @@ async def test_parse_chat_messages_single_image_async(
582 phi3v_tokenizer,
583 image_url,
584 ):
310 - conversation, mm_future = parse_chat_messages_futures(
585 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
586 [{
587 "role":
588 "user",
@@ -334,6 +609,7 @@ async def test_parse_chat_messages_single_image_async(
609 "content": "<|image_1|>\nWhat's in the image?"
610 }]
611 _assert_mm_data_is_image_input(await mm_future, 1)
612 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
613
614
615 def test_parse_chat_messages_multiple_images(
@@ -341,7 +617,7 @@ def test_parse_chat_messages_multiple_images(
617 phi3v_tokenizer,
618 image_url,
619 ):
344 - conversation, mm_data = parse_chat_messages(
620 + conversation, mm_data, mm_uuids = parse_chat_messages(
621 [{
622 "role":
623 "user",
@@ -374,6 +650,7 @@ def test_parse_chat_messages_multiple_images(
650 "<|image_1|>\n<|image_2|>\nWhat's in these images?",
651 }]
652 _assert_mm_data_is_image_input(mm_data, 2)
653 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
654
655
656 @pytest.mark.asyncio
@@ -382,7 +659,7 @@ async def test_parse_chat_messages_multiple_images_async(
659 phi3v_tokenizer,
660 image_url,
661 ):
385 - conversation, mm_future = parse_chat_messages_futures(
662 + conversation, mm_future, mm_uuids = parse_chat_messages_futures(
663 [{
664 "role":
665 "user",
@@ -415,6 +692,7 @@ async def test_parse_chat_messages_multiple_images_async(
692 "<|image_1|>\n<|image_2|>\nWhat's in these images?",
693 }]
694 _assert_mm_data_is_image_input(await mm_future, 2)
695 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
696
697
698 def test_parse_chat_messages_placeholder_already_in_prompt(
@@ -422,7 +700,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
700 phi3v_tokenizer,
701 image_url,
702 ):
425 - conversation, mm_data = parse_chat_messages(
703 + conversation, mm_data, mm_uuids = parse_chat_messages(
704 [{
705 "role":
706 "user",
@@ -458,6 +736,7 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
736 "What's in <|image_1|> and how does it compare to <|image_2|>?",
737 }]
738 _assert_mm_data_is_image_input(mm_data, 2)
739 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
740
741
742 def test_parse_chat_messages_placeholder_one_already_in_prompt(
@@ -465,7 +744,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
744 phi3v_tokenizer,
745 image_url,
746 ):
468 - conversation, mm_data = parse_chat_messages(
747 + conversation, mm_data, mm_uuids = parse_chat_messages(
748 [{
749 "role":
750 "user",
@@ -503,6 +782,7 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
782 "other one?",
783 }]
784 _assert_mm_data_is_image_input(mm_data, 2)
785 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
786
787
788 def test_parse_chat_messages_multiple_images_across_messages(
@@ -510,7 +790,7 @@ def test_parse_chat_messages_multiple_images_across_messages(
790 phi3v_tokenizer,
791 image_url,
792 ):
513 - conversation, mm_data = parse_chat_messages(
793 + conversation, mm_data, mm_uuids = parse_chat_messages(
794 [
795 {
796 "role":
@@ -569,28 +849,99 @@ def test_parse_chat_messages_multiple_images_across_messages(
849 },
850 ]
851 _assert_mm_data_is_image_input(mm_data, 2)
852 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
853
854
574 -def test_parse_chat_messages_context_text_format(
855 +def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
856 phi3v_model_config,
857 phi3v_tokenizer,
858 + image_url,
859 ):
578 - conversation, mm_data = parse_chat_messages(
860 + image_uuid = str(hash(image_url))
861 + conversation, mm_data, mm_uuids = parse_chat_messages(
862 [
863 {
581 - "role": "user",
582 - "content": [{
583 - "type": "text",
584 - "text": "What's in this text?"
585 - }],
864 + "role":
865 + "user",
866 + "content": [
867 + {
868 + "type": "image_url",
869 + "image_url": {
870 + "url": image_url
871 + },
872 + "uuid": image_uuid,
873 + },
874 + {
875 + "type": "text",
876 + "text": "What's in this image?"
877 + },
878 + ],
879 },
880 {
881 "role": "assistant",
882 "content": "Some stuff."
883 },
884 {
592 - "role": "user",
593 - "content": "What about this one?"
885 + "role":
886 + "user",
887 + "content": [
888 + {
889 + "type": "image_url",
890 + "image_url": {
891 + "url": image_url
892 + },
893 + "uuid": image_uuid,
894 + },
895 + {
896 + "type": "text",
897 + "text": "What about this one?"
898 + },
899 + ],
900 + },
901 + ],
902 + phi3v_model_config,
903 + phi3v_tokenizer,
904 + content_format="string",
905 + )
906 +
907 + assert conversation == [
908 + {
909 + "role": "user",
910 + "content": "<|image_1|>\nWhat's in this image?"
911 + },
912 + {
913 + "role": "assistant",
914 + "content": "Some stuff."
915 + },
916 + {
917 + "role": "user",
918 + "content": "<|image_2|>\nWhat about this one?"
919 + },
920 + ]
921 + _assert_mm_data_is_image_input(mm_data, 2)
922 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
923 +
924 +
925 +def test_parse_chat_messages_context_text_format(
926 + phi3v_model_config,
927 + phi3v_tokenizer,
928 +):
929 + conversation, mm_data, mm_uuids = parse_chat_messages(
930 + [
931 + {
932 + "role": "user",
933 + "content": [{
934 + "type": "text",
935 + "text": "What's in this text?"
936 + }],
937 + },
938 + {
939 + "role": "assistant",
940 + "content": "Some stuff."
941 + },
942 + {
943 + "role": "user",
944 + "content": "What about this one?"
945 },
946 ],
947 phi3v_model_config,
@@ -621,6 +972,8 @@ def test_parse_chat_messages_context_text_format(
972 }],
973 },
974 ]
975 + assert mm_data is None
976 + assert mm_uuids is None
977
978
979 def test_parse_chat_messages_rejects_too_many_images_in_one_message(
@@ -736,7 +1089,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
1089 phi3v_tokenizer,
1090 image_url,
1091 ):
739 - conversation, mm_data = parse_chat_messages(
1092 + conversation, mm_data, mm_uuids = parse_chat_messages(
1093 [{
1094 "role":
1095 "user",
@@ -762,6 +1115,7 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
1115 "<|image_1|>\n<|image_2|>\nWhat's in these images?",
1116 }]
1117 _assert_mm_data_is_image_input(mm_data, 2)
1118 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1119
1120
1121 def test_parse_chat_messages_multiple_images_interleave(
@@ -769,7 +1123,7 @@ def test_parse_chat_messages_multiple_images_interleave(
1123 phi3v_tokenizer,
1124 image_url,
1125 ):
772 - conversation, mm_data = parse_chat_messages(
1126 + conversation, mm_data, mm_uuids = parse_chat_messages(
1127 [{
1128 "role":
1129 "user",
@@ -813,6 +1167,7 @@ def test_parse_chat_messages_multiple_images_interleave(
1167 "Do they have differences?",
1168 }]
1169 _assert_mm_data_is_image_input(mm_data, 2)
1170 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1171
1172
1173 @pytest.mark.asyncio
@@ -821,7 +1176,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1176 phi3v_tokenizer,
1177 image_url,
1178 ):
824 - conversation, mm_data = parse_chat_messages_futures(
1179 + conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1180 [{
1181 "role":
1182 "user",
@@ -865,6 +1220,63 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1220 "Do they have differences?",
1221 }]
1222 _assert_mm_data_is_image_input(await mm_data, 2)
1223 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1224 +
1225 +
1226 +@pytest.mark.asyncio
1227 +async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1228 + phi3v_model_config_mm_interleaved,
1229 + phi3v_tokenizer,
1230 + image_url,
1231 +):
1232 + image_uuid = str(hash(image_url))
1233 + conversation, mm_data, mm_uuids = parse_chat_messages_futures(
1234 + [{
1235 + "role":
1236 + "user",
1237 + "content": [
1238 + {
1239 + "type": "text",
1240 + "text": "I need you to compare this image",
1241 + },
1242 + {
1243 + "type": "image_url",
1244 + "image_url": {
1245 + "url": image_url
1246 + },
1247 + "uuid": image_uuid,
1248 + },
1249 + {
1250 + "type": "text",
1251 + "text": "and this one"
1252 + },
1253 + {
1254 + "type": "image_url",
1255 + "image_url": {
1256 + "url": image_url
1257 + },
1258 + "uuid": image_uuid,
1259 + },
1260 + {
1261 + "type": "text",
1262 + "text": "Do they have differences?"
1263 + },
1264 + ],
1265 + }],
1266 + phi3v_model_config_mm_interleaved,
1267 + phi3v_tokenizer,
1268 + content_format="string",
1269 + )
1270 +
1271 + assert conversation == [{
1272 + "role":
1273 + "user",
1274 + "content":
1275 + "I need you to compare this image\n<|image_1|>\nand this one\n<|image_2|>\n" # noqa: E501
1276 + "Do they have differences?",
1277 + }]
1278 + _assert_mm_data_is_image_input(await mm_data, 2)
1279 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1280
1281
1282 def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
@@ -872,7 +1284,7 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1284 phi3v_tokenizer,
1285 image_url,
1286 ):
875 - conversation, mm_data = parse_chat_messages(
1287 + conversation, mm_data, mm_uuids = parse_chat_messages(
1288 [
1289 {
1290 "role":
@@ -935,6 +1347,81 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1347 },
1348 ]
1349 _assert_mm_data_is_image_input(mm_data, 2)
1350 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1351 +
1352 +
1353 +def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave( # noqa: E501
1354 + phi3v_model_config_mm_interleaved,
1355 + phi3v_tokenizer,
1356 + image_url,
1357 +):
1358 + image_uuid = str(hash(image_url))
1359 + conversation, mm_data, mm_uuids = parse_chat_messages(
1360 + [
1361 + {
1362 + "role":
1363 + "user",
1364 + "content": [
1365 + {
1366 + "type": "text",
1367 + "text": "What's on this image?"
1368 + },
1369 + {
1370 + "type": "image_url",
1371 + "image_url": {
1372 + "url": image_url
1373 + },
1374 + "uuid": image_uuid,
1375 + },
1376 + {
1377 + "type": "text",
1378 + "text": "Be accurate."
1379 + },
1380 + ],
1381 + },
1382 + {
1383 + "role": "assistant",
1384 + "content": "Some stuff."
1385 + },
1386 + {
1387 + "role":
1388 + "user",
1389 + "content": [
1390 + {
1391 + "type": "text",
1392 + "text": "What's on this image?"
1393 + },
1394 + {
1395 + "type": "image_url",
1396 + "image_url": {
1397 + "url": image_url
1398 + },
1399 + "uuid": image_uuid,
1400 + },
1401 + ],
1402 + },
1403 + ],
1404 + phi3v_model_config_mm_interleaved,
1405 + phi3v_tokenizer,
1406 + content_format="string",
1407 + )
1408 +
1409 + assert conversation == [
1410 + {
1411 + "role": "user",
1412 + "content": "What's on this image?\n<|image_1|>\nBe accurate.",
1413 + },
1414 + {
1415 + "role": "assistant",
1416 + "content": "Some stuff."
1417 + },
1418 + {
1419 + "role": "user",
1420 + "content": "What's on this image?\n<|image_2|>"
1421 + },
1422 + ]
1423 + _assert_mm_data_is_image_input(mm_data, 2)
1424 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid])
1425
1426
1427 def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
@@ -944,7 +1431,7 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1431 video_url,
1432 audio_url,
1433 ):
947 - conversation, mm_data = parse_chat_messages(
1434 + conversation, mm_data, mm_uuids = parse_chat_messages(
1435 [
1436 {
1437 "role":
@@ -1030,6 +1517,229 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1517 ]
1518
1519 _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1520 + _assert_mm_uuids(mm_uuids,
1521 + 2,
1522 + modality="image",
1523 + expected_uuids=[None, None])
1524 + _assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
1525 + _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
1526 +
1527 +
1528 +def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave( # noqa: E501
1529 + qwen25omni_model_config_mm_interleaved,
1530 + qwen25omni_tokenizer,
1531 + image_url,
1532 + video_url,
1533 + audio_url,
1534 +):
1535 + conversation, mm_data, mm_uuids = parse_chat_messages(
1536 + [
1537 + {
1538 + "role":
1539 + "user",
1540 + "content": [
1541 + {
1542 + "type": "text",
1543 + "text": "What's on this image?"
1544 + },
1545 + {
1546 + "type": "image_url",
1547 + "image_url": {
1548 + "url": image_url
1549 + },
1550 + "uuid": "image_123",
1551 + },
1552 + {
1553 + "type": "text",
1554 + "text": "Now listen to this audio"
1555 + },
1556 + {
1557 + "type": "audio_url",
1558 + "audio_url": {
1559 + "url": audio_url
1560 + },
1561 + "uuid": "audio_123",
1562 + },
1563 + ],
1564 + },
1565 + {
1566 + "role": "assistant",
1567 + "content": "Some stuff."
1568 + },
1569 + {
1570 + "role":
1571 + "user",
1572 + "content": [
1573 + {
1574 + "type": "text",
1575 + "text": "What's on this image?"
1576 + },
1577 + {
1578 + "type": "image_url",
1579 + "image_url": {
1580 + "url": image_url
1581 + },
1582 + "uuid": "image_123",
1583 + },
1584 + {
1585 + "type": "text",
1586 + "text": "And what's in the video?"
1587 + },
1588 + {
1589 + "type": "video_url",
1590 + "video_url": {
1591 + "url": video_url
1592 + },
1593 + "uuid": "video_123",
1594 + },
1595 + ],
1596 + },
1597 + ],
1598 + qwen25omni_model_config_mm_interleaved,
1599 + qwen25omni_tokenizer,
1600 + content_format="string",
1601 + )
1602 +
1603 + assert conversation == [
1604 + {
1605 + "role":
1606 + "user",
1607 + "content":
1608 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1609 + "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1610 + },
1611 + {
1612 + "role": "assistant",
1613 + "content": "Some stuff."
1614 + },
1615 + {
1616 + "role":
1617 + "user",
1618 + "content":
1619 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1620 + "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1621 + },
1622 + ]
1623 +
1624 + _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1625 + _assert_mm_uuids(mm_uuids,
1626 + 2,
1627 + modality="image",
1628 + expected_uuids=["image_123", "image_123"])
1629 + _assert_mm_uuids(mm_uuids,
1630 + 1,
1631 + modality="video",
1632 + expected_uuids=["video_123"])
1633 + _assert_mm_uuids(mm_uuids,
1634 + 1,
1635 + modality="audio",
1636 + expected_uuids=["audio_123"])
1637 +
1638 +
1639 +def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501
1640 + qwen25omni_model_config_mm_interleaved,
1641 + qwen25omni_tokenizer,
1642 + image_url,
1643 + video_url,
1644 + audio_url,
1645 +):
1646 + conversation, mm_data, mm_uuids = parse_chat_messages(
1647 + [
1648 + {
1649 + "role":
1650 + "user",
1651 + "content": [
1652 + {
1653 + "type": "text",
1654 + "text": "What's on this image?"
1655 + },
1656 + {
1657 + "type": "image_url",
1658 + "image_url": {
1659 + "url": image_url
1660 + },
1661 + "uuid": "image_123",
1662 + },
1663 + {
1664 + "type": "text",
1665 + "text": "Now listen to this audio"
1666 + },
1667 + {
1668 + "type": "audio_url",
1669 + "audio_url": {
1670 + "url": audio_url
1671 + }
1672 + },
1673 + ],
1674 + },
1675 + {
1676 + "role": "assistant",
1677 + "content": "Some stuff."
1678 + },
1679 + {
1680 + "role":
1681 + "user",
1682 + "content": [
1683 + {
1684 + "type": "text",
1685 + "text": "What's on this image?"
1686 + },
1687 + {
1688 + "type": "image_url",
1689 + "image_url": {
1690 + "url": image_url
1691 + }
1692 + },
1693 + {
1694 + "type": "text",
1695 + "text": "And what's in the video?"
1696 + },
1697 + {
1698 + "type": "video_url",
1699 + "video_url": {
1700 + "url": video_url
1701 + },
1702 + "uuid": "video_123",
1703 + },
1704 + ],
1705 + },
1706 + ],
1707 + qwen25omni_model_config_mm_interleaved,
1708 + qwen25omni_tokenizer,
1709 + content_format="string",
1710 + )
1711 +
1712 + assert conversation == [
1713 + {
1714 + "role":
1715 + "user",
1716 + "content":
1717 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1718 + "Now listen to this audio\nAudio 1: <|audio_bos|><|AUDIO|><|audio_eos|>", # noqa: E501
1719 + },
1720 + {
1721 + "role": "assistant",
1722 + "content": "Some stuff."
1723 + },
1724 + {
1725 + "role":
1726 + "user",
1727 + "content":
1728 + "What's on this image?\n<|vision_start|><|IMAGE|><|vision_end|>\n"
1729 + "And what's in the video?\n<|vision_start|><|VIDEO|><|vision_end|>",
1730 + },
1731 + ]
1732 +
1733 + _assert_mm_data_inputs(mm_data, {"image": 2, "video": 1, "audio": 1})
1734 + _assert_mm_uuids(mm_uuids,
1735 + 2,
1736 + modality="image",
1737 + expected_uuids=["image_123", None])
1738 + _assert_mm_uuids(mm_uuids,
1739 + 1,
1740 + modality="video",
1741 + expected_uuids=["video_123"])
1742 + _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
1743
1744
1745 def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
@@ -1081,7 +1791,7 @@ def test_mllama_single_image(
1791 image_url,
1792 ):
1793 """Ensures that a single image is parsed correctly mllama."""
1084 - conversation, mm_data = parse_chat_messages(
1794 + conversation, mm_data, mm_uuids = parse_chat_messages(
1795 [{
1796 "role":
1797 "user",
@@ -1100,6 +1810,7 @@ def test_mllama_single_image(
1810 content_format="openai",
1811 )
1812 _assert_mm_data_is_image_input(mm_data, 1)
1813 + _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
1814 assert conversation == [{
1815 "role":
1816 "user",
@@ -1121,7 +1832,7 @@ def test_mllama_interleaved_images(
1832 image_url,
1833 ):
1834 """Ensures that multiple image are parsed as interleaved dicts."""
1124 - conversation, mm_data = parse_chat_messages(
1835 + conversation, mm_data, mm_uuids = parse_chat_messages(
1836 [{
1837 "role":
1838 "user",
@@ -1147,6 +1858,7 @@ def test_mllama_interleaved_images(
1858 content_format="openai",
1859 )
1860 _assert_mm_data_is_image_input(mm_data, 2)
1861 + _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
1862 assert conversation == [{
1863 "role":
1864 "user",
@@ -1227,7 +1939,7 @@ def test_multimodal_image_parsing_matches_hf(model, image_url):
1939
1940 # Now parse with vLLMs chat utils & apply the template
1941 vllm_conversation = get_conversation(is_hf=False)
1230 - conversation, _ = parse_chat_messages(
1942 + conversation, _, _ = parse_chat_messages(
1943 vllm_conversation,
1944 model_config,
1945 tokenizer_group,
@@ -1518,7 +2230,7 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config,
2230 }],
2231 }]
2232
1521 - conversation_with_thinking, _ = parse_chat_messages(
2233 + conversation_with_thinking, _, _ = parse_chat_messages(
2234 messages,
2235 mistral_model_config,
2236 mistral_tokenizer,
vllm/entrypoints/chat_utils.py
+138 -43
@@ -41,7 +41,8 @@ from typing_extensions import Required, TypeAlias, TypedDict
41 from vllm.config import ModelConfig
42 from vllm.logger import init_logger
43 from vllm.model_executor.models import SupportsMultiModal
44 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
44 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
45 + MultiModalUUIDDict)
46 from vllm.multimodal.utils import MediaConnector
47 # yapf: disable
48 from vllm.transformers_utils.chat_templates import (
@@ -72,6 +73,11 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
73
74 type: Required[Literal["audio_url"]]
75 """The type of the content part."""
76 + uuid: Optional[str]
77 + """
78 + User-provided UUID of a media. User must guarantee that it is properly
79 + generated and unique for different medias.
80 + """
81
82
83 class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
@@ -83,6 +89,11 @@ class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
89 """
90 type: Required[Literal["image_embeds"]]
91 """The type of the content part."""
92 + uuid: Optional[str]
93 + """
94 + User-provided UUID of a media. User must guarantee that it is properly
95 + generated and unique for different medias.
96 + """
97
98
99 class VideoURL(TypedDict, total=False):
@@ -97,6 +108,11 @@ class ChatCompletionContentPartVideoParam(TypedDict, total=False):
108
109 type: Required[Literal["video_url"]]
110 """The type of the content part."""
111 + uuid: Optional[str]
112 + """
113 + User-provided UUID of a media. User must guarantee that it is properly
114 + generated and unique for different medias.
115 + """
116
117
118 class PILImage(BaseModel):
@@ -118,6 +134,11 @@ class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
134 """
135
136 image_pil: Required[PILImage]
137 + uuid: Optional[str]
138 + """
139 + User-provided UUID of a media. User must guarantee that it is properly
140 + generated and unique for different medias.
141 + """
142
143
144 class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
@@ -131,6 +152,11 @@ class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
152 """
153
154 image_url: Required[str]
155 + uuid: Optional[str]
156 + """
157 + User-provided UUID of a media. User must guarantee that it is properly
158 + generated and unique for different medias.
159 + """
160
161
162 class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
@@ -155,6 +181,11 @@ class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
181 """
182
183 video_url: Required[str]
184 + uuid: Optional[str]
185 + """
186 + User-provided UUID of a media. User must guarantee that it is properly
187 + generated and unique for different medias.
188 + """
189
190
191 class CustomThinkCompletionContentParam(TypedDict, total=False):
@@ -567,6 +598,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
598 self._tokenizer = tokenizer
599
600 self._items_by_modality = defaultdict[str, list[_T]](list)
601 + self._uuids_by_modality = defaultdict[str, list[Optional[str]]](list)
602
603 @property
604 def model_config(self) -> ModelConfig:
@@ -591,10 +623,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
623 def mm_processor(self):
624 return self.mm_registry.create_processor(self.model_config)
625
594 - def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
626 + def add(
627 + self, modality: ModalityStr, item: _T, uuid: Optional[str] = None
628 + ) -> Optional[str]:
629 """
630 Add a multi-modal item to the current prompt and returns the
631 placeholder string to use, if any.
632 +
633 + An optional uuid can be added which serves as a unique identifier of the
634 + media.
635 """
636 input_modality = modality.replace("_embeds", "")
637 num_items = len(self._items_by_modality[modality]) + 1
@@ -602,9 +639,35 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
639 self.mm_processor.validate_num_items(input_modality, num_items)
640
641 self._items_by_modality[modality].append(item)
642 + self._uuids_by_modality[modality].append(uuid)
643
644 return self.model_cls.get_placeholder_str(modality, num_items)
645
646 + def all_mm_uuids(self) -> Optional[MultiModalUUIDDict]:
647 + if not self._items_by_modality:
648 + return None
649 + mm_uuids = {}
650 + uuids_by_modality = dict(self._uuids_by_modality)
651 + if "image" in uuids_by_modality and "image_embeds" in uuids_by_modality:
652 + raise ValueError(
653 + "Mixing raw image and embedding inputs is not allowed"
654 + )
655 +
656 + if "image_embeds" in uuids_by_modality:
657 + image_embeds_uuids = uuids_by_modality["image_embeds"]
658 + if len(image_embeds_uuids) > 1:
659 + raise ValueError(
660 + "Only one message can have {'type': 'image_embeds'}"
661 + )
662 + mm_uuids["image"] = uuids_by_modality["image_embeds"]
663 + if "image" in uuids_by_modality:
664 + mm_uuids["image"] = uuids_by_modality["image"] # UUIDs of images
665 + if "audio" in uuids_by_modality:
666 + mm_uuids["audio"] = uuids_by_modality["audio"] # UUIDs of audios
667 + if "video" in uuids_by_modality:
668 + mm_uuids["video"] = uuids_by_modality["video"] # UUIDs of videos
669 + return mm_uuids
670 +
671 @abstractmethod
672 def create_parser(self) -> "BaseMultiModalContentParser":
673 raise NotImplementedError
@@ -697,29 +760,35 @@ class BaseMultiModalContentParser(ABC):
760 return dict(self._placeholder_storage)
761
762 @abstractmethod
700 - def parse_image(self, image_url: str) -> None:
763 + def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
764 raise NotImplementedError
765
766 @abstractmethod
767 def parse_image_embeds(
705 - self, image_embeds: Union[str, dict[str, str]]
768 + self,
769 + image_embeds: Union[str, dict[str, str]],
770 + uuid: Optional[str] = None,
771 ) -> None:
772 raise NotImplementedError
773
774 @abstractmethod
710 - def parse_image_pil(self, image_pil: Image.Image) -> None:
775 + def parse_image_pil(
776 + self, image_pil: Image.Image, uuid: Optional[str] = None
777 + ) -> None:
778 raise NotImplementedError
779
780 @abstractmethod
714 - def parse_audio(self, audio_url: str) -> None:
781 + def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
782 raise NotImplementedError
783
784 @abstractmethod
718 - def parse_input_audio(self, input_audio: InputAudio) -> None:
785 + def parse_input_audio(
786 + self, input_audio: InputAudio, uuid: Optional[str] = None
787 + ) -> None:
788 raise NotImplementedError
789
790 @abstractmethod
722 - def parse_video(self, video_url: str) -> None:
791 + def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
792 raise NotImplementedError
793
794
@@ -734,49 +803,55 @@ class MultiModalContentParser(BaseMultiModalContentParser):
803 allowed_local_media_path=tracker.allowed_local_media_path,
804 )
805
737 - def parse_image(self, image_url: str) -> None:
806 + def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
807 image = self._connector.fetch_image(image_url)
808
740 - placeholder = self._tracker.add("image", image)
809 + placeholder = self._tracker.add("image", image, uuid)
810 self._add_placeholder("image", placeholder)
811
812 def parse_image_embeds(
744 - self, image_embeds: Union[str, dict[str, str]]
813 + self,
814 + image_embeds: Union[str, dict[str, str]],
815 + uuid: Optional[str] = None,
816 ) -> None:
817 if isinstance(image_embeds, dict):
818 embeds = {
819 k: self._connector.fetch_image_embedding(v)
820 for k, v in image_embeds.items()
821 }
751 - placeholder = self._tracker.add("image_embeds", embeds)
822 + placeholder = self._tracker.add("image_embeds", embeds, uuid)
823
824 if isinstance(image_embeds, str):
825 embedding = self._connector.fetch_image_embedding(image_embeds)
755 - placeholder = self._tracker.add("image_embeds", embedding)
826 + placeholder = self._tracker.add("image_embeds", embedding, uuid)
827
828 self._add_placeholder("image", placeholder)
829
759 - def parse_image_pil(self, image_pil: Image.Image) -> None:
760 - placeholder = self._tracker.add("image", image_pil)
830 + def parse_image_pil(
831 + self, image_pil: Image.Image, uuid: Optional[str] = None
832 + ) -> None:
833 + placeholder = self._tracker.add("image", image_pil, uuid)
834 self._add_placeholder("image", placeholder)
835
763 - def parse_audio(self, audio_url: str) -> None:
836 + def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
837 audio = self._connector.fetch_audio(audio_url)
838
766 - placeholder = self._tracker.add("audio", audio)
839 + placeholder = self._tracker.add("audio", audio, uuid)
840 self._add_placeholder("audio", placeholder)
841
769 - def parse_input_audio(self, input_audio: InputAudio) -> None:
842 + def parse_input_audio(
843 + self, input_audio: InputAudio, uuid: Optional[str] = None
844 + ) -> None:
845 audio_data = input_audio.get("data", "")
846 audio_format = input_audio.get("format", "")
847 audio_url = f"data:audio/{audio_format};base64,{audio_data}"
848
774 - return self.parse_audio(audio_url)
849 + return self.parse_audio(audio_url, uuid)
850
776 - def parse_video(self, video_url: str) -> None:
851 + def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
852 video = self._connector.fetch_video(video_url=video_url)
853
779 - placeholder = self._tracker.add("video", video)
854 + placeholder = self._tracker.add("video", video, uuid)
855 self._add_placeholder("video", placeholder)
856
857
@@ -790,14 +865,16 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
865 allowed_local_media_path=tracker.allowed_local_media_path,
866 )
867
793 - def parse_image(self, image_url: str) -> None:
868 + def parse_image(self, image_url: str, uuid: Optional[str] = None) -> None:
869 image_coro = self._connector.fetch_image_async(image_url)
870
796 - placeholder = self._tracker.add("image", image_coro)
871 + placeholder = self._tracker.add("image", image_coro, uuid)
872 self._add_placeholder("image", placeholder)
873
874 def parse_image_embeds(
800 - self, image_embeds: Union[str, dict[str, str]]
875 + self,
876 + image_embeds: Union[str, dict[str, str]],
877 + uuid: Optional[str] = None,
878 ) -> None:
879 future: asyncio.Future[Union[str, dict[str, str]]] = asyncio.Future()
880
@@ -812,33 +889,37 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
889 embedding = self._connector.fetch_image_embedding(image_embeds)
890 future.set_result(embedding)
891
815 - placeholder = self._tracker.add("image_embeds", future)
892 + placeholder = self._tracker.add("image_embeds", future, uuid)
893 self._add_placeholder("image", placeholder)
894
818 - def parse_image_pil(self, image_pil: Image.Image) -> None:
895 + def parse_image_pil(
896 + self, image_pil: Image.Image, uuid: Optional[str] = None
897 + ) -> None:
898 future: asyncio.Future[Image.Image] = asyncio.Future()
899 future.set_result(image_pil)
900
822 - placeholder = self._tracker.add("image", future)
901 + placeholder = self._tracker.add("image", future, uuid)
902 self._add_placeholder("image", placeholder)
903
825 - def parse_audio(self, audio_url: str) -> None:
904 + def parse_audio(self, audio_url: str, uuid: Optional[str] = None) -> None:
905 audio_coro = self._connector.fetch_audio_async(audio_url)
906
828 - placeholder = self._tracker.add("audio", audio_coro)
907 + placeholder = self._tracker.add("audio", audio_coro, uuid)
908 self._add_placeholder("audio", placeholder)
909
831 - def parse_input_audio(self, input_audio: InputAudio) -> None:
910 + def parse_input_audio(
911 + self, input_audio: InputAudio, uuid: Optional[str] = None
912 + ) -> None:
913 audio_data = input_audio.get("data", "")
914 audio_format = input_audio.get("format", "")
915 audio_url = f"data:audio/{audio_format};base64,{audio_data}"
916
836 - return self.parse_audio(audio_url)
917 + return self.parse_audio(audio_url, uuid)
918
838 - def parse_video(self, video_url: str) -> None:
919 + def parse_video(self, video_url: str, uuid: Optional[str] = None) -> None:
920 video = self._connector.fetch_video_async(video_url=video_url)
921
841 - placeholder = self._tracker.add("video", video)
922 + placeholder = self._tracker.add("video", video, uuid)
923 self._add_placeholder("video", placeholder)
924
925
@@ -1177,30 +1258,36 @@ def _parse_chat_message_content_part(
1258 else:
1259 return str_content
1260
1261 + # For media items, if a user has provided one, use it. Otherwise, insert
1262 + # a placeholder empty uuid.
1263 + uuid = part.get("uuid", None)
1264 + if uuid is not None:
1265 + uuid = str(uuid)
1266 +
1267 modality = None
1268 if part_type == "image_pil":
1269 image_content = cast(Image.Image, content)
1183 - mm_parser.parse_image_pil(image_content)
1270 + mm_parser.parse_image_pil(image_content, uuid)
1271 modality = "image"
1272 elif part_type in ("image_url", "input_image"):
1273 str_content = cast(str, content)
1187 - mm_parser.parse_image(str_content)
1274 + mm_parser.parse_image(str_content, uuid)
1275 modality = "image"
1276 elif part_type == "image_embeds":
1277 content = cast(Union[str, dict[str, str]], content)
1191 - mm_parser.parse_image_embeds(content)
1278 + mm_parser.parse_image_embeds(content, uuid)
1279 modality = "image"
1280 elif part_type == "audio_url":
1281 str_content = cast(str, content)
1195 - mm_parser.parse_audio(str_content)
1282 + mm_parser.parse_audio(str_content, uuid)
1283 modality = "audio"
1284 elif part_type == "input_audio":
1285 dict_content = cast(InputAudio, content)
1199 - mm_parser.parse_input_audio(dict_content)
1286 + mm_parser.parse_input_audio(dict_content, uuid)
1287 modality = "audio"
1288 elif part_type == "video_url":
1289 str_content = cast(str, content)
1203 - mm_parser.parse_video(str_content)
1290 + mm_parser.parse_video(str_content, uuid)
1291 modality = "video"
1292 else:
1293 raise NotImplementedError(f"Unknown part type: {part_type}")
@@ -1288,7 +1375,11 @@ def parse_chat_messages(
1375 model_config: ModelConfig,
1376 tokenizer: AnyTokenizer,
1377 content_format: _ChatTemplateContentFormat,
1291 -) -> tuple[list[ConversationMessage], Optional[MultiModalDataDict]]:
1378 +) -> tuple[
1379 + list[ConversationMessage],
1380 + Optional[MultiModalDataDict],
1381 + Optional[MultiModalUUIDDict],
1382 +]:
1383 conversation: list[ConversationMessage] = []
1384 mm_tracker = MultiModalItemTracker(model_config, tokenizer)
1385
@@ -1308,7 +1399,7 @@ def parse_chat_messages(
1399
1400 _postprocess_messages(conversation)
1401
1311 - return conversation, mm_tracker.all_mm_data()
1402 + return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1403
1404
1405 def parse_chat_messages_futures(
@@ -1316,7 +1407,11 @@ def parse_chat_messages_futures(
1407 model_config: ModelConfig,
1408 tokenizer: AnyTokenizer,
1409 content_format: _ChatTemplateContentFormat,
1319 -) -> tuple[list[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
1410 +) -> tuple[
1411 + list[ConversationMessage],
1412 + Awaitable[Optional[MultiModalDataDict]],
1413 + Optional[MultiModalUUIDDict],
1414 +]:
1415 conversation: list[ConversationMessage] = []
1416 mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
1417
@@ -1336,7 +1431,7 @@ def parse_chat_messages_futures(
1431
1432 _postprocess_messages(conversation)
1433
1339 - return conversation, mm_tracker.all_mm_data()
1434 + return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1435
1436
1437 def apply_hf_chat_template(
vllm/entrypoints/llm.py
+4 -1
@@ -796,7 +796,7 @@ class LLM:
796 # NOTE: _parse_chat_message_content_parts() currently doesn't
797 # handle mm_processor_kwargs, since there is no implementation in
798 # the chat message parsing for it.
799 - conversation, mm_data = parse_chat_messages(
799 + conversation, mm_data, mm_uuids = parse_chat_messages(
800 msgs,
801 model_config,
802 tokenizer,
@@ -826,6 +826,9 @@ class LLM:
826 if mm_data is not None:
827 prompt["multi_modal_data"] = mm_data
828
829 + if mm_uuids is not None:
830 + prompt["multi_modal_uuids"] = mm_uuids
831 +
832 if mm_processor_kwargs is not None:
833 prompt["mm_processor_kwargs"] = mm_processor_kwargs
834
vllm/entrypoints/openai/serving_engine.py
+5 -1
@@ -929,7 +929,7 @@ class OpenAIServing:
929 tokenizer,
930 model_config=model_config,
931 )
932 - conversation, mm_data_future = parse_chat_messages_futures(
932 + conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
933 messages,
934 model_config,
935 tokenizer,
@@ -1006,6 +1006,10 @@ class OpenAIServing:
1006 prompt_token_ids=prompt_inputs["prompt_token_ids"])
1007 if mm_data is not None:
1008 engine_prompt["multi_modal_data"] = mm_data
1009 +
1010 + if mm_uuids is not None:
1011 + engine_prompt["multi_modal_uuids"] = mm_uuids
1012 +
1013 if request.mm_processor_kwargs is not None:
1014 engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
1015
vllm/inputs/preprocess.py
+34 -2
@@ -276,13 +276,23 @@ class InputPreprocessor:
276 if mm_processor_kwargs is None:
277 mm_processor_kwargs = {}
278
279 - return mm_processor.apply(
279 + mm_input = mm_processor.apply(
280 prompt,
281 mm_data,
282 hf_processor_mm_kwargs=mm_processor_kwargs,
283 tokenization_kwargs=tokenization_kwargs,
284 mm_hash_overrides=mm_hash_overrides,
285 )
286 + mm_hashes = mm_input["mm_hashes"]
287 +
288 + # Validate that all mm items have a string as their hash
289 + if not contains_only_strings(mm_hashes):
290 + raise ValueError(
291 + f"mm_hashes must contain only strings, got: {mm_hashes}. "
292 + "This is likely due to an incorrect custom implementation of "
293 + "MultiModalProcessor.apply method.")
294 +
295 + return mm_input
296
297 async def _process_multimodal_async(
298 self,
@@ -310,13 +320,23 @@ class InputPreprocessor:
320 if mm_processor_kwargs is None:
321 mm_processor_kwargs = {}
322
313 - return mm_processor.apply(
323 + mm_input = mm_processor.apply(
324 prompt,
325 mm_data,
326 hf_processor_mm_kwargs=mm_processor_kwargs,
327 tokenization_kwargs=tokenization_kwargs,
328 mm_hash_overrides=mm_hash_overrides,
329 )
330 + mm_hashes = mm_input["mm_hashes"]
331 +
332 + # Validate that all mm items have a string as their hash
333 + if not contains_only_strings(mm_hashes):
334 + raise ValueError(
335 + f"mm_hashes must contain only strings, got: {mm_hashes}. "
336 + "This is likely due to an incorrect custom implementation of "
337 + "MultiModalProcessor.apply method.")
338 +
339 + return mm_input
340
341 def _process_embeds(
342 self,
@@ -953,3 +973,15 @@ class InputPreprocessor:
973 def clear_cache(self) -> None:
974 if self.mm_processor_cache is not None:
975 self.mm_processor_cache.clear_cache()
976 +
977 +
978 +# Helper function to validate that a nested dictionary contains
979 +# only strings or list of strings as the leaf values.
980 +def contains_only_strings(obj: object):
981 + if isinstance(obj, str):
982 + return True
983 + if isinstance(obj, list):
984 + return all(isinstance(x, str) for x in obj)
985 + if isinstance(obj, dict):
986 + return all(contains_only_strings(v) for v in obj.values())
987 + return False
vllm/model_executor/models/terratorch.py
+4 -3
@@ -174,9 +174,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor):
174
175 mm_items = self._to_mm_items(mm_data)
176 tokenization_kwargs = tokenization_kwargs or {}
177 - mm_hashes = (mm_hash_overrides if mm_hash_overrides is not None else
178 - self._hash_mm_items(mm_items, hf_processor_mm_kwargs,
179 - tokenization_kwargs))
177 + mm_hashes = self._hash_mm_items(mm_items,
178 + hf_processor_mm_kwargs,
179 + tokenization_kwargs,
180 + mm_hash_overrides=mm_hash_overrides)
181 mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
182
183 mm_processed_data = BatchFeature(image_data)