@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ae2f60374
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 02:15 UTC

[Misc] Various cleanups for MM input processing (#29970)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 4, 2025 at 14:22 UTC 9ae2f603748446317c90fe40f1eb269e9a027815
14 files changed +67 -225
docs/features/multimodal_inputs.md
+3 -5
@@ -795,14 +795,12 @@ The following example demonstrates how to pass image embeddings to the OpenAI se
795 ??? code
796
797 ```python
798 + from vllm.utils.serial_utils import tensor2base64
799 +
800 image_embedding = torch.load(...)
801 grid_thw = torch.load(...) # Required by Qwen/Qwen2-VL-2B-Instruct
802
801 - buffer = io.BytesIO()
802 - torch.save(image_embedding, buffer)
803 - buffer.seek(0)
804 - binary_data = buffer.read()
805 - base64_image_embedding = base64.b64encode(binary_data).decode('utf-8')
803 + base64_image_embedding = tensor2base64(image_embedding)
804
805 client = OpenAI(
806 # defaults to os.environ.get("OPENAI_API_KEY")
examples/online_serving/prompt_embed_inference_with_openai_client.py
+3 -9
@@ -28,13 +28,11 @@ Dependencies:
28 - openai
29 """
30
31 -import base64
32 -import io
33 -
34 -import torch
31 import transformers
32 from openai import OpenAI
33
34 +from vllm.utils.serial_utils import tensor2base64
35 +
36
37 def main():
38 client = OpenAI(
@@ -58,11 +56,7 @@ def main():
56 prompt_embeds = embedding_layer(token_ids).squeeze(0)
57
58 # Prompt embeddings
61 - buffer = io.BytesIO()
62 - torch.save(prompt_embeds, buffer)
63 - buffer.seek(0)
64 - binary_data = buffer.read()
65 - encoded_embeds = base64.b64encode(binary_data).decode("utf-8")
59 + encoded_embeds = tensor2base64(prompt_embeds)
60
61 completion = client.completions.create(
62 model=model_name,
tests/entrypoints/openai/test_vision_embeds.py
+31 -44
@@ -2,64 +2,47 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import base64
5 -import io
5
6 import numpy as np
7 import pytest
8 import requests
9 import torch
10
12 -from ...utils import RemoteOpenAIServer
11 +from vllm.utils.serial_utils import tensor2base64
12
14 -MODEL_NAME = "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"
15 -DTYPE = "float16"
13 +from ...utils import RemoteOpenAIServer
14
15
18 -def _terratorch_dummy_inputs(model_name: str):
16 +def _terratorch_dummy_messages():
17 pixel_values = torch.full((6, 512, 512), 1.0, dtype=torch.float16)
18 location_coords = torch.full((1, 2), 1.0, dtype=torch.float16)
19
22 - buffer_tiff = io.BytesIO()
23 - torch.save(pixel_values, buffer_tiff)
24 - buffer_tiff.seek(0)
25 - binary_data = buffer_tiff.read()
26 - base64_tensor_embedding = base64.b64encode(binary_data).decode("utf-8")
27 -
28 - buffer_coord = io.BytesIO()
29 - torch.save(location_coords, buffer_coord)
30 - buffer_coord.seek(0)
31 - binary_data = buffer_coord.read()
32 - base64_coord_embedding = base64.b64encode(binary_data).decode("utf-8")
33 -
34 - return {
35 - "model": model_name,
36 - "additional_data": {"prompt_token_ids": [1]},
37 - "encoding_format": "base64",
38 - "messages": [
39 - {
40 - "role": "user",
41 - "content": [
42 - {
43 - "type": "image_embeds",
44 - "image_embeds": {
45 - "pixel_values": base64_tensor_embedding,
46 - "location_coords": base64_coord_embedding,
47 - },
48 - }
49 - ],
50 - }
51 - ],
52 - }
20 + return [
21 + {
22 + "role": "user",
23 + "content": [
24 + {
25 + "type": "image_embeds",
26 + "image_embeds": {
27 + "pixel_values": tensor2base64(pixel_values),
28 + "location_coords": tensor2base64(location_coords),
29 + },
30 + }
31 + ],
32 + }
33 + ]
34
35
55 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
56 -async def test_single_request(model_name: str):
36 +@pytest.mark.parametrize(
37 + "model_name", ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"]
38 +)
39 +def test_single_request(model_name: str):
40 args = [
41 "--runner",
42 "pooling",
43 # use half precision for speed and memory savings in CI environment
44 "--dtype",
62 - DTYPE,
45 + "float16",
46 "--enforce-eager",
47 "--trust-remote-code",
48 "--max-num-seqs",
@@ -70,11 +53,15 @@ async def test_single_request(model_name: str):
53 "--enable-mm-embeds",
54 ]
55
73 - with RemoteOpenAIServer(MODEL_NAME, args) as server:
74 - prompt = _terratorch_dummy_inputs(model_name)
75 -
76 - # test single pooling
77 - response = requests.post(server.url_for("pooling"), json=prompt)
56 + with RemoteOpenAIServer(model_name, args) as server:
57 + response = requests.post(
58 + server.url_for("pooling"),
59 + json={
60 + "model": model_name,
61 + "messages": _terratorch_dummy_messages(),
62 + "encoding_format": "base64",
63 + },
64 + )
65 response.raise_for_status()
66
67 output = response.json()["data"][0]["data"]
tests/entrypoints/test_chat_utils.py
+4 -123
@@ -29,6 +29,7 @@ from vllm.multimodal.utils import (
29 encode_video_base64,
30 )
31 from vllm.tokenizers import MistralTokenizer, get_tokenizer
32 +from vllm.utils.serial_utils import tensor2base64
33
34 from ..models.registry import HF_EXAMPLE_MODELS
35 from ..utils import VLLM_PATH
@@ -85,11 +86,6 @@ def phi3v_model_config_image_embeds():
86 )
87
88
88 -@pytest.fixture(scope="module")
89 -def phi3v_tokenizer():
90 - return get_tokenizer(PHI3V_MODEL_ID)
91 -
92 -
89 @pytest.fixture(scope="function")
90 def qwen2_audio_model_config():
91 return ModelConfig(
@@ -115,11 +111,6 @@ def audio_embeds_model_config():
111 )
112
113
118 -@pytest.fixture(scope="module")
119 -def qwen2_audio_tokenizer():
120 - return get_tokenizer(QWEN2AUDIO_MODEL_ID)
121 -
122 -
114 @pytest.fixture(scope="function")
115 def qwen25omni_model_config_mm_interleaved():
116 return ModelConfig(
@@ -134,11 +125,6 @@ def qwen25omni_model_config_mm_interleaved():
125 )
126
127
137 -@pytest.fixture(scope="module")
138 -def qwen25omni_tokenizer():
139 - return get_tokenizer(QWEN25OMNI_MODEL_ID)
140 -
141 -
128 @pytest.fixture(scope="function")
129 def mistral_model_config():
130 return ModelConfig(
@@ -150,11 +136,6 @@ def mistral_model_config():
136 )
137
138
153 -@pytest.fixture(scope="module")
154 -def mistral_tokenizer():
155 - return get_tokenizer(MISTRAL_MODEL_ID)
156 -
157 -
139 @pytest.fixture(scope="module")
140 def image_url():
141 image = ImageAsset("cherry_blossom")
@@ -239,7 +220,6 @@ def _assert_mm_data_inputs(
220
221 def test_parse_chat_messages_single_image(
222 phi3v_model_config,
242 - phi3v_tokenizer,
223 image_url,
224 ):
225 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -253,7 +233,6 @@ def test_parse_chat_messages_single_image(
233 }
234 ],
235 phi3v_model_config,
256 - phi3v_tokenizer,
236 content_format="string",
237 )
238
@@ -266,7 +245,6 @@ def test_parse_chat_messages_single_image(
245
246 def test_parse_chat_messages_single_image_with_uuid(
247 phi3v_model_config,
269 - phi3v_tokenizer,
248 image_url,
249 ):
250 image_uuid = str(hash(image_url))
@@ -287,7 +265,6 @@ def test_parse_chat_messages_single_image_with_uuid(
265 }
266 ],
267 phi3v_model_config,
290 - phi3v_tokenizer,
268 content_format="string",
269 )
270
@@ -300,7 +277,6 @@ def test_parse_chat_messages_single_image_with_uuid(
277
278 def test_parse_chat_messages_single_empty_image_with_uuid(
279 phi3v_model_config,
303 - phi3v_tokenizer,
280 image_url,
281 ):
282 image_uuid = str(hash(image_url))
@@ -319,7 +295,6 @@ def test_parse_chat_messages_single_empty_image_with_uuid(
295 }
296 ],
297 phi3v_model_config,
322 - phi3v_tokenizer,
298 content_format="string",
299 )
300
@@ -332,7 +307,6 @@ def test_parse_chat_messages_single_empty_image_with_uuid(
307
308 def test_parse_chat_messages_single_image_with_bad_uuid_format(
309 phi3v_model_config,
335 - phi3v_tokenizer,
310 image_url,
311 ):
312 image_uuid = str(hash(image_url))
@@ -354,7 +328,6 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format(
328 }
329 ],
330 phi3v_model_config,
357 - phi3v_tokenizer,
331 content_format="string",
332 )
333
@@ -367,7 +340,6 @@ def test_parse_chat_messages_single_image_with_bad_uuid_format(
340
341 def test_parse_chat_messages_multiple_images_with_uuids(
342 phi3v_model_config,
370 - phi3v_tokenizer,
343 image_url,
344 ):
345 image_uuid1 = "my_uuid_1"
@@ -397,7 +369,6 @@ def test_parse_chat_messages_multiple_images_with_uuids(
369 }
370 ],
371 phi3v_model_config,
400 - phi3v_tokenizer,
372 content_format="string",
373 )
374
@@ -413,7 +384,6 @@ def test_parse_chat_messages_multiple_images_with_uuids(
384
385 def test_parse_chat_messages_multiple_empty_images_with_uuids(
386 phi3v_model_config,
416 - phi3v_tokenizer,
387 image_url,
388 ):
389 image_uuid1 = "my_uuid_1"
@@ -439,7 +409,6 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids(
409 }
410 ],
411 phi3v_model_config,
442 - phi3v_tokenizer,
412 content_format="string",
413 )
414
@@ -455,7 +424,6 @@ def test_parse_chat_messages_multiple_empty_images_with_uuids(
424
425 def test_parse_chat_messages_mixed_empty_images_with_uuids(
426 phi3v_model_config,
458 - phi3v_tokenizer,
427 image_url,
428 ):
429 image_uuid1 = "my_uuid_1"
@@ -483,7 +451,6 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids(
451 }
452 ],
453 phi3v_model_config,
486 - phi3v_tokenizer,
454 content_format="string",
455 )
456
@@ -500,7 +467,6 @@ def test_parse_chat_messages_mixed_empty_images_with_uuids(
467 @pytest.mark.asyncio
468 async def test_parse_chat_messages_single_image_with_uuid_async(
469 phi3v_model_config,
503 - phi3v_tokenizer,
470 image_url,
471 ):
472 image_uuid = str(hash(image_url))
@@ -519,7 +485,6 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
485 }
486 ],
487 phi3v_model_config,
522 - phi3v_tokenizer,
488 content_format="string",
489 )
490
@@ -533,7 +498,6 @@ async def test_parse_chat_messages_single_image_with_uuid_async(
498 @pytest.mark.asyncio
499 async def test_parse_chat_messages_empty_image_with_uuid_async(
500 phi3v_model_config,
536 - phi3v_tokenizer,
501 image_url,
502 ):
503 image_uuid = str(hash(image_url))
@@ -552,7 +516,6 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
516 }
517 ],
518 phi3v_model_config,
555 - phi3v_tokenizer,
519 content_format="string",
520 )
521
@@ -566,7 +529,6 @@ async def test_parse_chat_messages_empty_image_with_uuid_async(
529 @pytest.mark.asyncio
530 async def test_parse_chat_messages_multiple_images_with_uuids_async(
531 phi3v_model_config,
569 - phi3v_tokenizer,
532 image_url,
533 ):
534 image_uuid1 = "my_uuid_1"
@@ -592,7 +554,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
554 }
555 ],
556 phi3v_model_config,
595 - phi3v_tokenizer,
557 content_format="string",
558 )
559
@@ -609,7 +570,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async(
570 @pytest.mark.asyncio
571 async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
572 phi3v_model_config,
612 - phi3v_tokenizer,
573 image_url,
574 ):
575 image_uuid1 = "my_uuid_1"
@@ -635,7 +595,6 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
595 }
596 ],
597 phi3v_model_config,
638 - phi3v_tokenizer,
598 content_format="string",
599 )
600
@@ -652,7 +611,6 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async(
611 @pytest.mark.asyncio
612 async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
613 phi3v_model_config,
655 - phi3v_tokenizer,
614 image_url,
615 ):
616 image_uuid2 = "my_uuid_2"
@@ -676,7 +634,6 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
634 }
635 ],
636 phi3v_model_config,
679 - phi3v_tokenizer,
637 content_format="string",
638 )
639
@@ -692,7 +649,6 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async(
649
650 def test_parse_chat_messages_empty_system(
651 mistral_model_config,
695 - mistral_tokenizer,
652 ):
653 # Test string format
654 conversation, _, _ = parse_chat_messages(
@@ -704,7 +660,6 @@ def test_parse_chat_messages_empty_system(
660 },
661 ],
662 mistral_model_config,
707 - mistral_tokenizer,
663 content_format="string",
664 )
665 assert conversation == [
@@ -722,7 +677,6 @@ def test_parse_chat_messages_empty_system(
677 },
678 ],
679 mistral_model_config,
725 - mistral_tokenizer,
680 content_format="openai",
681 )
682 assert conversation == [
@@ -734,7 +688,6 @@ def test_parse_chat_messages_empty_system(
688 @pytest.mark.asyncio
689 async def test_parse_chat_messages_single_image_async(
690 phi3v_model_config,
737 - phi3v_tokenizer,
691 image_url,
692 ):
693 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
@@ -748,7 +701,6 @@ async def test_parse_chat_messages_single_image_async(
701 }
702 ],
703 phi3v_model_config,
751 - phi3v_tokenizer,
704 content_format="string",
705 )
706
@@ -761,7 +713,6 @@ async def test_parse_chat_messages_single_image_async(
713
714 def test_parse_chat_messages_multiple_images(
715 phi3v_model_config,
764 - phi3v_tokenizer,
716 image_url,
717 ):
718 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -779,7 +730,6 @@ def test_parse_chat_messages_multiple_images(
730 }
731 ],
732 phi3v_model_config,
782 - phi3v_tokenizer,
733 content_format="string",
734 )
735
@@ -795,7 +745,6 @@ def test_parse_chat_messages_multiple_images(
745
746 def test_parse_chat_messages_empty_pil_image_with_uuid(
747 phi3v_model_config,
798 - phi3v_tokenizer,
748 ):
749 uuid = "abcd"
750 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -809,7 +758,6 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(
758 }
759 ],
760 phi3v_model_config,
812 - phi3v_tokenizer,
761 content_format="string",
762 )
763
@@ -825,7 +773,6 @@ def test_parse_chat_messages_empty_pil_image_with_uuid(
773
774 def test_parse_chat_messages_empty_image_embeds_with_uuid(
775 phi3v_model_config_image_embeds,
828 - phi3v_tokenizer,
776 ):
777 uuid = "abcd"
778 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -839,7 +786,6 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(
786 }
787 ],
788 phi3v_model_config_image_embeds,
842 - phi3v_tokenizer,
789 content_format="string",
790 )
791
@@ -857,7 +803,6 @@ def test_parse_chat_messages_empty_image_embeds_with_uuid(
803
804 def test_parse_chat_messages_empty_audio_embeds_with_uuid(
805 audio_embeds_model_config,
860 - qwen2_audio_tokenizer,
806 ):
807 """Test audio_embeds with UUID (no actual embeds data)."""
808 uuid = "test-audio-uuid-123"
@@ -873,7 +818,6 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid(
818 }
819 ],
820 audio_embeds_model_config,
876 - qwen2_audio_tokenizer,
821 content_format="string",
822 )
823
@@ -889,11 +833,8 @@ def test_parse_chat_messages_empty_audio_embeds_with_uuid(
833
834 def test_parse_chat_messages_audio_embeds_with_string(
835 audio_embeds_model_config,
892 - qwen2_audio_tokenizer,
836 ):
837 """Test audio_embeds with base64 string embedding data."""
895 - import base64
896 - import io
838
839 import torch
840
@@ -901,11 +842,7 @@ def test_parse_chat_messages_audio_embeds_with_string(
842 audio_embedding = torch.randn(1, 128, 768)
843
844 # Encode it as base64
904 - buffer = io.BytesIO()
905 - torch.save(audio_embedding, buffer)
906 - buffer.seek(0)
907 - binary_data = buffer.read()
908 - base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8")
845 + base64_audio_embedding = tensor2base64(audio_embedding)
846
847 conversation, mm_data, mm_uuids = parse_chat_messages(
848 [
@@ -921,7 +858,6 @@ def test_parse_chat_messages_audio_embeds_with_string(
858 }
859 ],
860 audio_embeds_model_config,
924 - qwen2_audio_tokenizer,
861 content_format="string",
862 )
863
@@ -939,11 +875,8 @@ def test_parse_chat_messages_audio_embeds_with_string(
875 @pytest.mark.asyncio
876 async def test_parse_chat_messages_audio_embeds_async(
877 audio_embeds_model_config,
942 - qwen2_audio_tokenizer,
878 ):
879 """Test audio_embeds with async futures."""
945 - import base64
946 - import io
880
881 import torch
882
@@ -951,11 +884,7 @@ async def test_parse_chat_messages_audio_embeds_async(
884 audio_embedding = torch.randn(1, 128, 768)
885
886 # Encode it as base64
954 - buffer = io.BytesIO()
955 - torch.save(audio_embedding, buffer)
956 - buffer.seek(0)
957 - binary_data = buffer.read()
958 - base64_audio_embedding = base64.b64encode(binary_data).decode("utf-8")
887 + base64_audio_embedding = tensor2base64(audio_embedding)
888
889 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
890 [
@@ -971,7 +900,6 @@ async def test_parse_chat_messages_audio_embeds_async(
900 }
901 ],
902 audio_embeds_model_config,
974 - qwen2_audio_tokenizer,
903 content_format="string",
904 )
905
@@ -990,7 +918,6 @@ async def test_parse_chat_messages_audio_embeds_async(
918 @pytest.mark.asyncio
919 async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
920 phi3v_model_config_image_embeds,
993 - phi3v_tokenizer,
921 ):
922 uuid = "abcd"
923 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
@@ -1004,7 +931,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
931 }
932 ],
933 phi3v_model_config_image_embeds,
1007 - phi3v_tokenizer,
934 content_format="string",
935 )
936
@@ -1024,7 +950,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async(
950 @pytest.mark.asyncio
951 async def test_parse_chat_messages_multiple_images_async(
952 phi3v_model_config,
1027 - phi3v_tokenizer,
953 image_url,
954 ):
955 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
@@ -1042,7 +967,6 @@ async def test_parse_chat_messages_multiple_images_async(
967 }
968 ],
969 phi3v_model_config,
1045 - phi3v_tokenizer,
970 content_format="string",
971 )
972
@@ -1058,7 +982,6 @@ async def test_parse_chat_messages_multiple_images_async(
982
983 def test_parse_chat_messages_placeholder_already_in_prompt(
984 phi3v_model_config,
1061 - phi3v_tokenizer,
985 image_url,
986 ):
987 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1076,7 +999,6 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
999 }
1000 ],
1001 phi3v_model_config,
1079 - phi3v_tokenizer,
1002 content_format="string",
1003 )
1004 assert conversation == [
@@ -1091,7 +1013,6 @@ def test_parse_chat_messages_placeholder_already_in_prompt(
1013
1014 def test_parse_chat_messages_placeholder_one_already_in_prompt(
1015 phi3v_model_config,
1094 - phi3v_tokenizer,
1016 image_url,
1017 ):
1018 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1110,7 +1031,6 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
1031 }
1032 ],
1033 phi3v_model_config,
1113 - phi3v_tokenizer,
1034 content_format="string",
1035 )
1036
@@ -1127,7 +1047,6 @@ def test_parse_chat_messages_placeholder_one_already_in_prompt(
1047
1048 def test_parse_chat_messages_multiple_images_across_messages(
1049 phi3v_model_config,
1130 - phi3v_tokenizer,
1050 image_url,
1051 ):
1052 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1149,7 +1068,6 @@ def test_parse_chat_messages_multiple_images_across_messages(
1068 },
1069 ],
1070 phi3v_model_config,
1152 - phi3v_tokenizer,
1071 content_format="string",
1072 )
1073
@@ -1164,7 +1082,6 @@ def test_parse_chat_messages_multiple_images_across_messages(
1082
1083 def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1084 phi3v_model_config,
1167 - phi3v_tokenizer,
1085 image_url,
1086 ):
1087 image_uuid = str(hash(image_url))
@@ -1195,7 +1112,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1112 },
1113 ],
1114 phi3v_model_config,
1198 - phi3v_tokenizer,
1115 content_format="string",
1116 )
1117
@@ -1210,7 +1126,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_across_messages(
1126
1127 def test_parse_chat_messages_context_text_format(
1128 phi3v_model_config,
1213 - phi3v_tokenizer,
1129 ):
1130 conversation, mm_data, mm_uuids = parse_chat_messages(
1131 [
@@ -1222,7 +1137,6 @@ def test_parse_chat_messages_context_text_format(
1137 {"role": "user", "content": "What about this one?"},
1138 ],
1139 phi3v_model_config,
1225 - phi3v_tokenizer,
1140 content_format="openai",
1141 )
1142
@@ -1246,7 +1160,6 @@ def test_parse_chat_messages_context_text_format(
1160
1161 def test_parse_chat_messages_rejects_too_many_images_in_one_message(
1162 phi3v_model_config,
1249 - phi3v_tokenizer,
1163 image_url,
1164 ):
1165 with warnings.catch_warnings():
@@ -1277,14 +1190,12 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
1190 }
1191 ],
1192 phi3v_model_config,
1280 - phi3v_tokenizer,
1193 content_format="string",
1194 )
1195
1196
1197 def test_parse_chat_messages_rejects_too_many_images_across_messages(
1198 phi3v_model_config,
1287 - phi3v_tokenizer,
1199 image_url,
1200 ):
1201 with warnings.catch_warnings():
@@ -1322,14 +1233,12 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
1233 },
1234 ],
1235 phi3v_model_config,
1325 - phi3v_tokenizer,
1236 content_format="string",
1237 )
1238
1239
1240 def test_parse_chat_messages_multiple_images_uncommon_input(
1241 phi3v_model_config,
1332 - phi3v_tokenizer,
1242 image_url,
1243 ):
1244 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1344,7 +1253,6 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
1253 }
1254 ],
1255 phi3v_model_config,
1347 - phi3v_tokenizer,
1256 content_format="string",
1257 )
1258
@@ -1360,7 +1268,6 @@ def test_parse_chat_messages_multiple_images_uncommon_input(
1268
1269 def test_parse_chat_messages_multiple_images_interleave(
1270 phi3v_model_config_mm_interleaved,
1363 - phi3v_tokenizer,
1271 image_url,
1272 ):
1273 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1380,7 +1287,6 @@ def test_parse_chat_messages_multiple_images_interleave(
1287 }
1288 ],
1289 phi3v_model_config_mm_interleaved,
1383 - phi3v_tokenizer,
1290 content_format="string",
1291 )
1292
@@ -1398,7 +1304,6 @@ def test_parse_chat_messages_multiple_images_interleave(
1304 @pytest.mark.asyncio
1305 async def test_parse_chat_messages_multiple_images_interleave_async(
1306 phi3v_model_config_mm_interleaved,
1401 - phi3v_tokenizer,
1307 image_url,
1308 ):
1309 conversation, mm_data, mm_uuids = parse_chat_messages_futures(
@@ -1418,7 +1323,6 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1323 }
1324 ],
1325 phi3v_model_config_mm_interleaved,
1421 - phi3v_tokenizer,
1326 content_format="string",
1327 )
1328
@@ -1436,7 +1340,6 @@ async def test_parse_chat_messages_multiple_images_interleave_async(
1340 @pytest.mark.asyncio
1341 async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1342 phi3v_model_config_mm_interleaved,
1439 - phi3v_tokenizer,
1343 image_url,
1344 ):
1345 image_uuid = str(hash(image_url))
@@ -1465,7 +1368,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1368 }
1369 ],
1370 phi3v_model_config_mm_interleaved,
1468 - phi3v_tokenizer,
1371 content_format="string",
1372 )
1373
@@ -1482,7 +1384,6 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async(
1384
1385 def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1386 phi3v_model_config_mm_interleaved,
1485 - phi3v_tokenizer,
1387 image_url,
1388 ):
1389 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -1505,7 +1406,6 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1406 },
1407 ],
1408 phi3v_model_config_mm_interleaved,
1508 - phi3v_tokenizer,
1409 content_format="string",
1410 )
1411
@@ -1523,7 +1423,6 @@ def test_parse_chat_messages_multiple_images_multiple_messages_interleave(
1423
1424 def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interleave(
1425 phi3v_model_config_mm_interleaved,
1526 - phi3v_tokenizer,
1426 image_url,
1427 ):
1428 image_uuid = str(hash(image_url))
@@ -1555,7 +1454,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl
1454 },
1455 ],
1456 phi3v_model_config_mm_interleaved,
1558 - phi3v_tokenizer,
1457 content_format="string",
1458 )
1459
@@ -1573,7 +1471,6 @@ def test_parse_chat_messages_multiple_images_with_uuids_multiple_messages_interl
1471
1472 def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1473 qwen25omni_model_config_mm_interleaved,
1576 - qwen25omni_tokenizer,
1474 image_url,
1475 video_url,
1476 audio_url,
@@ -1601,7 +1498,6 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1498 },
1499 ],
1500 qwen25omni_model_config_mm_interleaved,
1604 - qwen25omni_tokenizer,
1501 content_format="string",
1502 )
1503
@@ -1627,7 +1523,6 @@ def test_parse_chat_messages_multiple_modals_multiple_messages_interleave(
1523
1524 def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interleave(
1525 qwen25omni_model_config_mm_interleaved,
1630 - qwen25omni_tokenizer,
1526 image_url,
1527 video_url,
1528 audio_url,
@@ -1671,7 +1566,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
1566 },
1567 ],
1568 qwen25omni_model_config_mm_interleaved,
1674 - qwen25omni_tokenizer,
1569 content_format="string",
1570 )
1571
@@ -1699,7 +1593,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_messages_interl
1593
1594 def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_messages_interleave( # noqa: E501
1595 qwen25omni_model_config_mm_interleaved,
1702 - qwen25omni_tokenizer,
1596 image_url,
1597 video_url,
1598 audio_url,
@@ -1743,7 +1636,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes
1636 },
1637 ],
1638 qwen25omni_model_config_mm_interleaved,
1746 - qwen25omni_tokenizer,
1639 content_format="string",
1640 )
1641
@@ -1775,7 +1667,6 @@ def test_parse_chat_messages_multiple_modals_with_uuids_multiple_empty_media_mes
1667
1668 def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_messages_interleave( # noqa: E501
1669 qwen25omni_model_config_mm_interleaved,
1778 - qwen25omni_tokenizer,
1670 image_url,
1671 video_url,
1672 audio_url,
@@ -1811,7 +1702,6 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message
1702 },
1703 ],
1704 qwen25omni_model_config_mm_interleaved,
1814 - qwen25omni_tokenizer,
1705 content_format="string",
1706 )
1707
@@ -1837,7 +1727,6 @@ def test_parse_chat_messages_multiple_modals_with_partial_uuids_multiple_message
1727
1728 def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
1729 phi3v_model_config_mm_interleaved,
1840 - phi3v_tokenizer,
1730 image_url,
1731 ):
1732 with pytest.raises(
@@ -1861,7 +1750,6 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
1750 }
1751 ],
1752 phi3v_model_config_mm_interleaved,
1864 - phi3v_tokenizer,
1753 content_format="string",
1754 )
1755
@@ -2237,9 +2125,7 @@ def test_resolve_content_format_examples(template_path, expected_format):
2125 assert resolved_format == expected_format
2126
2127
2240 -def test_parse_chat_messages_include_thinking_chunk(
2241 - mistral_model_config, mistral_tokenizer
2242 -):
2128 +def test_parse_chat_messages_include_thinking_chunk(mistral_model_config):
2129 messages = [
2130 {
2131 "role": "system",
@@ -2269,7 +2155,6 @@ def test_parse_chat_messages_include_thinking_chunk(
2155 conversation_with_thinking, _, _ = parse_chat_messages(
2156 messages,
2157 mistral_model_config,
2272 - mistral_tokenizer,
2158 content_format="openai",
2159 )
2160
@@ -2353,7 +2238,6 @@ def test_apply_mistral_chat_template_thinking_chunk():
2238
2239 def test_parse_chat_messages_single_empty_audio_with_uuid(
2240 qwen2_audio_model_config,
2356 - qwen2_audio_tokenizer,
2241 ):
2242 audio_uuid = "abcd"
2243 conversation, mm_data, mm_uuids = parse_chat_messages(
@@ -2371,7 +2255,6 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2255 }
2256 ],
2257 qwen2_audio_model_config,
2374 - qwen2_audio_tokenizer,
2258 content_format="string",
2259 )
2260
@@ -2389,7 +2272,6 @@ def test_parse_chat_messages_single_empty_audio_with_uuid(
2272 @pytest.mark.asyncio
2273 async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2274 qwen2_audio_model_config,
2392 - qwen2_audio_tokenizer,
2275 ):
2276 audio_uuid = "abcd"
2277 conversation, mm_future, mm_uuids = parse_chat_messages_futures(
@@ -2407,7 +2289,6 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async(
2289 }
2290 ],
2291 qwen2_audio_model_config,
2410 - qwen2_audio_tokenizer,
2292 content_format="string",
2293 )
2294
tests/v1/entrypoints/openai/test_completion_with_image_embeds.py
+2 -15
@@ -1,8 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import base64
5 -import io
4 import json
5
6 import openai # use the official client for correctness check
@@ -13,6 +11,7 @@ from transformers import AutoConfig
11
12 from tests.conftest import ImageTestAssets
13 from tests.utils import RemoteOpenAIServer
14 +from vllm.utils.serial_utils import tensor2base64
15
16 # any model with a chat template should work here
17 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
@@ -50,18 +49,6 @@ async def client_with_image_embeds(server_with_image_embeds):
49 yield async_client
50
51
53 -def encode_image_embedding_to_base64(image_embedding) -> str:
54 - """
55 - Encode image embedding to base64 string
56 - """
57 - buffer = io.BytesIO()
58 - torch.save(image_embedding, buffer)
59 - buffer.seek(0)
60 - binary_data = buffer.read()
61 - base64_image_embedding = base64.b64encode(binary_data).decode("utf-8")
62 - return base64_image_embedding
63 -
64 -
52 @pytest.mark.asyncio
53 @pytest.mark.parametrize("model_name", [MODEL_NAME])
54 @pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32])
@@ -73,7 +60,7 @@ async def test_completions_with_image_embeds(
60 ):
61 # Test case: Single image embeds input
62 image_embeds = image_assets[0].image_embeds.to(dtype=dtype)
76 - base64_image_embedding = encode_image_embedding_to_base64(image_embeds)
63 + base64_image_embedding = tensor2base64(image_embeds)
64 chat_completion = await client_with_image_embeds.chat.completions.create(
65 messages=[
66 {"role": "system", "content": "You are a helpful assistant."},
vllm/entrypoints/chat_utils.py
+5 -8
@@ -536,7 +536,7 @@ def resolve_hf_chat_template(
536 def _resolve_chat_template_content_format(
537 chat_template: str | None,
538 tools: list[dict[str, Any]] | None,
539 - tokenizer: TokenizerLike,
539 + tokenizer: TokenizerLike | None,
540 *,
541 model_config: ModelConfig,
542 ) -> _ChatTemplateContentFormat:
@@ -593,7 +593,7 @@ def resolve_chat_template_content_format(
593 chat_template: str | None,
594 tools: list[dict[str, Any]] | None,
595 given_format: ChatTemplateContentFormatOption,
596 - tokenizer: TokenizerLike,
596 + tokenizer: TokenizerLike | None,
597 *,
598 model_config: ModelConfig,
599 ) -> _ChatTemplateContentFormat:
@@ -627,11 +627,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
627 maximum per prompt.
628 """
629
630 - def __init__(self, model_config: ModelConfig, tokenizer: TokenizerLike):
630 + def __init__(self, model_config: ModelConfig):
631 super().__init__()
632
633 self._model_config = model_config
634 - self._tokenizer = tokenizer
634
635 self._items_by_modality = defaultdict[str, list[_T | None]](list)
636 self._uuids_by_modality = defaultdict[str, list[str | None]](list)
@@ -1612,7 +1611,6 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:
1611 def parse_chat_messages(
1612 messages: list[ChatCompletionMessageParam],
1613 model_config: ModelConfig,
1615 - tokenizer: TokenizerLike,
1614 content_format: _ChatTemplateContentFormat,
1615 ) -> tuple[
1616 list[ConversationMessage],
@@ -1620,7 +1618,7 @@ def parse_chat_messages(
1618 MultiModalUUIDDict | None,
1619 ]:
1620 conversation: list[ConversationMessage] = []
1623 - mm_tracker = MultiModalItemTracker(model_config, tokenizer)
1621 + mm_tracker = MultiModalItemTracker(model_config)
1622
1623 for msg in messages:
1624 sub_messages = _parse_chat_message_content(
@@ -1644,7 +1642,6 @@ def parse_chat_messages(
1642 def parse_chat_messages_futures(
1643 messages: list[ChatCompletionMessageParam],
1644 model_config: ModelConfig,
1647 - tokenizer: TokenizerLike,
1645 content_format: _ChatTemplateContentFormat,
1646 ) -> tuple[
1647 list[ConversationMessage],
@@ -1652,7 +1649,7 @@ def parse_chat_messages_futures(
1649 MultiModalUUIDDict | None,
1650 ]:
1651 conversation: list[ConversationMessage] = []
1655 - mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)
1652 + mm_tracker = AsyncMultiModalItemTracker(model_config)
1653
1654 for msg in messages:
1655 sub_messages = _parse_chat_message_content(
vllm/entrypoints/llm.py
-1
@@ -834,7 +834,6 @@ class LLM:
834 conversation, mm_data, mm_uuids = parse_chat_messages(
835 msgs,
836 model_config,
837 - tokenizer,
837 content_format=resolved_content_format,
838 )
839
vllm/entrypoints/openai/serving_engine.py
-6
@@ -1088,11 +1088,6 @@ class OpenAIServing:
1088 Sequence[RequestPrompt],
1089 list[EngineTokensPrompt],
1090 ]:
1091 - if tokenizer is None:
1092 - raise ValueError(
1093 - "Unable to get tokenizer because `skip_tokenizer_init=True`"
1094 - )
1095 -
1091 model_config = self.model_config
1092
1093 resolved_content_format = resolve_chat_template_content_format(
@@ -1105,7 +1100,6 @@ class OpenAIServing:
1100 conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
1101 messages,
1102 model_config,
1108 - tokenizer,
1103 content_format=resolved_content_format,
1104 )
1105
vllm/entrypoints/score_utils.py
+1 -4
@@ -89,12 +89,10 @@ def parse_score_data(
89 data_1: str | ScoreContentPartParam,
90 data_2: str | ScoreContentPartParam,
91 model_config: ModelConfig,
92 - tokenizer: TokenizerLike,
92 ) -> tuple[str, str, MultiModalDataDict | None]:
94 - mm_tracker = MultiModalItemTracker(model_config, tokenizer)
93 + mm_tracker = MultiModalItemTracker(model_config)
94
95 content_1 = _parse_score_content(data_1, mm_tracker)
97 -
96 content_2 = _parse_score_content(data_2, mm_tracker)
97
98 def ensure_str(content: _ContentPart | None) -> str:
@@ -188,7 +186,6 @@ def get_score_prompt(
186 data_1,
187 data_2,
188 model_config,
191 - tokenizer,
189 )
190 from vllm.model_executor.model_loader import get_model_cls
191
vllm/model_executor/models/hunyuan_vision.py
+2 -1
@@ -62,6 +62,7 @@ from vllm.multimodal.inputs import (
62 from vllm.multimodal.parse import (
63 DictEmbeddingItems,
64 ImageSize,
65 + ModalityDataItems,
66 MultiModalDataItems,
67 MultiModalDataParser,
68 )
@@ -570,7 +571,7 @@ class HunYuanVLMultiModalDataParser(MultiModalDataParser):
571 def _parse_image_data(
572 self,
573 data: dict[str, torch.Tensor] | ModalityData[ImageItem],
573 - ):
574 + ) -> ModalityDataItems[Any, Any] | None:
575 if isinstance(data, dict):
576 return DictEmbeddingItems(
577 data,
vllm/model_executor/models/keye.py
+2 -2
@@ -1000,7 +1000,7 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
1000 def _parse_image_data(
1001 self,
1002 data: dict[str, torch.Tensor] | ModalityData[ImageItem],
1003 - ) -> ModalityDataItems[Any, Any]:
1003 + ) -> ModalityDataItems[Any, Any] | None:
1004 if isinstance(data, dict):
1005 return DictEmbeddingItems(
1006 data,
@@ -1017,7 +1017,7 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
1017 def _parse_video_data(
1018 self,
1019 data: dict[str, torch.Tensor] | ModalityData[VideoItem],
1020 - ) -> ModalityDataItems[Any, Any]:
1020 + ) -> ModalityDataItems[Any, Any] | None:
1021 if isinstance(data, dict):
1022 return DictEmbeddingItems(
1023 data,
vllm/model_executor/models/keye_vl1_5.py
+2 -2
@@ -333,7 +333,7 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
333 def _parse_image_data(
334 self,
335 data: dict[str, torch.Tensor] | ModalityData[ImageItem],
336 - ) -> ModalityDataItems[Any, Any]:
336 + ) -> ModalityDataItems[Any, Any] | None:
337 if isinstance(data, dict):
338 return DictEmbeddingItems(
339 data,
@@ -350,7 +350,7 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
350 def _parse_video_data(
351 self,
352 data: dict[str, torch.Tensor] | ModalityData[VideoItem],
353 - ) -> ModalityDataItems[Any, Any]:
353 + ) -> ModalityDataItems[Any, Any] | None:
354 if isinstance(data, dict):
355 return DictEmbeddingItems(
356 data,
vllm/multimodal/audio.py
+2 -5
@@ -11,6 +11,7 @@ import pybase64
11 import torch
12
13 from vllm.utils.import_utils import PlaceholderModule
14 +from vllm.utils.serial_utils import tensor2base64
15
16 from .base import MediaIO
17
@@ -135,8 +136,4 @@ class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
136 return torch.load(filepath, weights_only=True)
137
138 def encode_base64(self, media: torch.Tensor) -> str:
138 - buffer = BytesIO()
139 - torch.save(media, buffer)
140 - buffer.seek(0)
141 - binary_data = buffer.read()
142 - return pybase64.b64encode(binary_data).decode("utf-8")
139 + return tensor2base64(media)
vllm/utils/serial_utils.py
+10
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import base64
4 +import io
5 import sys
6 from dataclasses import dataclass
7 from typing import Literal
@@ -52,6 +53,15 @@ Endianness = Literal["native", "big", "little"]
53 EncodingFormat = Literal["float", "base64", "bytes"]
54
55
56 +def tensor2base64(x: torch.Tensor) -> str:
57 + with io.BytesIO() as buf:
58 + torch.save(x, buf)
59 + buf.seek(0)
60 + binary_data = buf.read()
61 +
62 + return base64.b64encode(binary_data).decode("utf-8")
63 +
64 +
65 def tensor2binary(
66 tensor: torch.Tensor, embed_dtype: EmbedDType, endianness: Endianness
67 ) -> bytes: