@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b3cf368d79
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[V1][Molmo] Fix get_multimodal_embeddings() in molmo.py (#14161)

lkchen committed Mar 4, 2025 at 07:43 UTC b3cf368d79b4409120ab90f2096d254f52a43fcc
22 files changed +249 -150
examples/offline_inference/vision_language.py
+176 -118
@@ -21,7 +21,7 @@ from vllm.utils import FlexibleArgumentParser
21
22
23 # Aria
24 -def run_aria(question: str, modality: str):
24 +def run_aria(questions: list[str], modality: str):
25 assert modality == "image"
26 model_name = "rhymes-ai/Aria"
27
@@ -32,41 +32,42 @@ def run_aria(question: str, modality: str):
32 dtype="bfloat16",
33 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
34
35 - prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
36 - "<|im_end|>\n<|im_start|>assistant\n")
35 + prompts = [(f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
36 + "<|im_end|>\n<|im_start|>assistant\n")
37 + for question in questions]
38
39 stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
39 - return llm, prompt, stop_token_ids
40 + return llm, prompts, stop_token_ids
41
42
43 # BLIP-2
43 -def run_blip2(question: str, modality: str):
44 +def run_blip2(questions: list[str], modality: str):
45 assert modality == "image"
46
47 # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
48 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
48 - prompt = f"Question: {question} Answer:"
49 + prompts = [f"Question: {question} Answer:" for question in questions]
50 llm = LLM(model="Salesforce/blip2-opt-2.7b",
51 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
52 stop_token_ids = None
52 - return llm, prompt, stop_token_ids
53 + return llm, prompts, stop_token_ids
54
55
56 # Chameleon
56 -def run_chameleon(question: str, modality: str):
57 +def run_chameleon(questions: list[str], modality: str):
58 assert modality == "image"
59
59 - prompt = f"{question}<image>"
60 + prompts = [f"{question}<image>" for question in questions]
61 llm = LLM(model="facebook/chameleon-7b",
62 max_model_len=4096,
63 max_num_seqs=2,
64 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
65 stop_token_ids = None
65 - return llm, prompt, stop_token_ids
66 + return llm, prompts, stop_token_ids
67
68
69 # Deepseek-VL2
69 -def run_deepseek_vl2(question: str, modality: str):
70 +def run_deepseek_vl2(questions: list[str], modality: str):
71 assert modality == "image"
72
73 model_name = "deepseek-ai/deepseek-vl2-tiny"
@@ -77,9 +78,12 @@ def run_deepseek_vl2(question: str, modality: str):
78 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
79 hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]})
80
80 - prompt = f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
81 + prompts = [
82 + f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
83 + for question in questions
84 + ]
85 stop_token_ids = None
82 - return llm, prompt, stop_token_ids
86 + return llm, prompts, stop_token_ids
87
88
89 # Florence2
@@ -99,20 +103,20 @@ def run_florence2(question: str, modality: str):
103
104
105 # Fuyu
102 -def run_fuyu(question: str, modality: str):
106 +def run_fuyu(questions: list[str], modality: str):
107 assert modality == "image"
108
105 - prompt = f"{question}\n"
109 + prompts = [f"{question}\n" for question in questions]
110 llm = LLM(model="adept/fuyu-8b",
111 max_model_len=2048,
112 max_num_seqs=2,
113 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
114 stop_token_ids = None
111 - return llm, prompt, stop_token_ids
115 + return llm, prompts, stop_token_ids
116
117
118 # GLM-4v
115 -def run_glm4v(question: str, modality: str):
119 +def run_glm4v(questions: list[str], modality: str):
120 assert modality == "image"
121 model_name = "THUDM/glm-4v-9b"
122
@@ -124,15 +128,17 @@ def run_glm4v(question: str, modality: str):
128 hf_overrides={"architectures": ["GLM4VForCausalLM"]},
129 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
130
127 - prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
128 - {question}<|assistant|>"
131 + prompts = [
132 + f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
133 + {question}<|assistant|>" for question in questions
134 + ]
135
136 stop_token_ids = [151329, 151336, 151338]
131 - return llm, prompt, stop_token_ids
137 + return llm, prompts, stop_token_ids
138
139
140 # H2OVL-Mississippi
135 -def run_h2ovl(question: str, modality: str):
141 +def run_h2ovl(questions: list[str], modality: str):
142 assert modality == "image"
143
144 model_name = "h2oai/h2ovl-mississippi-800m"
@@ -146,19 +152,24 @@ def run_h2ovl(question: str, modality: str):
152
153 tokenizer = AutoTokenizer.from_pretrained(model_name,
154 trust_remote_code=True)
149 - messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
150 - prompt = tokenizer.apply_chat_template(messages,
151 - tokenize=False,
152 - add_generation_prompt=True)
155 + prompts = [
156 + tokenizer.apply_chat_template([{
157 + 'role': 'user',
158 + 'content': f"<image>\n{question}"
159 + }],
160 + tokenize=False,
161 + add_generation_prompt=True)
162 + for question in questions
163 + ]
164
165 # Stop tokens for H2OVL-Mississippi
166 # https://huggingface.co/h2oai/h2ovl-mississippi-800m
167 stop_token_ids = [tokenizer.eos_token_id]
157 - return llm, prompt, stop_token_ids
168 + return llm, prompts, stop_token_ids
169
170
171 # Idefics3-8B-Llama3
161 -def run_idefics3(question: str, modality: str):
172 +def run_idefics3(questions: list[str], modality: str):
173 assert modality == "image"
174 model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
175
@@ -176,15 +187,15 @@ def run_idefics3(question: str, modality: str):
187 },
188 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
189 )
179 - prompt = (
190 + prompts = [(
191 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
181 - )
192 + ) for question in questions]
193 stop_token_ids = None
183 - return llm, prompt, stop_token_ids
194 + return llm, prompts, stop_token_ids
195
196
197 # InternVL
187 -def run_internvl(question: str, modality: str):
198 +def run_internvl(questions: list[str], modality: str):
199 assert modality == "image"
200
201 model_name = "OpenGVLab/InternVL2-2B"
@@ -198,10 +209,15 @@ def run_internvl(question: str, modality: str):
209
210 tokenizer = AutoTokenizer.from_pretrained(model_name,
211 trust_remote_code=True)
201 - messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
202 - prompt = tokenizer.apply_chat_template(messages,
203 - tokenize=False,
204 - add_generation_prompt=True)
212 + prompts = [
213 + tokenizer.apply_chat_template([{
214 + 'role': 'user',
215 + 'content': f"<image>\n{question}"
216 + }],
217 + tokenize=False,
218 + add_generation_prompt=True)
219 + for question in questions
220 + ]
221
222 # Stop tokens for InternVL
223 # models variants may have different stop tokens
@@ -209,71 +225,82 @@ def run_internvl(question: str, modality: str):
225 # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
226 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
227 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
212 - return llm, prompt, stop_token_ids
228 + return llm, prompts, stop_token_ids
229
230
231 # LLaVA-1.5
216 -def run_llava(question: str, modality: str):
232 +def run_llava(questions: list[str], modality: str):
233 assert modality == "image"
234
219 - prompt = f"USER: <image>\n{question}\nASSISTANT:"
235 + prompts = [
236 + f"USER: <image>\n{question}\nASSISTANT:" for question in questions
237 + ]
238
239 llm = LLM(model="llava-hf/llava-1.5-7b-hf",
240 max_model_len=4096,
241 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
242 stop_token_ids = None
225 - return llm, prompt, stop_token_ids
243 + return llm, prompts, stop_token_ids
244
245
246 # LLaVA-1.6/LLaVA-NeXT
229 -def run_llava_next(question: str, modality: str):
247 +def run_llava_next(questions: list[str], modality: str):
248 assert modality == "image"
249
232 - prompt = f"[INST] <image>\n{question} [/INST]"
250 + prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
251 llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf",
252 max_model_len=8192,
253 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
254 stop_token_ids = None
237 - return llm, prompt, stop_token_ids
255 + return llm, prompts, stop_token_ids
256
257
258 # LlaVA-NeXT-Video
259 # Currently only support for video input
242 -def run_llava_next_video(question: str, modality: str):
260 +def run_llava_next_video(questions: list[str], modality: str):
261 assert modality == "video"
262
245 - prompt = f"USER: <video>\n{question} ASSISTANT:"
263 + prompts = [
264 + f"USER: <video>\n{question} ASSISTANT:" for question in questions
265 + ]
266 llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf",
267 max_model_len=8192,
268 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
269 stop_token_ids = None
250 - return llm, prompt, stop_token_ids
270 + return llm, prompts, stop_token_ids
271
272
273 # LLaVA-OneVision
254 -def run_llava_onevision(question: str, modality: str):
274 +def run_llava_onevision(questions: list[str], modality: str):
275
276 if modality == "video":
257 - prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
258 - <|im_start|>assistant\n"
277 + prompts = [
278 + f"<|im_start|>user <video>\n{question}<|im_end|> \
279 + <|im_start|>assistant\n" for question in questions
280 + ]
281
282 elif modality == "image":
261 - prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
262 - <|im_start|>assistant\n"
283 + prompts = [
284 + f"<|im_start|>user <image>\n{question}<|im_end|> \
285 + <|im_start|>assistant\n" for question in questions
286 + ]
287
288 llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
289 max_model_len=16384,
290 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
291 stop_token_ids = None
268 - return llm, prompt, stop_token_ids
292 + return llm, prompts, stop_token_ids
293
294
295 # Mantis
272 -def run_mantis(question: str, modality: str):
296 +def run_mantis(questions: list[str], modality: str):
297 assert modality == "image"
298
299 llama3_template = '<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' # noqa: E501
276 - prompt = llama3_template.format(f"{question}\n<image>")
300 + prompts = [
301 + llama3_template.format(f"{question}\n<image>")
302 + for question in questions
303 + ]
304
305 llm = LLM(
306 model="TIGER-Lab/Mantis-8B-siglip-llama3",
@@ -282,11 +309,11 @@ def run_mantis(question: str, modality: str):
309 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
310 )
311 stop_token_ids = [128009]
285 - return llm, prompt, stop_token_ids
312 + return llm, prompts, stop_token_ids
313
314
315 # MiniCPM-V
289 -def run_minicpmv_base(question: str, modality: str, model_name):
316 +def run_minicpmv_base(questions: list[str], modality: str, model_name):
317 assert modality in ["image", "video"]
318 # If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language.py` # noqa
319
@@ -333,26 +360,28 @@ def run_minicpmv_base(question: str, modality: str, model_name):
360 "video": "(<video>./</video>)",
361 }
362
336 - messages = [{
337 - 'role': 'user',
338 - 'content': f'{modality_placeholder[modality]}\n{question}'
339 - }]
340 - prompt = tokenizer.apply_chat_template(messages,
341 - tokenize=False,
342 - add_generation_prompt=True)
343 - return llm, prompt, stop_token_ids
363 + prompts = [
364 + tokenizer.apply_chat_template(
365 + [{
366 + 'role': 'user',
367 + 'content': f"{modality_placeholder[modality]}\n{question}"
368 + }],
369 + tokenize=False,
370 + add_generation_prompt=True) for question in questions
371 + ]
372 + return llm, prompts, stop_token_ids
373
374
346 -def run_minicpmo(question: str, modality: str):
347 - return run_minicpmv_base(question, modality, "openbmb/MiniCPM-o-2_6")
375 +def run_minicpmo(questions: list[str], modality: str):
376 + return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
377
378
350 -def run_minicpmv(question: str, modality: str):
351 - return run_minicpmv_base(question, modality, "openbmb/MiniCPM-V-2_6")
379 +def run_minicpmv(questions: list[str], modality: str):
380 + return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
381
382
383 # LLama 3.2
355 -def run_mllama(question: str, modality: str):
384 +def run_mllama(questions: list[str], modality: str):
385 assert modality == "image"
386
387 model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
@@ -379,16 +408,16 @@ def run_mllama(question: str, modality: str):
408 "type": "text",
409 "text": f"{question}"
410 }]
382 - }]
383 - prompt = tokenizer.apply_chat_template(messages,
384 - add_generation_prompt=True,
385 - tokenize=False)
411 + } for question in questions]
412 + prompts = tokenizer.apply_chat_template(messages,
413 + add_generation_prompt=True,
414 + tokenize=False)
415 stop_token_ids = None
387 - return llm, prompt, stop_token_ids
416 + return llm, prompts, stop_token_ids
417
418
419 # Molmo
391 -def run_molmo(question, modality):
420 +def run_molmo(questions: list[str], modality: str):
421 assert modality == "image"
422
423 model_name = "allenai/Molmo-7B-D-0924"
@@ -400,13 +429,16 @@ def run_molmo(question, modality):
429 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
430 )
431
403 - prompt = question
432 + prompts = [
433 + f"<|im_start|>user <image>\n{question}<|im_end|> \
434 + <|im_start|>assistant\n" for question in questions
435 + ]
436 stop_token_ids = None
405 - return llm, prompt, stop_token_ids
437 + return llm, prompts, stop_token_ids
438
439
440 # NVLM-D
409 -def run_nvlm_d(question: str, modality: str):
441 +def run_nvlm_d(questions: list[str], modality: str):
442 assert modality == "image"
443
444 model_name = "nvidia/NVLM-D-72B"
@@ -422,12 +454,15 @@ def run_nvlm_d(question: str, modality: str):
454
455 tokenizer = AutoTokenizer.from_pretrained(model_name,
456 trust_remote_code=True)
425 - messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
426 - prompt = tokenizer.apply_chat_template(messages,
427 - tokenize=False,
428 - add_generation_prompt=True)
457 + messages = [{
458 + 'role': 'user',
459 + 'content': f"<image>\n{question}"
460 + } for question in questions]
461 + prompts = tokenizer.apply_chat_template(messages,
462 + tokenize=False,
463 + add_generation_prompt=True)
464 stop_token_ids = None
430 - return llm, prompt, stop_token_ids
465 + return llm, prompts, stop_token_ids
466
467
468 # PaliGemma
@@ -435,7 +470,7 @@ def run_paligemma(question: str, modality: str):
470 assert modality == "image"
471
472 # PaliGemma has special prompt format for VQA
438 - prompt = "caption en"
473 + prompt = ["caption en"]
474 llm = LLM(model="google/paligemma-3b-mix-224",
475 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
476 stop_token_ids = None
@@ -447,7 +482,7 @@ def run_paligemma2(question: str, modality: str):
482 assert modality == "image"
483
484 # PaliGemma 2 has special prompt format for VQA
450 - prompt = "caption en"
485 + prompt = ["caption en"]
486 llm = LLM(model="google/paligemma2-3b-ft-docci-448",
487 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
488 stop_token_ids = None
@@ -455,10 +490,13 @@ def run_paligemma2(question: str, modality: str):
490
491
492 # Phi-3-Vision
458 -def run_phi3v(question: str, modality: str):
493 +def run_phi3v(questions: list[str], modality: str):
494 assert modality == "image"
495
461 - prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
496 + prompts = [
497 + f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n"
498 + for question in questions
499 + ]
500
501 # num_crops is an override kwarg to the multimodal image processor;
502 # For some models, e.g., Phi-3.5-vision-instruct, it is recommended
@@ -482,11 +520,11 @@ def run_phi3v(question: str, modality: str):
520 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
521 )
522 stop_token_ids = None
485 - return llm, prompt, stop_token_ids
523 + return llm, prompts, stop_token_ids
524
525
526 # Pixtral HF-format
489 -def run_pixtral_hf(question: str, modality: str):
527 +def run_pixtral_hf(questions: list[str], modality: str):
528 assert modality == "image"
529
530 model_name = "mistral-community/pixtral-12b"
@@ -499,13 +537,13 @@ def run_pixtral_hf(question: str, modality: str):
537 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
538 )
539
502 - prompt = f"<s>[INST]{question}\n[IMG][/INST]"
540 + prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
541 stop_token_ids = None
504 - return llm, prompt, stop_token_ids
542 + return llm, prompts, stop_token_ids
543
544
545 # Qwen
508 -def run_qwen_vl(question: str, modality: str):
546 +def run_qwen_vl(questions: list[str], modality: str):
547 assert modality == "image"
548
549 llm = LLM(
@@ -517,13 +555,13 @@ def run_qwen_vl(question: str, modality: str):
555 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
556 )
557
520 - prompt = f"{question}Picture 1: <img></img>\n"
558 + prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]
559 stop_token_ids = None
522 - return llm, prompt, stop_token_ids
560 + return llm, prompts, stop_token_ids
561
562
563 # Qwen2-VL
526 -def run_qwen2_vl(question: str, modality: str):
564 +def run_qwen2_vl(questions: list[str], modality: str):
565
566 model_name = "Qwen/Qwen2-VL-7B-Instruct"
567
@@ -544,16 +582,18 @@ def run_qwen2_vl(question: str, modality: str):
582 elif modality == "video":
583 placeholder = "<|video_pad|>"
584
547 - prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
548 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
549 - f"{question}<|im_end|>\n"
550 - "<|im_start|>assistant\n")
585 + prompts = [
586 + ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
587 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
588 + f"{question}<|im_end|>\n"
589 + "<|im_start|>assistant\n") for question in questions
590 + ]
591 stop_token_ids = None
552 - return llm, prompt, stop_token_ids
592 + return llm, prompts, stop_token_ids
593
594
595 # Qwen2.5-VL
556 -def run_qwen2_5_vl(question: str, modality: str):
596 +def run_qwen2_5_vl(questions: list[str], modality: str):
597
598 model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
599
@@ -574,12 +614,14 @@ def run_qwen2_5_vl(question: str, modality: str):
614 elif modality == "video":
615 placeholder = "<|video_pad|>"
616
577 - prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
578 - f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
579 - f"{question}<|im_end|>\n"
580 - "<|im_start|>assistant\n")
617 + prompts = [
618 + ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
619 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
620 + f"{question}<|im_end|>\n"
621 + "<|im_start|>assistant\n") for question in questions
622 + ]
623 stop_token_ids = None
582 - return llm, prompt, stop_token_ids
624 + return llm, prompts, stop_token_ids
625
626
627 model_example_map = {
@@ -624,29 +666,35 @@ def get_multi_modal_input(args):
666 # Input image and question
667 image = ImageAsset("cherry_blossom") \
668 .pil_image.convert("RGB")
627 - img_question = "What is the content of this image?"
669 + img_questions = [
670 + "What is the content of this image?",
671 + "Describe the content of this image in detail.",
672 + "What's in the image?",
673 + "Where is this image taken?",
674 + ]
675
676 return {
677 "data": image,
631 - "question": img_question,
678 + "questions": img_questions,
679 }
680
681 if args.modality == "video":
682 # Input video and question
683 video = VideoAsset(name="sample_demo_1.mp4",
684 num_frames=args.num_frames).np_ndarrays
638 - vid_question = "Why is this video funny?"
685 + vid_questions = ["Why is this video funny?"]
686
687 return {
688 "data": video,
642 - "question": vid_question,
689 + "questions": vid_questions,
690 }
691
692 msg = f"Modality {args.modality} is not supported."
693 raise ValueError(msg)
694
695
649 -def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
696 +def apply_image_repeat(image_repeat_prob, num_prompts, data,
697 + prompts: list[str], modality):
698 """Repeats images with provided probability of "image_repeat_prob".
699 Used to simulate hit/miss for the MM preprocessor cache.
700 """
@@ -666,7 +714,7 @@ def apply_image_repeat(image_repeat_prob, num_prompts, data, prompt, modality):
714 cur_image.putpixel((0, 0), new_val)
715
716 inputs.append({
669 - "prompt": prompt,
717 + "prompt": prompts[i % len(prompts)],
718 "multi_modal_data": {
719 modality: cur_image
720 }
@@ -683,9 +731,14 @@ def main(args):
731 modality = args.modality
732 mm_input = get_multi_modal_input(args)
733 data = mm_input["data"]
686 - question = mm_input["question"]
734 + questions = mm_input["questions"]
735
688 - llm, prompt, stop_token_ids = model_example_map[model](question, modality)
736 + llm, prompts, stop_token_ids = model_example_map[model](questions,
737 + modality)
738 + # Don't want to check the flag multiple times, so just hijack `prompts`.
739 + prompts = prompts if args.use_different_prompt_per_request else [
740 + prompts[0]
741 + ]
742
743 # We set temperature to 0.2 so that outputs can be different
744 # even when all prompts are identical when running batch inference.
@@ -697,27 +750,26 @@ def main(args):
750 if args.num_prompts == 1:
751 # Single inference
752 inputs = {
700 - "prompt": prompt,
753 + "prompt": prompts[0],
754 "multi_modal_data": {
755 modality: data
756 },
757 }
705 -
758 else:
759 # Batch inference
760 if args.image_repeat_prob is not None:
761 # Repeat images with specified probability of "image_repeat_prob"
762 inputs = apply_image_repeat(args.image_repeat_prob,
711 - args.num_prompts, data, prompt,
763 + args.num_prompts, data, prompts,
764 modality)
765 else:
766 # Use the same image for all prompts
767 inputs = [{
716 - "prompt": prompt,
768 + "prompt": prompts[i % len(prompts)],
769 "multi_modal_data": {
770 modality: data
771 },
720 - } for _ in range(args.num_prompts)]
772 + } for i in range(args.num_prompts)]
773
774 if args.time_generate:
775 import time
@@ -775,5 +827,11 @@ if __name__ == "__main__":
827 action='store_true',
828 help='If True, then print the total generate() call time')
829
830 + parser.add_argument(
831 + '--use-different-prompt-per-request',
832 + action='store_true',
833 + help='If True, then use different prompt (with the same multi-modal '
834 + 'data) for each request.')
835 +
836 args = parser.parse_args()
837 main(args)
vllm/model_executor/models/aria.py
+3 -1
@@ -602,7 +602,9 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
602
603 return self.multi_modal_projector(image_outputs, image_attn_mask)
604
605 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
605 + def get_multimodal_embeddings(
606 + self, **kwargs
607 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
608 image_input = self._parse_and_validate_image_input(**kwargs)
609 if image_input is None:
610 return None
vllm/model_executor/models/blip2.py
+3 -1
@@ -628,7 +628,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
628
629 return self.language_projection(query_output)
630
631 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
631 + def get_multimodal_embeddings(
632 + self, **kwargs
633 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
634 image_input = self._parse_and_validate_image_input(**kwargs)
635 if image_input is None:
636 return None
vllm/model_executor/models/chameleon.py
+3 -1
@@ -986,7 +986,9 @@ class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
986 data=self._validate_pixel_values(pixel_values),
987 )
988
989 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
989 + def get_multimodal_embeddings(
990 + self, **kwargs
991 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
992 image_input = self._parse_and_validate_image_input(**kwargs)
993 if image_input is None:
994 return None
vllm/model_executor/models/deepseek_vl2.py
+3 -1
@@ -606,7 +606,9 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
606 return self._pixel_values_to_embedding(
607 pixel_values=pixel_values, images_spatial_crop=images_spatial_crop)
608
609 - def get_multimodal_embeddings(self, **kwargs: object) -> torch.Tensor:
609 + def get_multimodal_embeddings(
610 + self, **kwargs: object
611 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
612 image_input = self._parse_and_validate_image_input(**kwargs)
613 if image_input is None:
614 return None
vllm/model_executor/models/florence2.py
+3 -1
@@ -1037,7 +1037,9 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal):
1037 pixel_values = image_input["data"]
1038 return self._encode_image(pixel_values)
1039
1040 - def get_multimodal_embeddings(self, **kwargs: object) -> torch.Tensor:
1040 + def get_multimodal_embeddings(
1041 + self, **kwargs: object
1042 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
1043 image_input = self._parse_and_validate_image_input(**kwargs)
1044 if image_input is None:
1045 return None
vllm/model_executor/models/fuyu.py
+4 -2
@@ -18,7 +18,7 @@
18 """ PyTorch Fuyu model."""
19 import math
20 from collections.abc import Iterable, Mapping, Sequence
21 -from typing import List, Literal, Optional, Set, Tuple, TypedDict
21 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
22
23 import torch
24 import torch.nn as nn
@@ -327,7 +327,9 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
327 image_patches_flat)
328 return vision_embeddings_flat.split(patches_per_image, dim=0)
329
330 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
330 + def get_multimodal_embeddings(
331 + self, **kwargs
332 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
333 image_input = self._parse_and_validate_image_input(**kwargs)
334 if image_input is None:
335 return None
vllm/model_executor/models/glm4v.py
+3 -1
@@ -595,7 +595,9 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
595
596 return self.transformer.vision(pixel_values)
597
598 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
598 + def get_multimodal_embeddings(
599 + self, **kwargs
600 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
601 image_input = self._parse_and_validate_image_input(**kwargs)
602 if image_input is None:
603 return None
vllm/model_executor/models/idefics3.py
+3 -1
@@ -617,7 +617,9 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
617 self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
618 self.sampler = get_sampler()
619
620 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
620 + def get_multimodal_embeddings(
621 + self, **kwargs
622 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
623 image_input = self.model._parse_and_validate_image_input(**kwargs)
624 if image_input is None:
625 return None
vllm/model_executor/models/interfaces.py
+9 -9
@@ -4,6 +4,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,
4 Protocol, Type, Union, overload, runtime_checkable)
5
6 import torch
7 +from torch import Tensor
8 from typing_extensions import TypeIs, TypeVar
9
10 from vllm.logger import init_logger
@@ -15,12 +16,11 @@ from .interfaces_base import is_pooling_model
16
17 if TYPE_CHECKING:
18 from vllm.attention import AttentionMetadata
18 - from vllm.multimodal.inputs import NestedTensors # noqa: F401
19 from vllm.sequence import IntermediateTensors
20
21 logger = init_logger(__name__)
22
23 -T = TypeVar("T", default="NestedTensors")
23 +T = TypeVar("T", default=Union[list[Tensor], Tensor, tuple[Tensor, ...]])
24
25
26 @runtime_checkable
@@ -36,7 +36,7 @@ class SupportsMultiModal(Protocol):
36 MRO of your model class.
37 """
38
39 - def get_multimodal_embeddings(self, **kwargs) -> Optional[T]:
39 + def get_multimodal_embeddings(self, **kwargs) -> T:
40 """
41 Returns multimodal embeddings generated from multimodal kwargs
42 to be merged with text embeddings.
@@ -59,18 +59,18 @@ class SupportsMultiModal(Protocol):
59 @overload
60 def get_input_embeddings(
61 self,
62 - input_ids: torch.Tensor,
62 + input_ids: Tensor,
63 multimodal_embeddings: Optional[T] = None,
64 attn_metadata: Optional["AttentionMetadata"] = None,
65 - ) -> torch.Tensor:
65 + ) -> Tensor:
66 ...
67
68 @overload
69 def get_input_embeddings(
70 self,
71 - input_ids: torch.Tensor,
71 + input_ids: Tensor,
72 multimodal_embeddings: Optional[T] = None,
73 - ) -> torch.Tensor:
73 + ) -> Tensor:
74 """
75 Returns the input embeddings merged from the text embeddings from
76 input_ids and the multimodal embeddings generated from multimodal
@@ -210,7 +210,7 @@ class SupportsPP(Protocol):
210 self,
211 *,
212 intermediate_tensors: Optional["IntermediateTensors"],
213 - ) -> Union[torch.Tensor, "IntermediateTensors"]:
213 + ) -> Union[Tensor, "IntermediateTensors"]:
214 """
215 Accept :class:`IntermediateTensors` when PP rank > 0.
216
@@ -237,7 +237,7 @@ class _SupportsPPType(Protocol):
237 self,
238 *,
239 intermediate_tensors: Optional["IntermediateTensors"],
240 - ) -> Union[torch.Tensor, "IntermediateTensors"]:
240 + ) -> Union[Tensor, "IntermediateTensors"]:
241 ...
242
243
vllm/model_executor/models/internvl.py
+3 -1
@@ -904,7 +904,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
904 else:
905 self.visual_token_mask = None
906
907 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
907 + def get_multimodal_embeddings(
908 + self, **kwargs
909 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
910 image_input = self._parse_and_validate_image_input(**kwargs)
911 if image_input is None:
912 return None
vllm/model_executor/models/llava.py
+3 -1
@@ -635,7 +635,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
635 image_features = self._process_image_pixels(image_input)
636 return self.multi_modal_projector(image_features)
637
638 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
638 + def get_multimodal_embeddings(
639 + self, **kwargs
640 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
641 image_input = self._parse_and_validate_image_input(**kwargs)
642 if image_input is None:
643 return None
vllm/model_executor/models/llava_next.py
+3 -1
@@ -479,7 +479,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
479 for i, patch_features_batch in enumerate(patch_embeddings)
480 ]
481
482 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
482 + def get_multimodal_embeddings(
483 + self, **kwargs
484 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
485 image_input = self._parse_and_validate_image_input(**kwargs)
486 if image_input is None:
487 return None
vllm/model_executor/models/llava_next_video.py
+3 -1
@@ -420,7 +420,9 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
420 raise ValueError(
421 f"Unsupported type of video input {type(video_pixels)}")
422
423 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
423 + def get_multimodal_embeddings(
424 + self, **kwargs
425 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
426 video_input = self._parse_and_validate_video_input(**kwargs)
427 if video_input is None:
428 return None
vllm/model_executor/models/molmo.py
+6 -3
@@ -50,7 +50,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
50 PromptInsertion, PromptUpdate)
51 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52 from vllm.sequence import IntermediateTensors
53 -from vllm.utils import JSONTree, json_map_leaves
53 +from vllm.utils import JSONTree, flatten_2d_lists, json_map_leaves
54
55 from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,
56 SupportsQuant)
@@ -1576,14 +1576,16 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1576
1577 return embeds_in_batch
1578
1579 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
1579 + def get_multimodal_embeddings(
1580 + self, **kwargs
1581 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
1582 image_input = self._parse_and_validate_image_input(**kwargs)
1583 if image_input is None:
1584 return None
1585
1586 image_features = self._process_image_input(image_input)
1587
1586 - return [
1588 + nested_embeds = [
1589 self._get_mm_embeds(*args) for args in zip(
1590 image_features,
1591 image_input["feat_is_patch"],
@@ -1591,6 +1593,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1593 image_input["embed_is_patch"],
1594 )
1595 ]
1596 + return flatten_2d_lists(nested_embeds)
1597
1598 def get_input_embeddings(
1599 self,
vllm/model_executor/models/paligemma.py
+3 -1
@@ -263,7 +263,9 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
263
264 return self.multi_modal_projector(image_features)
265
266 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
266 + def get_multimodal_embeddings(
267 + self, **kwargs
268 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
269 image_input = self._parse_and_validate_image_input(**kwargs)
270 if image_input is None:
271 return None
vllm/model_executor/models/phi3v.py
+3 -1
@@ -648,7 +648,9 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
648
649 return image_embeds
650
651 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
651 + def get_multimodal_embeddings(
652 + self, **kwargs
653 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
654 image_input = self._parse_and_validate_image_input(**kwargs)
655 if image_input is None:
656 return None
vllm/model_executor/models/pixtral.py
+3 -1
@@ -220,7 +220,9 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
220
221 return get_sampler()
222
223 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
223 + def get_multimodal_embeddings(
224 + self, **kwargs
225 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
226 image_input, image_tokens = self._parse_and_validate_image_input(
227 **kwargs)
228 if image_input is None:
vllm/model_executor/models/qwen2_audio.py
+3 -1
@@ -356,7 +356,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
356 return torch.split(masked_audio_features,
357 audio_output_lengths.flatten().tolist())
358
359 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
359 + def get_multimodal_embeddings(
360 + self, **kwargs
361 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
362 audio_input = self._parse_and_validate_audio_input(**kwargs)
363 if audio_input is None:
364 return None
vllm/model_executor/models/qwen_vl.py
+3 -1
@@ -740,7 +740,9 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,
740
741 return self.transformer.visual(image_input["data"])
742
743 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
743 + def get_multimodal_embeddings(
744 + self, **kwargs
745 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
746 image_input = self._parse_and_validate_image_input(**kwargs)
747 if image_input is None:
748 return None
vllm/model_executor/models/ultravox.py
+3 -1
@@ -476,7 +476,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
476
477 return result
478
479 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
479 + def get_multimodal_embeddings(
480 + self, **kwargs
481 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
482 audio_input = self._parse_and_validate_audio_input(**kwargs)
483 if audio_input is None:
484 return None
vllm/model_executor/models/whisper.py
+3 -1
@@ -692,7 +692,9 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
692 )
693 return decoder_outputs
694
695 - def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
695 + def get_multimodal_embeddings(
696 + self, **kwargs
697 + ) -> Union[list[torch.Tensor], torch.Tensor, tuple[torch.Tensor, ...]]:
698 # TODO: This method does not obey the interface for SupportsMultiModal.
699 # Refactor this once encoder/decoder support is implemented in V1.
700 audio_input = self._parse_and_validate_audio_input(**kwargs)