vision_language_offline.py · tduncan/Nvidia-Optimization-Stack-VLLM

@tduncan / Nvidia-Optimization-Stack-VLLM

Code Commits Issues Pulls Cloud Agent CI/CD

main

py 2,669 lines 76.5 KB

Raw

1	# SPDX-License-Identifier: Apache-2.0
2	# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3	"""This example shows how to use vLLM for running offline inference with
4	the correct prompt format on vision language models for text generation.
5
6	For most models, the prompt format should follow corresponding examples
7	on HuggingFace model repository.
8	"""
9
10	import os
11	import random
12	from contextlib import contextmanager
13	from typing import NamedTuple
14
15	from huggingface_hub import snapshot_download
16	from transformers import AutoProcessor, AutoTokenizer
17
18	from vllm import LLM, EngineArgs, SamplingParams
19	from vllm.assets.image import ImageAsset
20	from vllm.assets.video import VideoAsset
21	from vllm.lora.request import LoRARequest
22	from vllm.multimodal.image import convert_image_mode
23	from vllm.platforms import current_platform
24	from vllm.utils.argparse_utils import FlexibleArgumentParser
25
26
27	class ModelRequestData(NamedTuple):
28	engine_args: EngineArgs
29	prompts: list[str]
30	stop_token_ids: list[int] \| None = None
31	lora_requests: list[LoRARequest] \| None = None
32	sampling_params: list[SamplingParams] \| None = None
33
34
35	# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
36	# lower-end GPUs.
37	# Unless specified, these settings have been tested to work on a single L4.
38
39
40	# Aria
41	def run_aria(questions: list[str], modality: str) -> ModelRequestData:
42	assert modality == "image"
43	model_name = "rhymes-ai/Aria"
44
45	# NOTE: Need L40 (or equivalent) to avoid OOM
46	engine_args = EngineArgs(
47	model=model_name,
48	max_model_len=4096,
49	max_num_seqs=2,
50	dtype="bfloat16",
51	limit_mm_per_prompt={modality: 1},
52	)
53
54	prompts = [
55	(
56	f"<\|im_start\|>user\n<fim_prefix><\|img\|><fim_suffix>{question}"
57	"<\|im_end\|>\n<\|im_start\|>assistant\n"
58	)
59	for question in questions
60	]
61
62	stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
63
64	return ModelRequestData(
65	engine_args=engine_args,
66	prompts=prompts,
67	stop_token_ids=stop_token_ids,
68	)
69
70
71	# Bee-8B
72	def run_bee(questions: list[str], modality: str) -> ModelRequestData:
73	assert modality == "image"
74	model_name = "Open-Bee/Bee-8B-RL"
75
76	prompts = [
77	(
78	f"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
79	f"<\|im_start\|>user\n<image>\n{question}<\|im_end\|>"
80	f"<\|im_start\|>assistant\n<think>\n"
81	)
82	for question in questions
83	]
84
85	engine_args = EngineArgs(
86	model=model_name,
87	max_model_len=16384,
88	limit_mm_per_prompt={modality: 1},
89	trust_remote_code=True,
90	)
91
92	return ModelRequestData(
93	engine_args=engine_args,
94	prompts=prompts,
95	)
96
97
98	def run_bagel(questions: list[str], modality: str) -> ModelRequestData:
99	assert modality == "image"
100	model_name = "ByteDance-Seed/BAGEL-7B-MoT"
101
102	engine_args = EngineArgs(
103	model=model_name,
104	trust_remote_code=True,
105	max_model_len=8192,
106	max_num_seqs=2,
107	limit_mm_per_prompt={modality: 1},
108	)
109
110	prompts = [
111	(
112	f"<\|im_start\|>user\n<\|image_pad\|>\n{question}<\|im_end\|>\n"
113	f"<\|im_start\|>assistant\n"
114	)
115	for question in questions
116	]
117
118	return ModelRequestData(
119	engine_args=engine_args,
120	prompts=prompts,
121	)
122
123
124	# BLIP-2
125	def run_blip2(questions: list[str], modality: str) -> ModelRequestData:
126	assert modality == "image"
127
128	# BLIP-2 prompt format is inaccurate on HuggingFace model repository.
129	# See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
130	prompts = [f"Question: {question} Answer:" for question in questions]
131	engine_args = EngineArgs(
132	model="Salesforce/blip2-opt-2.7b",
133	limit_mm_per_prompt={modality: 1},
134	)
135
136	return ModelRequestData(
137	engine_args=engine_args,
138	prompts=prompts,
139	)
140
141
142	def run_command_a_vision(questions: list[str], modality: str) -> ModelRequestData:
143	assert modality == "image"
144
145	model_name = "CohereLabs/command-a-vision-07-2025"
146
147	engine_args = EngineArgs(
148	model=model_name,
149	max_model_len=32768,
150	tensor_parallel_size=4,
151	limit_mm_per_prompt={modality: 1},
152	)
153
154	prompts = [
155	f"<\|START_OF_TURN_TOKEN\|><\|USER_TOKEN\|><\|IMG_PATCH\|>{question}<\|END_OF_TURN_TOKEN\|><\|START_OF_TURN_TOKEN\|><\|CHATBOT_TOKEN\|>"
156	for question in questions
157	]
158
159	return ModelRequestData(
160	engine_args=engine_args,
161	prompts=prompts,
162	)
163
164
165	# Deepseek-VL2
166	def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:
167	assert modality == "image"
168
169	model_name = "deepseek-ai/deepseek-vl2-tiny"
170
171	engine_args = EngineArgs(
172	model=model_name,
173	max_model_len=4096,
174	max_num_seqs=2,
175	hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
176	limit_mm_per_prompt={modality: 1},
177	)
178
179	prompts = [
180	f"<\|User\|>: <image>\n{question}\n\n<\|Assistant\|>:" for question in questions
181	]
182
183	return ModelRequestData(
184	engine_args=engine_args,
185	prompts=prompts,
186	)
187
188
189	def run_deepseek_ocr(questions: list[str], modality: str) -> ModelRequestData:
190	from vllm.model_executor.models.deepseek_ocr import NGramPerReqLogitsProcessor
191
192	assert modality == "image"
193
194	model_name = "deepseek-ai/DeepSeek-OCR"
195
196	engine_args = EngineArgs(
197	model=model_name,
198	limit_mm_per_prompt={modality: 1},
199	logits_processors=[NGramPerReqLogitsProcessor],
200	)
201
202	# deepseek-ocr use plain prompt template
203	prompts = [f"<image>\n{question}" for question in questions]
204
205	# The following sampling params config is taken from
206	# the official Deepseek-OCR inference example.
207	# (IMPORTANT) Use the custom logits processor and avoid skipping
208	# special tokens for this model for the optimal OCR performance.
209	sampling_params = [
210	SamplingParams(
211	temperature=0.0,
212	max_tokens=8192,
213	# ngram logit processor args
214	extra_args=dict(
215	ngram_size=30,
216	window_size=90,
217	# whitelist: <td>, </td>
218	whitelist_token_ids={128821, 128822},
219	),
220	skip_special_tokens=False,
221	)
222	for _ in questions
223	]
224
225	return ModelRequestData(
226	engine_args=engine_args,
227	prompts=prompts,
228	sampling_params=sampling_params,
229	)
230
231
232	def run_deepseek_ocr2(questions: list[str], modality: str) -> ModelRequestData:
233	from vllm.model_executor.models.deepseek_ocr import NGramPerReqLogitsProcessor
234
235	assert modality == "image"
236
237	model_name = "deepseek-ai/DeepSeek-OCR-2"
238
239	engine_args = EngineArgs(
240	model=model_name,
241	limit_mm_per_prompt={modality: 1},
242	logits_processors=[NGramPerReqLogitsProcessor],
243	)
244
245	# deepseek-ocr use plain prompt template
246	prompts = [f"<image>\n{question}" for question in questions]
247
248	# The following sampling params config is taken from
249	# the official Deepseek-OCR inference example.
250	# (IMPORTANT) Use the custom logits processor and avoid skipping
251	# special tokens for this model for the optimal OCR performance.
252	sampling_params = [
253	SamplingParams(
254	temperature=0.0,
255	max_tokens=8192,
256	# ngram logit processor args
257	extra_args=dict(
258	ngram_size=30,
259	window_size=90,
260	# whitelist: <td>, </td>
261	whitelist_token_ids={128821, 128822},
262	),
263	skip_special_tokens=False,
264	)
265	for _ in questions
266	]
267
268	return ModelRequestData(
269	engine_args=engine_args,
270	prompts=prompts,
271	sampling_params=sampling_params,
272	)
273
274
275	# Dots-OCR
276	def run_dots_ocr(questions: list[str], modality: str) -> ModelRequestData:
277	assert modality == "image"
278
279	prompts = [f"<\|img\|><\|imgpad\|><\|endofimg\|>{question}" for question in questions]
280	engine_args = EngineArgs(
281	model="rednote-hilab/dots.ocr",
282	limit_mm_per_prompt={modality: 1},
283	trust_remote_code=True,
284	)
285
286	return ModelRequestData(
287	engine_args=engine_args,
288	prompts=prompts,
289	)
290
291
292	# Eagle2.5-VL
293	def run_eagle2_5(questions: list[str], modality: str) -> ModelRequestData:
294	assert modality == "image"
295
296	model_name = "nvidia/Eagle2.5-8B"
297
298	engine_args = EngineArgs(
299	model=model_name,
300	max_model_len=4096,
301	max_num_seqs=2,
302	trust_remote_code=True,
303	limit_mm_per_prompt={modality: 1},
304	)
305
306	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
307	messages = [
308	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
309	]
310	prompts = tokenizer.apply_chat_template(
311	messages, tokenize=False, add_generation_prompt=True
312	)
313
314	# Stop tokens for Eagle2.5 (Qwen2 based)
315	stop_tokens = ["<\|endoftext\|>", "<\|im_start\|>", "<\|im_end\|>"]
316	stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
317	stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
318
319	return ModelRequestData(
320	engine_args=engine_args,
321	prompts=prompts,
322	stop_token_ids=stop_token_ids,
323	)
324
325
326	# Ernie4.5-VL
327	def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:
328	model_name = "baidu/ERNIE-4.5-VL-28B-A3B-PT"
329
330	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
331	engine_args = EngineArgs(
332	model=model_name,
333	max_model_len=4096,
334	max_num_seqs=5,
335	limit_mm_per_prompt=mm_limit,
336	trust_remote_code=True,
337	)
338
339	image_placeholder = "Picture 1:<\|IMAGE_START\|><\|image@placeholder\|><\|IMAGE_END\|>"
340	video_placeholder = "Video 1:<\|VIDEO_START\|><\|video@placeholder\|><\|VIDEO_END\|>"
341
342	if modality == "image":
343	placeholder = image_placeholder
344	elif modality == "video":
345	placeholder = video_placeholder
346	elif modality == "image+video":
347	placeholder = image_placeholder + video_placeholder
348
349	prompts = [
350	(
351	f"<\|begin_of_sentence\|>User: {question}{placeholder}\n"
352	"Assistant: <think></think>"
353	)
354	for question in questions
355	]
356
357	return ModelRequestData(
358	engine_args=engine_args,
359	prompts=prompts,
360	)
361
362
363	# EXAONE-4.5
364	def run_exaone4_5(questions: list[str], modality: str) -> ModelRequestData:
365	model_name = "LGAI-EXAONE/EXAONE-4.5-33B"
366
367	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
368	engine_args = EngineArgs(
369	model=model_name,
370	max_model_len=4096,
371	max_num_seqs=5,
372	mm_processor_kwargs={
373	"min_pixels": 28 * 28,
374	"max_pixels": 1280 * 28 * 28,
375	"fps": 1,
376	},
377	limit_mm_per_prompt=mm_limit,
378	)
379
380	image_placeholder = "<vision><\|image_pad\|></vision>"
381	video_placeholder = "<vision><\|video_pad\|></vision>"
382
383	if modality == "image":
384	placeholder = image_placeholder
385	elif modality == "video":
386	placeholder = video_placeholder
387	elif modality == "image+video":
388	placeholder = image_placeholder + video_placeholder
389
390	prompts = [
391	(
392	"<\|system\|>\nYou are a helpful assistant.<\|endofturn\|>\n"
393	f"<\|user\|>\n{placeholder}"
394	f"{question}<\|endofturn\|>\n"
395	"<\|assistant\|>\n"
396	)
397	for question in questions
398	]
399
400	return ModelRequestData(
401	engine_args=engine_args,
402	prompts=prompts,
403	)
404
405
406	# Gemma 3
407	def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:
408	assert modality == "image"
409	model_name = "google/gemma-3-4b-it"
410
411	engine_args = EngineArgs(
412	model=model_name,
413	max_model_len=2048,
414	max_num_seqs=2,
415	mm_processor_kwargs={"do_pan_and_scan": True},
416	limit_mm_per_prompt={modality: 1},
417	)
418
419	prompts = [
420	(
421	"<bos><start_of_turn>user\n"
422	f"<start_of_image>{question}<end_of_turn>\n"
423	"<start_of_turn>model\n"
424	)
425	for question in questions
426	]
427	return ModelRequestData(
428	engine_args=engine_args,
429	prompts=prompts,
430	)
431
432
433	# Gemma3N
434	def run_gemma3n(questions: list[str], modality: str) -> ModelRequestData:
435	assert modality == "image"
436	model_name = "google/gemma-3n-E2B-it"
437
438	engine_args = EngineArgs(
439	model=model_name,
440	max_model_len=2048,
441	max_num_seqs=2,
442	limit_mm_per_prompt={modality: 1},
443	enforce_eager=True,
444	)
445
446	prompts = [
447	(
448	"<start_of_turn>user\n"
449	f"<image_soft_token>{question}<end_of_turn>\n"
450	"<start_of_turn>model\n"
451	)
452	for question in questions
453	]
454	return ModelRequestData(
455	engine_args=engine_args,
456	prompts=prompts,
457	)
458
459
460	# Gemma 4
461	def run_gemma4(questions: list[str], modality: str) -> ModelRequestData:
462	assert modality in ("image", "video")
463	model_name = "google/gemma-4-31B-it"
464
465	# NOTE: Gemma-4-31B is a large model. Users running into Out-Of-Memory (OOM)
466	# errors might need to set `tensor_parallel_size` to > 1.
467	engine_args = EngineArgs(
468	model=model_name,
469	max_model_len=4096,
470	max_num_seqs=2,
471	limit_mm_per_prompt={modality: 1},
472	)
473
474	if modality == "image":
475	prompts = [
476	(
477	"<bos><start_of_turn>user\n"
478	f"<\|image\|>\n{question}<end_of_turn>\n"
479	"<start_of_turn>model\n"
480	)
481	for question in questions
482	]
483	else: # video
484	prompts = [
485	(
486	"<bos><start_of_turn>user\n"
487	f"<\|video\|>\n{question}<end_of_turn>\n"
488	"<start_of_turn>model\n"
489	)
490	for question in questions
491	]
492
493	return ModelRequestData(
494	engine_args=engine_args,
495	prompts=prompts,
496	)
497
498
499	# GLM-4v
500	def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:
501	assert modality == "image"
502	model_name = "zai-org/glm-4v-9b"
503
504	engine_args = EngineArgs(
505	model=model_name,
506	max_model_len=2048,
507	max_num_seqs=2,
508	trust_remote_code=True,
509	enforce_eager=True,
510	hf_overrides={"architectures": ["GLM4VForCausalLM"]},
511	limit_mm_per_prompt={modality: 1},
512	)
513
514	prompts = [
515	(
516	"<\|user\|>\n<\|begin_of_image\|><\|endoftext\|><\|end_of_image\|>"
517	f"{question}<\|assistant\|>"
518	)
519	for question in questions
520	]
521
522	stop_token_ids = [151329, 151336, 151338]
523
524	return ModelRequestData(
525	engine_args=engine_args,
526	prompts=prompts,
527	stop_token_ids=stop_token_ids,
528	)
529
530
531	# GLM-4.1V
532	def run_glm4_1v(questions: list[str], modality: str) -> ModelRequestData:
533	model_name = "zai-org/GLM-4.1V-9B-Thinking"
534
535	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
536	engine_args = EngineArgs(
537	model=model_name,
538	max_model_len=4096,
539	max_num_seqs=2,
540	mm_processor_kwargs={
541	"size": {"shortest_edge": 12544, "longest_edge": 47040000},
542	"fps": 1,
543	},
544	limit_mm_per_prompt=mm_limit,
545	enforce_eager=True,
546	)
547
548	image_placeholder = "<\|begin_of_image\|><\|image\|><\|end_of_image\|>"
549	video_placeholder = "<\|begin_of_video\|><\|video\|><\|end_of_video\|>"
550
551	if modality == "image":
552	placeholder = image_placeholder
553	elif modality == "video":
554	placeholder = video_placeholder
555	elif modality == "image+video":
556	placeholder = image_placeholder + video_placeholder
557
558	prompts = [
559	(
560	"[gMASK]<sop><\|system\|>\nYou are a helpful assistant.<\|user\|>\n"
561	f"{placeholder}"
562	f"{question}<\|assistant\|>assistant\n"
563	)
564	for question in questions
565	]
566
567	return ModelRequestData(
568	engine_args=engine_args,
569	prompts=prompts,
570	)
571
572
573	# GLM-4.5V
574	def run_glm4_5v(questions: list[str], modality: str) -> ModelRequestData:
575	model_name = "zai-org/GLM-4.5V"
576
577	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
578	engine_args = EngineArgs(
579	model=model_name,
580	max_model_len=4096,
581	max_num_seqs=2,
582	mm_processor_kwargs={
583	"size": {"shortest_edge": 12544, "longest_edge": 47040000},
584	"fps": 1,
585	},
586	limit_mm_per_prompt=mm_limit,
587	enforce_eager=True,
588	tensor_parallel_size=4,
589	)
590
591	image_placeholder = "<\|begin_of_image\|><\|image\|><\|end_of_image\|>"
592	video_placeholder = "<\|begin_of_video\|><\|video\|><\|end_of_video\|>"
593
594	if modality == "image":
595	placeholder = image_placeholder
596	elif modality == "video":
597	placeholder = video_placeholder
598	elif modality == "image+video":
599	placeholder = image_placeholder + video_placeholder
600
601	prompts = [
602	(
603	"[gMASK]<sop><\|system\|>\nYou are a helpful assistant.<\|user\|>\n"
604	f"{placeholder}"
605	f"{question}<\|assistant\|>assistant\n"
606	)
607	for question in questions
608	]
609
610	return ModelRequestData(
611	engine_args=engine_args,
612	prompts=prompts,
613	)
614
615
616	# GLM-4.5V-FP8
617	def run_glm4_5v_fp8(questions: list[str], modality: str) -> ModelRequestData:
618	model_name = "zai-org/GLM-4.5V-FP8"
619
620	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
621	engine_args = EngineArgs(
622	model=model_name,
623	max_model_len=4096,
624	max_num_seqs=2,
625	mm_processor_kwargs={
626	"size": {"shortest_edge": 12544, "longest_edge": 47040000},
627	"fps": 1,
628	},
629	limit_mm_per_prompt=mm_limit,
630	enforce_eager=True,
631	tensor_parallel_size=4,
632	)
633
634	image_placeholder = "<\|begin_of_image\|><\|image\|><\|end_of_image\|>"
635	video_placeholder = "<\|begin_of_video\|><\|video\|><\|end_of_video\|>"
636
637	if modality == "image":
638	placeholder = image_placeholder
639	elif modality == "video":
640	placeholder = video_placeholder
641	elif modality == "image+video":
642	placeholder = image_placeholder + video_placeholder
643
644	prompts = [
645	(
646	"[gMASK]<sop><\|system\|>\nYou are a helpful assistant.<\|user\|>\n"
647	f"{placeholder}"
648	f"{question}<\|assistant\|>assistant\n"
649	)
650	for question in questions
651	]
652
653	return ModelRequestData(
654	engine_args=engine_args,
655	prompts=prompts,
656	)
657
658
659	# GLM-OCR
660	def run_glm_ocr(questions: list[str], modality: str) -> ModelRequestData:
661	model_name = "zai-org/GLM-OCR"
662
663	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
664	engine_args = EngineArgs(
665	model=model_name,
666	max_model_len=4096,
667	max_num_seqs=2,
668	mm_processor_kwargs={
669	"size": {"shortest_edge": 12544, "longest_edge": 47040000},
670	"fps": 1,
671	},
672	limit_mm_per_prompt=mm_limit,
673	enforce_eager=True,
674	)
675
676	image_placeholder = "<\|begin_of_image\|><\|image\|><\|end_of_image\|>"
677	video_placeholder = "<\|begin_of_video\|><\|video\|><\|end_of_video\|>"
678
679	if modality == "image":
680	placeholder = image_placeholder
681	elif modality == "video":
682	placeholder = video_placeholder
683	elif modality == "image+video":
684	placeholder = image_placeholder + video_placeholder
685
686	prompts = [
687	(
688	"[gMASK]<sop><\|system\|>\nYou are a helpful assistant.<\|user\|>\n"
689	f"{placeholder}"
690	f"{question}<\|assistant\|>assistant\n"
691	)
692	for question in questions
693	]
694
695	return ModelRequestData(
696	engine_args=engine_args,
697	prompts=prompts,
698	)
699
700
701	# H2OVL-Mississippi
702	def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:
703	assert modality == "image"
704
705	model_name = "h2oai/h2ovl-mississippi-800m"
706
707	engine_args = EngineArgs(
708	model=model_name,
709	trust_remote_code=True,
710	max_model_len=8192,
711	limit_mm_per_prompt={modality: 1},
712	)
713
714	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
715	messages = [
716	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
717	]
718	prompts = tokenizer.apply_chat_template(
719	messages, tokenize=False, add_generation_prompt=True
720	)
721
722	# Stop tokens for H2OVL-Mississippi
723	# https://huggingface.co/h2oai/h2ovl-mississippi-800m
724	stop_token_ids = [tokenizer.eos_token_id]
725
726	return ModelRequestData(
727	engine_args=engine_args,
728	prompts=prompts,
729	stop_token_ids=stop_token_ids,
730	)
731
732
733	# HunyuanOCR
734	def run_hunyuan_vl(questions: list[str], modality: str) -> ModelRequestData:
735	assert modality == "image"
736
737	model_name = "tencent/HunyuanOCR"
738
739	engine_args = EngineArgs(
740	model=model_name,
741	max_model_len=8192,
742	limit_mm_per_prompt={modality: 1},
743	)
744
745	placeholder = "<｜hy_place▁holder▁no▁100｜><｜hy_place▁holder▁no▁102｜><｜hy_place▁holder▁no▁101｜>" # noqa: E501
746	prompts = [
747	f"<｜hy_begin▁of▁sentence｜>{placeholder}{question}<｜hy_User｜>"
748	for question in questions
749	]
750
751	return ModelRequestData(
752	engine_args=engine_args,
753	prompts=prompts,
754	stop_token_ids=None,
755	)
756
757
758	# Idefics3-8B-Llama3
759	def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:
760	assert modality == "image"
761	model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
762
763	engine_args = EngineArgs(
764	model=model_name,
765	max_model_len=8192,
766	max_num_seqs=2,
767	enforce_eager=True,
768	# if you are running out of memory, you can reduce the "longest_edge".
769	# see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
770	mm_processor_kwargs={
771	"size": {"longest_edge": 3 * 364},
772	},
773	limit_mm_per_prompt={modality: 1},
774	)
775	prompts = [
776	(f"<\|begin_of_text\|>User:<image>{question}<end_of_utterance>\nAssistant:")
777	for question in questions
778	]
779
780	return ModelRequestData(
781	engine_args=engine_args,
782	prompts=prompts,
783	)
784
785
786	# Intern-S1
787	def run_interns1(questions: list[str], modality: str) -> ModelRequestData:
788	model_name = "internlm/Intern-S1-mini"
789
790	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
791	engine_args = EngineArgs(
792	model=model_name,
793	trust_remote_code=True,
794	max_model_len=8192,
795	max_num_seqs=2,
796	limit_mm_per_prompt=mm_limit,
797	enforce_eager=True,
798	)
799
800	image_placeholder = "<IMG_CONTEXT>"
801	video_placeholder = "<video>"
802
803	if modality == "image":
804	placeholder = image_placeholder
805	elif modality == "video":
806	placeholder = video_placeholder
807	elif modality == "image+video":
808	placeholder = image_placeholder + "\n" + video_placeholder
809
810	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
811	messages = [
812	[{"role": "user", "content": f"{placeholder}\n{question}"}]
813	for question in questions
814	]
815	prompts = tokenizer.apply_chat_template(
816	messages, tokenize=False, add_generation_prompt=True
817	)
818
819	return ModelRequestData(
820	engine_args=engine_args,
821	prompts=prompts,
822	)
823
824
825	# Intern-S1-Pro
826	def run_interns1_pro(questions: list[str], modality: str) -> ModelRequestData:
827	model_name = "internlm/Intern-S1-Pro"
828
829	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
830	engine_args = EngineArgs(
831	model=model_name,
832	trust_remote_code=True,
833	max_model_len=8192,
834	max_num_seqs=2,
835	limit_mm_per_prompt=mm_limit,
836	enforce_eager=True,
837	tensor_parallel_size=4,
838	)
839
840	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
841	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
842
843	if modality == "image":
844	placeholder = image_placeholder
845	elif modality == "video":
846	placeholder = video_placeholder
847	elif modality == "image+video":
848	placeholder = image_placeholder + video_placeholder
849
850	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
851	messages = [
852	[{"role": "user", "content": f"{placeholder}\n{question}"}]
853	for question in questions
854	]
855	prompts = tokenizer.apply_chat_template(
856	messages, tokenize=False, add_generation_prompt=True
857	)
858
859	return ModelRequestData(
860	engine_args=engine_args,
861	prompts=prompts,
862	)
863
864
865	# InternVL
866	def run_internvl(questions: list[str], modality: str) -> ModelRequestData:
867	model_name = "OpenGVLab/InternVL3-2B"
868
869	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
870	engine_args = EngineArgs(
871	model=model_name,
872	trust_remote_code=True,
873	max_model_len=8192,
874	limit_mm_per_prompt=mm_limit,
875	)
876
877	image_placeholder = "<image>"
878	video_placeholder = "<video>"
879
880	if modality == "image":
881	placeholder = image_placeholder
882	elif modality == "video":
883	placeholder = video_placeholder
884	elif modality == "image+video":
885	placeholder = image_placeholder + "\n" + video_placeholder
886
887	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
888	messages = [
889	[{"role": "user", "content": f"{placeholder}\n{question}"}]
890	for question in questions
891	]
892	prompts = tokenizer.apply_chat_template(
893	messages, tokenize=False, add_generation_prompt=True
894	)
895
896	# Stop tokens for InternVL
897	# models variants may have different stop tokens
898	# please refer to the model card for the correct "stop words":
899	# https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
900	stop_tokens = ["<\|endoftext\|>", "<\|im_start\|>", "<\|im_end\|>", "<\|end\|>"]
901	stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
902	stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
903
904	return ModelRequestData(
905	engine_args=engine_args,
906	prompts=prompts,
907	stop_token_ids=stop_token_ids,
908	)
909
910
911	# Kanana-V
912	def run_kanana_v(questions: list[str], modality: str) -> ModelRequestData:
913	assert modality == "image"
914
915	model_name = "kakaocorp/kanana-1.5-v-3b-instruct"
916
917	engine_args = EngineArgs(
918	model=model_name,
919	max_model_len=8192,
920	trust_remote_code=True,
921	limit_mm_per_prompt={modality: 1},
922	)
923
924	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
925	messages = [
926	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
927	]
928	prompts = tokenizer.apply_chat_template(
929	messages, tokenize=False, add_generation_prompt=True
930	)
931
932	return ModelRequestData(
933	engine_args=engine_args,
934	prompts=prompts,
935	)
936
937
938	# Keye-VL
939	def run_keye_vl(questions: list[str], modality: str) -> ModelRequestData:
940	model_name = "Kwai-Keye/Keye-VL-8B-Preview"
941
942	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
943	engine_args = EngineArgs(
944	model=model_name,
945	max_model_len=8192,
946	trust_remote_code=True,
947	limit_mm_per_prompt=mm_limit,
948	)
949
950	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
951	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
952
953	if modality == "image":
954	placeholder = image_placeholder
955	elif modality == "video":
956	placeholder = video_placeholder
957	elif modality == "image+video":
958	placeholder = image_placeholder + video_placeholder
959
960	prompts = [
961	(
962	f"<\|im_start\|>user\n{placeholder}"
963	f"{question}<\|im_end\|>\n"
964	"<\|im_start\|>assistant\n"
965	)
966	for question in questions
967	]
968
969	return ModelRequestData(
970	engine_args=engine_args,
971	prompts=prompts,
972	)
973
974
975	# Keye-VL-1.5
976	def run_keye_vl1_5(questions: list[str], modality: str) -> ModelRequestData:
977	model_name = "Kwai-Keye/Keye-VL-1.5-8B"
978
979	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
980	engine_args = EngineArgs(
981	model=model_name,
982	max_model_len=8192,
983	trust_remote_code=True,
984	limit_mm_per_prompt=mm_limit,
985	)
986
987	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
988	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
989
990	if modality == "image":
991	placeholder = image_placeholder
992	elif modality == "video":
993	placeholder = video_placeholder
994	elif modality == "image+video":
995	placeholder = image_placeholder + video_placeholder
996
997	prompts = [
998	(
999	f"<\|im_start\|>user\n{placeholder}"
1000	f"{question}<\|im_end\|>\n"
1001	"<\|im_start\|>assistant\n"
1002	)
1003	for question in questions
1004	]
1005
1006	return ModelRequestData(
1007	engine_args=engine_args,
1008	prompts=prompts,
1009	)
1010
1011
1012	# Kimi-VL
1013	def run_kimi_vl(questions: list[str], modality: str) -> ModelRequestData:
1014	assert modality == "image"
1015
1016	prompts = [
1017	"<\|im_user\|>user<\|im_middle\|><\|media_start\|>image<\|media_content\|>"
1018	f"<\|media_pad\|><\|media_end\|>{question}<\|im_end\|>"
1019	"<\|im_assistant\|>assistant<\|im_middle\|>"
1020	for question in questions
1021	]
1022
1023	engine_args = EngineArgs(
1024	model="moonshotai/Kimi-VL-A3B-Instruct",
1025	trust_remote_code=True,
1026	max_model_len=4096,
1027	limit_mm_per_prompt={modality: 1},
1028	)
1029
1030	return ModelRequestData(
1031	engine_args=engine_args,
1032	prompts=prompts,
1033	)
1034
1035
1036	# Kimi-VL
1037	def run_kimi_k25(questions: list[str], modality: str) -> ModelRequestData:
1038	assert modality == "vision_chunk"
1039
1040	prompts = [
1041	"<\|im_user\|>user<\|media_begin\|>image<\|media_content\|>"
1042	f"<\|media_pad\|><\|media_end\|>{question}<\|im_end\|>"
1043	"<\|im_assistant\|>assistant<\|im_middle\|>"
1044	for question in questions
1045	]
1046
1047	engine_args = EngineArgs(
1048	model="moonshotai/Kimi-K2.5",
1049	trust_remote_code=True,
1050	max_model_len=4096,
1051	limit_mm_per_prompt={modality: 1},
1052	tensor_parallel_size=4,
1053	)
1054
1055	return ModelRequestData(
1056	engine_args=engine_args,
1057	prompts=prompts,
1058	)
1059
1060
1061	# LightOnOCR
1062	def run_lightonocr(questions: list[str], modality: str) -> ModelRequestData:
1063	assert modality == "image"
1064
1065	prompts = [
1066	"<\|im_start\|>system<\|im_end\|>\n<\|im_start\|>user\n<\|image_pad\|><\|im_end\|>\n<\|im_start\|>assistant\n"
1067	for _ in questions
1068	]
1069
1070	engine_args = EngineArgs(
1071	model="lightonai/LightOnOCR-1B",
1072	limit_mm_per_prompt={modality: 1},
1073	)
1074
1075	return ModelRequestData(
1076	engine_args=engine_args,
1077	prompts=prompts,
1078	)
1079
1080
1081	def run_lfm2_vl(questions: list[str], modality: str) -> ModelRequestData:
1082	assert modality == "image"
1083
1084	model_name = "LiquidAI/LFM2-VL-450M"
1085
1086	engine_args = EngineArgs(
1087	model=model_name,
1088	max_model_len=4096,
1089	limit_mm_per_prompt={modality: 1},
1090	)
1091
1092	processor = AutoProcessor.from_pretrained(model_name)
1093	messages = [
1094	[
1095	{
1096	"role": "user",
1097	"content": [{"type": "image"}, {"type": "text", "text": question}],
1098	}
1099	]
1100	for question in questions
1101	]
1102	prompts = processor.apply_chat_template(
1103	messages, tokenize=False, add_generation_prompt=True
1104	)
1105
1106	return ModelRequestData(
1107	engine_args=engine_args,
1108	prompts=prompts,
1109	)
1110
1111
1112	def run_llama4(questions: list[str], modality: str) -> ModelRequestData:
1113	assert modality == "image"
1114
1115	model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"
1116
1117	engine_args = EngineArgs(
1118	model=model_name,
1119	max_model_len=8192,
1120	max_num_seqs=4,
1121	tensor_parallel_size=8,
1122	gpu_memory_utilization=0.4,
1123	limit_mm_per_prompt={modality: 1},
1124	)
1125
1126	tokenizer = AutoTokenizer.from_pretrained(model_name)
1127	messages = [
1128	[
1129	{
1130	"role": "user",
1131	"content": [{"type": "image"}, {"type": "text", "text": f"{question}"}],
1132	}
1133	]
1134	for question in questions
1135	]
1136	prompts = tokenizer.apply_chat_template(
1137	messages, add_generation_prompt=True, tokenize=False
1138	)
1139	stop_token_ids = None
1140	return ModelRequestData(
1141	engine_args=engine_args,
1142	prompts=prompts,
1143	stop_token_ids=stop_token_ids,
1144	)
1145
1146
1147	# LLaVA-1.5
1148	def run_llava(questions: list[str], modality: str) -> ModelRequestData:
1149	assert modality == "image"
1150
1151	prompts = [f"USER: <image>\n{question}\nASSISTANT:" for question in questions]
1152
1153	engine_args = EngineArgs(
1154	model="llava-hf/llava-1.5-7b-hf",
1155	max_model_len=4096,
1156	limit_mm_per_prompt={modality: 1},
1157	)
1158
1159	return ModelRequestData(
1160	engine_args=engine_args,
1161	prompts=prompts,
1162	)
1163
1164
1165	# LLaVA-1.6/LLaVA-NeXT
1166	def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:
1167	assert modality == "image"
1168
1169	prompts = [f"[INST] <image>\n{question} [/INST]" for question in questions]
1170	engine_args = EngineArgs(
1171	model="llava-hf/llava-v1.6-mistral-7b-hf",
1172	max_model_len=8192,
1173	limit_mm_per_prompt={modality: 1},
1174	)
1175
1176	return ModelRequestData(
1177	engine_args=engine_args,
1178	prompts=prompts,
1179	)
1180
1181
1182	# LlaVA-NeXT-Video
1183	# Currently only support for video input
1184	def run_llava_next_video(questions: list[str], modality: str) -> ModelRequestData:
1185	assert modality == "video"
1186
1187	prompts = [f"USER: <video>\n{question} ASSISTANT:" for question in questions]
1188	engine_args = EngineArgs(
1189	model="llava-hf/LLaVA-NeXT-Video-7B-hf",
1190	max_model_len=8192,
1191	max_num_seqs=2,
1192	limit_mm_per_prompt={modality: 1},
1193	)
1194
1195	return ModelRequestData(
1196	engine_args=engine_args,
1197	prompts=prompts,
1198	)
1199
1200
1201	# LLaVA-OneVision
1202	def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData:
1203	image_placeholder = "<image>"
1204	video_placeholder = "<video>"
1205
1206	if modality == "image":
1207	placeholder = image_placeholder
1208	elif modality == "video":
1209	placeholder = video_placeholder
1210	elif modality == "image+video":
1211	placeholder = image_placeholder + "\n" + video_placeholder
1212
1213	prompts = [
1214	(f"<\|im_start\|>user {placeholder}\n{question}<\|im_end\|><\|im_start\|>assistant\n")
1215	for question in questions
1216	]
1217
1218	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1219	engine_args = EngineArgs(
1220	model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
1221	max_model_len=16384,
1222	limit_mm_per_prompt=mm_limit,
1223	)
1224
1225	return ModelRequestData(
1226	engine_args=engine_args,
1227	prompts=prompts,
1228	)
1229
1230
1231	# MiniCPM-V
1232	def run_minicpmv_base(questions: list[str], modality: str, model_name):
1233	assert modality in ["image", "video", "image+video"]
1234	# If you want to use `MiniCPM-o-2_6` with audio inputs, check `audio_language_offline.py` # noqa
1235
1236	# 2.0
1237	# The official repo doesn't work yet, so we need to use a fork for now
1238	# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
1239	# model_name = "HwwwH/MiniCPM-V-2"
1240
1241	# 2.5
1242	# model_name = "openbmb/MiniCPM-Llama3-V-2_5"
1243
1244	# 2.6
1245	# model_name = "openbmb/MiniCPM-V-2_6"
1246	# o2.6
1247
1248	# modality supports
1249	# 2.0: image
1250	# 2.5: image
1251	# 2.6: image, video
1252	# o2.6: image, video, audio
1253	# model_name = "openbmb/MiniCPM-o-2_6"
1254	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1255	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1256	engine_args = EngineArgs(
1257	model=model_name,
1258	max_model_len=4096,
1259	max_num_seqs=2,
1260	trust_remote_code=True,
1261	limit_mm_per_prompt=mm_limit,
1262	)
1263	# NOTE The stop_token_ids are different for various versions of MiniCPM-V
1264	# 2.0
1265	# stop_token_ids = [tokenizer.eos_id]
1266
1267	# 2.5
1268	# stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
1269
1270	# 2.6 / o2.6
1271	stop_tokens = ["<\|im_end\|>", "<\|endoftext\|>"]
1272	stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1273
1274	image_placeholder = "(<image>./</image>)"
1275	video_placeholder = "(<video>./</video>)"
1276
1277	if modality == "image":
1278	placeholder = image_placeholder
1279	elif modality == "video":
1280	placeholder = video_placeholder
1281	elif modality == "image+video":
1282	placeholder = image_placeholder + "\n" + video_placeholder
1283
1284	prompts = [
1285	tokenizer.apply_chat_template(
1286	[
1287	{
1288	"role": "user",
1289	"content": f"{placeholder}\n{question}",
1290	}
1291	],
1292	tokenize=False,
1293	add_generation_prompt=True,
1294	)
1295	for question in questions
1296	]
1297
1298	return ModelRequestData(
1299	engine_args=engine_args,
1300	prompts=prompts,
1301	stop_token_ids=stop_token_ids,
1302	)
1303
1304
1305	def run_minicpmo(questions: list[str], modality: str) -> ModelRequestData:
1306	return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-o-2_6")
1307
1308
1309	def run_minicpmv(questions: list[str], modality: str) -> ModelRequestData:
1310	return run_minicpmv_base(questions, modality, "openbmb/MiniCPM-V-2_6")
1311
1312
1313	# Mistral-3 HF-format
1314	def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
1315	assert modality == "image"
1316
1317	model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
1318
1319	# NOTE: Need L40 (or equivalent) to avoid OOM
1320	engine_args = EngineArgs(
1321	model=model_name,
1322	max_model_len=8192,
1323	max_num_seqs=2,
1324	tensor_parallel_size=2,
1325	limit_mm_per_prompt={modality: 1},
1326	ignore_patterns=["consolidated.safetensors"],
1327	)
1328
1329	prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
1330
1331	return ModelRequestData(
1332	engine_args=engine_args,
1333	prompts=prompts,
1334	)
1335
1336
1337	# Molmo
1338	def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
1339	assert modality == "image"
1340
1341	model_name = "allenai/Molmo-7B-D-0924"
1342
1343	engine_args = EngineArgs(
1344	model=model_name,
1345	trust_remote_code=True,
1346	dtype="bfloat16",
1347	limit_mm_per_prompt={modality: 1},
1348	)
1349
1350	prompts = [
1351	f"<\|im_start\|>user <image>\n{question}<\|im_end\|><\|im_start\|>assistant\n"
1352	for question in questions
1353	]
1354
1355	return ModelRequestData(
1356	engine_args=engine_args,
1357	prompts=prompts,
1358	)
1359
1360
1361	# Molmo2
1362	def run_molmo2(questions: list[str], modality: str) -> ModelRequestData:
1363	model_name = "allenai/Molmo2-8B"
1364
1365	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1366	engine_args = EngineArgs(
1367	model=model_name,
1368	trust_remote_code=True,
1369	dtype="bfloat16",
1370	limit_mm_per_prompt=mm_limit,
1371	max_num_batched_tokens=36864,
1372	)
1373
1374	image_placeholder = "<\|image\|>"
1375	video_placeholder = "<\|video\|>"
1376
1377	if modality == "image":
1378	placeholder = image_placeholder
1379	elif modality == "video":
1380	placeholder = video_placeholder
1381	elif modality == "image+video":
1382	placeholder = image_placeholder + video_placeholder
1383
1384	prompts = [
1385	f"{placeholder}<\|im_start\|>user\n{question}<\|im_end\|>\n<\|im_start\|>assistant\n"
1386	for question in questions
1387	]
1388
1389	return ModelRequestData(
1390	engine_args=engine_args,
1391	prompts=prompts,
1392	)
1393
1394
1395	# Nemontron_VL
1396	def run_nemotron_vl(questions: list[str], modality: str) -> ModelRequestData:
1397	model_name = "nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1"
1398
1399	engine_args = EngineArgs(
1400	model=model_name,
1401	trust_remote_code=True,
1402	max_model_len=8192,
1403	limit_mm_per_prompt={modality: 1},
1404	)
1405
1406	assert modality == "image"
1407	placeholder = "<image>"
1408
1409	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1410	messages = [
1411	[{"role": "user", "content": f"{placeholder}\n{question}"}]
1412	for question in questions
1413	]
1414	prompts = tokenizer.apply_chat_template(
1415	messages, tokenize=False, add_generation_prompt=True
1416	)
1417
1418	# Stop tokens for InternVL
1419	# models variants may have different stop tokens
1420	# please refer to the model card for the correct "stop words":
1421	# https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py
1422	stop_tokens = ["<\|endoftext\|>", "<\|im_start\|>", "<\|im_end\|>", "<\|end\|>"]
1423	stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
1424	stop_token_ids = [token_id for token_id in stop_token_ids if token_id is not None]
1425
1426	return ModelRequestData(
1427	engine_args=engine_args,
1428	prompts=prompts,
1429	stop_token_ids=stop_token_ids,
1430	)
1431
1432
1433	# NVLM-D
1434	def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:
1435	assert modality == "image"
1436
1437	model_name = "nvidia/NVLM-D-72B"
1438
1439	# Adjust this as necessary to fit in GPU
1440	engine_args = EngineArgs(
1441	model=model_name,
1442	trust_remote_code=True,
1443	max_model_len=4096,
1444	tensor_parallel_size=4,
1445	limit_mm_per_prompt={modality: 1},
1446	)
1447
1448	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1449	messages = [
1450	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
1451	]
1452	prompts = tokenizer.apply_chat_template(
1453	messages, tokenize=False, add_generation_prompt=True
1454	)
1455
1456	return ModelRequestData(
1457	engine_args=engine_args,
1458	prompts=prompts,
1459	)
1460
1461
1462	# OpenPangu
1463	def run_openpangu_vl(questions: list[str], modality: str) -> ModelRequestData:
1464	model_name = "FreedomIntelligence/openPangu-VL-7B"
1465
1466	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1467	engine_args = EngineArgs(
1468	model=model_name,
1469	max_model_len=4096,
1470	max_num_seqs=4,
1471	trust_remote_code=True,
1472	enforce_eager=True,
1473	limit_mm_per_prompt=mm_limit,
1474	)
1475
1476	image_placeholder = "[unused19]"
1477	video_placeholder = "[unused32]"
1478
1479	if modality == "image":
1480	placeholder = image_placeholder
1481	elif modality == "video":
1482	placeholder = video_placeholder
1483	elif modality == "image+video":
1484	placeholder = image_placeholder + video_placeholder
1485
1486	prompts = [
1487	(
1488	f"<s>[unused9]系统：[unused10][unused9]用户：[unused18]{placeholder}[unused20]{question}[unused10][unused9]助手："
1489	)
1490	for question in questions
1491	]
1492
1493	return ModelRequestData(
1494	engine_args=engine_args,
1495	prompts=prompts,
1496	)
1497
1498
1499	# Ovis
1500	def run_ovis(questions: list[str], modality: str) -> ModelRequestData:
1501	assert modality == "image"
1502
1503	model_name = "AIDC-AI/Ovis2-1B"
1504
1505	engine_args = EngineArgs(
1506	model=model_name,
1507	max_model_len=4096,
1508	max_num_seqs=2,
1509	trust_remote_code=True,
1510	dtype="half",
1511	limit_mm_per_prompt={modality: 1},
1512	)
1513
1514	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
1515	messages = [
1516	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
1517	]
1518	prompts = tokenizer.apply_chat_template(
1519	messages, tokenize=False, add_generation_prompt=True
1520	)
1521
1522	return ModelRequestData(
1523	engine_args=engine_args,
1524	prompts=prompts,
1525	)
1526
1527
1528	# Ovis2_5
1529	def run_ovis2_5(questions: list[str], modality: str) -> ModelRequestData:
1530	model_name = "AIDC-AI/Ovis2.5-2B"
1531
1532	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1533	engine_args = EngineArgs(
1534	model=model_name,
1535	max_model_len=4096,
1536	max_num_seqs=2,
1537	trust_remote_code=True,
1538	dtype="half",
1539	limit_mm_per_prompt=mm_limit,
1540	)
1541
1542	image_placeholder = "<image>"
1543	video_placeholder = "<video>"
1544
1545	if modality == "image":
1546	placeholder = image_placeholder
1547	elif modality == "video":
1548	placeholder = video_placeholder
1549	elif modality == "image+video":
1550	placeholder = image_placeholder + "\n" + video_placeholder
1551
1552	prompts = [
1553	f"<\|im_start\|>user\n\n{placeholder}\n{question}<\|im_end\|>\n<\|im_start\|>assistant\n"
1554	for question in questions
1555	]
1556
1557	return ModelRequestData(
1558	engine_args=engine_args,
1559	prompts=prompts,
1560	)
1561
1562
1563	# PaddleOCR-VL
1564	def run_paddleocr_vl(questions: list[str], modality: str) -> ModelRequestData:
1565	assert modality == "image"
1566
1567	model_name = "PaddlePaddle/PaddleOCR-VL"
1568
1569	engine_args = EngineArgs(
1570	model=model_name,
1571	max_model_len=4096,
1572	max_num_seqs=2,
1573	limit_mm_per_prompt={modality: 1},
1574	trust_remote_code=True,
1575	)
1576
1577	placeholder = "<\|IMAGE_START\|><\|IMAGE_PLACEHOLDER\|><\|IMAGE_END\|>"
1578	prompts = [
1579	(f"<\|begin_of_sentence\|>User: {question}{placeholder}\nAssistant: ")
1580	for question in questions
1581	]
1582
1583	return ModelRequestData(
1584	engine_args=engine_args,
1585	prompts=prompts,
1586	)
1587
1588
1589	# PaliGemma
1590	def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:
1591	assert modality == "image"
1592
1593	# PaliGemma has special prompt format for VQA
1594	prompts = ["caption en" for _ in questions]
1595	engine_args = EngineArgs(
1596	model="google/paligemma-3b-mix-224",
1597	limit_mm_per_prompt={modality: 1},
1598	)
1599
1600	return ModelRequestData(
1601	engine_args=engine_args,
1602	prompts=prompts,
1603	)
1604
1605
1606	# PaliGemma 2
1607	def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:
1608	assert modality == "image"
1609
1610	# PaliGemma 2 has special prompt format for VQA
1611	prompts = ["caption en" for _ in questions]
1612	engine_args = EngineArgs(
1613	model="google/paligemma2-3b-ft-docci-448",
1614	limit_mm_per_prompt={modality: 1},
1615	)
1616
1617	return ModelRequestData(
1618	engine_args=engine_args,
1619	prompts=prompts,
1620	)
1621
1622
1623	# Phi-3-Vision
1624	def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:
1625	assert modality == "image"
1626
1627	prompts = [
1628	f"<\|user\|>\n<\|image_1\|>\n{question}<\|end\|>\n<\|assistant\|>\n"
1629	for question in questions
1630	]
1631
1632	# num_crops is an override kwarg to the multimodal image processor;
1633	# For some models, e.g., Phi-3.5-vision-instruct, it is recommended
1634	# to use 16 for single frame scenarios, and 4 for multi-frame.
1635	#
1636	# Generally speaking, a larger value for num_crops results in more
1637	# tokens per image instance, because it may scale the image more in
1638	# the image preprocessing. Some references in the model docs and the
1639	# formula for image tokens after the preprocessing
1640	# transform can be found below.
1641	#
1642	# https://huggingface.co/microsoft/Phi-3.5-vision-instruct#loading-the-model-locally
1643	# https://huggingface.co/microsoft/Phi-3.5-vision-instruct/blob/main/processing_phi3_v.py#L194
1644	engine_args = EngineArgs(
1645	model="microsoft/Phi-3.5-vision-instruct",
1646	trust_remote_code=True,
1647	max_model_len=4096,
1648	max_num_seqs=2,
1649	# Note - mm_processor_kwargs can also be passed to generate/chat calls
1650	mm_processor_kwargs={"num_crops": 16},
1651	limit_mm_per_prompt={modality: 1},
1652	)
1653
1654	return ModelRequestData(
1655	engine_args=engine_args,
1656	prompts=prompts,
1657	)
1658
1659
1660	# Phi-4-multimodal-instruct
1661	def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
1662	"""Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
1663	show how to process image inputs.
1664	"""
1665	assert modality == "image"
1666	model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
1667	# Since the vision-lora and speech-lora co-exist with the base model,
1668	# we have to manually specify the path of the lora weights.
1669	vision_lora_path = os.path.join(model_path, "vision-lora")
1670	prompts = [
1671	f"<\|user\|><\|image_1\|>{question}<\|end\|><\|assistant\|>" for question in questions
1672	]
1673	engine_args = EngineArgs(
1674	model=model_path,
1675	trust_remote_code=True,
1676	max_model_len=5120,
1677	max_num_seqs=2,
1678	max_num_batched_tokens=12800,
1679	enable_lora=True,
1680	max_lora_rank=320,
1681	# Note - mm_processor_kwargs can also be passed to generate/chat calls
1682	mm_processor_kwargs={"dynamic_hd": 16},
1683	limit_mm_per_prompt={modality: 1},
1684	)
1685
1686	return ModelRequestData(
1687	engine_args=engine_args,
1688	prompts=prompts,
1689	lora_requests=[LoRARequest("vision", 1, vision_lora_path)],
1690	)
1691
1692
1693	# Phi-4-reasoning-vision
1694	def run_phi4siglip(questions: list[str], modality: str) -> ModelRequestData:
1695	assert modality == "image"
1696	model_name = "microsoft/Phi-4-reasoning-vision-15B"
1697	prompts = [
1698	f"<\|user\|>\n<image>\n{question}<\|end\|>\n<\|assistant\|>\n"
1699	for question in questions
1700	]
1701	engine_args = EngineArgs(
1702	model=model_name,
1703	trust_remote_code=True,
1704	max_model_len=8192,
1705	max_num_seqs=2,
1706	limit_mm_per_prompt={modality: 1},
1707	)
1708	return ModelRequestData(
1709	engine_args=engine_args,
1710	prompts=prompts,
1711	)
1712
1713
1714	# Pixtral HF-format
1715	def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
1716	assert modality == "image"
1717
1718	model_name = "mistral-community/pixtral-12b"
1719
1720	# NOTE: Need L40 (or equivalent) to avoid OOM
1721	engine_args = EngineArgs(
1722	model=model_name,
1723	max_model_len=6144,
1724	max_num_seqs=2,
1725	limit_mm_per_prompt={modality: 1},
1726	)
1727
1728	prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]
1729
1730	return ModelRequestData(
1731	engine_args=engine_args,
1732	prompts=prompts,
1733	)
1734
1735
1736	# Qwen2-VL
1737	def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:
1738	model_name = "Qwen/Qwen2-VL-7B-Instruct"
1739
1740	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1741	engine_args = EngineArgs(
1742	model=model_name,
1743	max_model_len=4096,
1744	max_num_seqs=5,
1745	# Note - mm_processor_kwargs can also be passed to generate/chat calls
1746	mm_processor_kwargs={
1747	"min_pixels": 28 * 28,
1748	"max_pixels": 1280 * 28 * 28,
1749	},
1750	limit_mm_per_prompt=mm_limit,
1751	)
1752
1753	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
1754	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
1755
1756	if modality == "image":
1757	placeholder = image_placeholder
1758	elif modality == "video":
1759	placeholder = video_placeholder
1760	elif modality == "image+video":
1761	placeholder = image_placeholder + video_placeholder
1762
1763	prompts = [
1764	(
1765	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
1766	f"<\|im_start\|>user\n{placeholder}"
1767	f"{question}<\|im_end\|>\n"
1768	"<\|im_start\|>assistant\n"
1769	)
1770	for question in questions
1771	]
1772
1773	return ModelRequestData(
1774	engine_args=engine_args,
1775	prompts=prompts,
1776	)
1777
1778
1779	# Qwen2.5-VL
1780	def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
1781	model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
1782
1783	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1784	engine_args = EngineArgs(
1785	model=model_name,
1786	max_model_len=4096,
1787	max_num_seqs=5,
1788	mm_processor_kwargs={
1789	"min_pixels": 28 * 28,
1790	"max_pixels": 1280 * 28 * 28,
1791	"fps": 1,
1792	},
1793	limit_mm_per_prompt=mm_limit,
1794	)
1795
1796	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
1797	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
1798
1799	if modality == "image":
1800	placeholder = image_placeholder
1801	elif modality == "video":
1802	placeholder = video_placeholder
1803	elif modality == "image+video":
1804	placeholder = image_placeholder + video_placeholder
1805
1806	prompts = [
1807	(
1808	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
1809	f"<\|im_start\|>user\n{placeholder}"
1810	f"{question}<\|im_end\|>\n"
1811	"<\|im_start\|>assistant\n"
1812	)
1813	for question in questions
1814	]
1815
1816	return ModelRequestData(
1817	engine_args=engine_args,
1818	prompts=prompts,
1819	)
1820
1821
1822	# Qwen2.5-Omni
1823	def run_qwen2_5_omni(questions: list[str], modality: str):
1824	model_name = "Qwen/Qwen2.5-Omni-7B"
1825
1826	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1827	engine_args = EngineArgs(
1828	model=model_name,
1829	max_model_len=4096,
1830	max_num_seqs=5,
1831	mm_processor_kwargs={
1832	"min_pixels": 28 * 28,
1833	"max_pixels": 1280 * 28 * 28,
1834	"fps": 1,
1835	},
1836	limit_mm_per_prompt=mm_limit,
1837	)
1838
1839	image_placeholder = "<\|vision_bos\|><\|IMAGE\|><\|vision_eos\|>"
1840	video_placeholder = "<\|vision_bos\|><\|VIDEO\|><\|vision_eos\|>"
1841
1842	if modality == "image":
1843	placeholder = image_placeholder
1844	elif modality == "video":
1845	placeholder = video_placeholder
1846	elif modality == "image+video":
1847	placeholder = image_placeholder + video_placeholder
1848
1849	default_system = (
1850	"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
1851	"Group, capable of perceiving auditory and visual inputs, as well as "
1852	"generating text and speech."
1853	)
1854
1855	prompts = [
1856	(
1857	f"<\|im_start\|>system\n{default_system}<\|im_end\|>\n"
1858	f"<\|im_start\|>user\n{placeholder}"
1859	f"{question}<\|im_end\|>\n"
1860	"<\|im_start\|>assistant\n"
1861	)
1862	for question in questions
1863	]
1864	return ModelRequestData(
1865	engine_args=engine_args,
1866	prompts=prompts,
1867	)
1868
1869
1870	# Qwen3-VL-Dense
1871	def run_qwen3_vl(questions: list[str], modality: str) -> ModelRequestData:
1872	model_name = "Qwen/Qwen3-VL-4B-Instruct"
1873
1874	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1875	engine_args = EngineArgs(
1876	model=model_name,
1877	max_model_len=4096,
1878	max_num_seqs=5,
1879	mm_processor_kwargs={
1880	"min_pixels": 28 * 28,
1881	"max_pixels": 1280 * 28 * 28,
1882	"fps": 1,
1883	},
1884	limit_mm_per_prompt=mm_limit,
1885	)
1886
1887	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
1888	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
1889
1890	if modality == "image":
1891	placeholder = image_placeholder
1892	elif modality == "video":
1893	placeholder = video_placeholder
1894	elif modality == "image+video":
1895	placeholder = image_placeholder + video_placeholder
1896
1897	prompts = [
1898	(
1899	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
1900	f"<\|im_start\|>user\n{placeholder}"
1901	f"{question}<\|im_end\|>\n"
1902	"<\|im_start\|>assistant\n"
1903	)
1904	for question in questions
1905	]
1906
1907	return ModelRequestData(
1908	engine_args=engine_args,
1909	prompts=prompts,
1910	)
1911
1912
1913	# Qwen3-VL-MOE
1914	def run_qwen3_vl_moe(questions: list[str], modality: str) -> ModelRequestData:
1915	model_name = "Qwen/Qwen3-VL-30B-A3B-Instruct"
1916
1917	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1918	engine_args = EngineArgs(
1919	model=model_name,
1920	max_model_len=4096,
1921	max_num_seqs=5,
1922	mm_processor_kwargs={
1923	"min_pixels": 28 * 28,
1924	"max_pixels": 1280 * 28 * 28,
1925	"fps": 1,
1926	},
1927	limit_mm_per_prompt=mm_limit,
1928	)
1929
1930	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
1931	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
1932
1933	if modality == "image":
1934	placeholder = image_placeholder
1935	elif modality == "video":
1936	placeholder = video_placeholder
1937	elif modality == "image+video":
1938	placeholder = image_placeholder + video_placeholder
1939
1940	prompts = [
1941	(
1942	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
1943	f"<\|im_start\|>user\n{placeholder}"
1944	f"{question}<\|im_end\|>\n"
1945	"<\|im_start\|>assistant\n"
1946	)
1947	for question in questions
1948	]
1949
1950	return ModelRequestData(
1951	engine_args=engine_args,
1952	prompts=prompts,
1953	)
1954
1955
1956	# Qwen3.5-Dense
1957	def run_qwen3_5(questions: list[str], modality: str) -> ModelRequestData:
1958	model_name = "Qwen/Qwen3.5-4B"
1959
1960	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
1961	engine_args = EngineArgs(
1962	model=model_name,
1963	max_model_len=4096,
1964	max_num_seqs=5,
1965	mm_processor_kwargs={
1966	"min_pixels": 28 * 28,
1967	"max_pixels": 1280 * 28 * 28,
1968	"fps": 1,
1969	},
1970	limit_mm_per_prompt=mm_limit,
1971	)
1972
1973	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
1974	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
1975
1976	if modality == "image":
1977	placeholder = image_placeholder
1978	elif modality == "video":
1979	placeholder = video_placeholder
1980	elif modality == "image+video":
1981	placeholder = image_placeholder + video_placeholder
1982
1983	prompts = [
1984	(
1985	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
1986	f"<\|im_start\|>user\n{placeholder}"
1987	f"{question}<\|im_end\|>\n"
1988	"<\|im_start\|>assistant\n"
1989	)
1990	for question in questions
1991	]
1992
1993	return ModelRequestData(
1994	engine_args=engine_args,
1995	prompts=prompts,
1996	)
1997
1998
1999	# Qwen3.5-MoE
2000	def run_qwen3_5_moe(questions: list[str], modality: str) -> ModelRequestData:
2001	model_name = "Qwen/Qwen3.5-35B-A3B"
2002
2003	mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
2004	engine_args = EngineArgs(
2005	model=model_name,
2006	max_model_len=4096,
2007	max_num_seqs=5,
2008	mm_processor_kwargs={
2009	"min_pixels": 28 * 28,
2010	"max_pixels": 1280 * 28 * 28,
2011	"fps": 1,
2012	},
2013	limit_mm_per_prompt=mm_limit,
2014	)
2015
2016	image_placeholder = "<\|vision_start\|><\|image_pad\|><\|vision_end\|>"
2017	video_placeholder = "<\|vision_start\|><\|video_pad\|><\|vision_end\|>"
2018
2019	if modality == "image":
2020	placeholder = image_placeholder
2021	elif modality == "video":
2022	placeholder = video_placeholder
2023	elif modality == "image+video":
2024	placeholder = image_placeholder + video_placeholder
2025
2026	prompts = [
2027	(
2028	"<\|im_start\|>system\nYou are a helpful assistant.<\|im_end\|>\n"
2029	f"<\|im_start\|>user\n{placeholder}"
2030	f"{question}<\|im_end\|>\n"
2031	"<\|im_start\|>assistant\n"
2032	)
2033	for question in questions
2034	]
2035
2036	return ModelRequestData(
2037	engine_args=engine_args,
2038	prompts=prompts,
2039	)
2040
2041
2042	# R-4B
2043	def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
2044	assert modality == "image"
2045	model_name = "YannQi/R-4B"
2046
2047	prompts = [
2048	f"<\|im_start\|>user <image>\n{question}<\|im_end\|><\|im_start\|>assistant\n"
2049	for question in questions
2050	]
2051
2052	engine_args = EngineArgs(
2053	model=model_name,
2054	max_model_len=16384,
2055	limit_mm_per_prompt={modality: 1},
2056	)
2057
2058	return ModelRequestData(
2059	engine_args=engine_args,
2060	prompts=prompts,
2061	)
2062
2063
2064	# SkyworkR1V
2065	def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
2066	assert modality == "image"
2067
2068	model_name = "Skywork/Skywork-R1V-38B"
2069
2070	engine_args = EngineArgs(
2071	model=model_name,
2072	trust_remote_code=True,
2073	max_model_len=4096,
2074	limit_mm_per_prompt={modality: 1},
2075	)
2076
2077	tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
2078	messages = [
2079	[{"role": "user", "content": f"<image>\n{question}"}] for question in questions
2080	]
2081	prompts = tokenizer.apply_chat_template(
2082	messages, tokenize=False, add_generation_prompt=True
2083	)
2084
2085	# Stop tokens for SkyworkR1V
2086	# https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
2087	stop_tokens = ["<｜end▁of▁sentence｜>", "<\|endoftext\|>"]
2088	stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
2089
2090	return ModelRequestData(
2091	engine_args=engine_args,
2092	prompts=prompts,
2093	stop_token_ids=stop_token_ids,
2094	)
2095
2096
2097	# SmolVLM2-2.2B-Instruct
2098	def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:
2099	assert modality == "image"
2100	model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
2101
2102	engine_args = EngineArgs(
2103	model=model_name,
2104	max_model_len=8192,
2105	max_num_seqs=2,
2106	enforce_eager=True,
2107	mm_processor_kwargs={
2108	"max_image_size": {"longest_edge": 384},
2109	},
2110	limit_mm_per_prompt={modality: 1},
2111	)
2112	prompts = [
2113	(f"<\|im_start\|>User:<image>{question}<end_of_utterance>\nAssistant:")
2114	for question in questions
2115	]
2116
2117	return ModelRequestData(
2118	engine_args=engine_args,
2119	prompts=prompts,
2120	)
2121
2122
2123	# Step3
2124	def run_step3(questions: list[str], modality: str) -> ModelRequestData:
2125	assert modality == "image"
2126
2127	model_name = "stepfun-ai/step3-fp8"
2128
2129	# NOTE: Below are verified configurations for step3-fp8
2130	# on 8xH100 GPUs.
2131	engine_args = EngineArgs(
2132	model=model_name,
2133	max_num_batched_tokens=4096,
2134	gpu_memory_utilization=0.85,
2135	tensor_parallel_size=8,
2136	limit_mm_per_prompt={modality: 1},
2137	reasoning_parser="step3",
2138	)
2139
2140	prompts = [
2141	"<｜begin▁of▁sentence｜> You are a helpful assistant. <\|BOT\|>user\n "
2142	f"<im_patch>{question} <\|EOT\|><\|BOT\|>assistant\n<think>\n"
2143	for question in questions
2144	]
2145
2146	return ModelRequestData(
2147	engine_args=engine_args,
2148	prompts=prompts,
2149	)
2150
2151
2152	# StepVL10B
2153	def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
2154	assert modality == "image"
2155
2156	model_name = "stepfun-ai/Step3-VL-10B"
2157	engine_args = EngineArgs(
2158	model=model_name,
2159	max_num_batched_tokens=4096,
2160	tensor_parallel_size=1,
2161	trust_remote_code=True,
2162	limit_mm_per_prompt={modality: 1},
2163	reasoning_parser="deepseek_r1",
2164	)
2165
2166	prompts = [
2167	"<｜begin▁of▁sentence｜> You are a helpful assistant.<\|BOT\|>user\n "
2168	f"<im_patch>{question} <\|EOT\|><\|BOT\|>assistant\n<think>\n"
2169	for question in questions
2170	]
2171
2172	return ModelRequestData(
2173	engine_args=engine_args,
2174	prompts=prompts,
2175	)
2176
2177
2178	model_example_map = {
2179	"aria": run_aria,
2180	"bagel": run_bagel,
2181	"bee": run_bee,
2182	"blip-2": run_blip2,
2183	"command_a_vision": run_command_a_vision,
2184	"deepseek_vl_v2": run_deepseek_vl2,
2185	"deepseek_ocr": run_deepseek_ocr,
2186	"deepseek_ocr2": run_deepseek_ocr2,
2187	"dots_ocr": run_dots_ocr,
2188	"eagle2_5": run_eagle2_5,
2189	"ernie45_vl": run_ernie45_vl,
2190	"exaone4_5": run_exaone4_5,
2191	"gemma3": run_gemma3,
2192	"gemma3n": run_gemma3n,
2193	"gemma4": run_gemma4,
2194	"glm4v": run_glm4v,
2195	"glm4_1v": run_glm4_1v,
2196	"glm4_5v": run_glm4_5v,
2197	"glm4_5v_fp8": run_glm4_5v_fp8,
2198	"glm_ocr": run_glm_ocr,
2199	"h2ovl_chat": run_h2ovl,
2200	"hunyuan_vl": run_hunyuan_vl,
2201	"idefics3": run_idefics3,
2202	"interns1": run_interns1,
2203	"interns1_pro": run_interns1_pro,
2204	"internvl_chat": run_internvl,
2205	"kanana_v": run_kanana_v,
2206	"keye_vl": run_keye_vl,
2207	"keye_vl1_5": run_keye_vl1_5,
2208	"kimi_vl": run_kimi_vl,
2209	"kimi_k25": run_kimi_k25,
2210	"lightonocr": run_lightonocr,
2211	"lfm2_vl": run_lfm2_vl,
2212	"llama4": run_llama4,
2213	"llava": run_llava,
2214	"llava-next": run_llava_next,
2215	"llava-next-video": run_llava_next_video,
2216	"llava-onevision": run_llava_onevision,
2217	"minicpmo": run_minicpmo,
2218	"minicpmv": run_minicpmv,
2219	"mistral3": run_mistral3,
2220	"molmo": run_molmo,
2221	"molmo2": run_molmo2,
2222	"nemotron_vl": run_nemotron_vl,
2223	"NVLM_D": run_nvlm_d,
2224	"openpangu_vl": run_openpangu_vl,
2225	"ovis": run_ovis,
2226	"ovis2_5": run_ovis2_5,
2227	"paddleocr_vl": run_paddleocr_vl,
2228	"paligemma": run_paligemma,
2229	"paligemma2": run_paligemma2,
2230	"phi3_v": run_phi3v,
2231	"phi4_mm": run_phi4mm,
2232	"phi4_siglip": run_phi4siglip,
2233	"pixtral_hf": run_pixtral_hf,
2234	"qwen2_vl": run_qwen2_vl,
2235	"qwen2_5_vl": run_qwen2_5_vl,
2236	"qwen2_5_omni": run_qwen2_5_omni,
2237	"qwen3_vl": run_qwen3_vl,
2238	"qwen3_vl_moe": run_qwen3_vl_moe,
2239	"qwen3_5": run_qwen3_5,
2240	"qwen3_5_moe": run_qwen3_5_moe,
2241	"rvl": run_r_vl,
2242	"skywork_chat": run_skyworkr1v,
2243	"smolvlm": run_smolvlm,
2244	"step3": run_step3,
2245	"stepvl": run_step_vl,
2246	}
2247
2248
2249	MODELS_NEED_VIDEO_METADATA = [
2250	"glm4_1v",
2251	"glm_ocr",
2252	"glm4_5v",
2253	"glm4_5v_fp8",
2254	"molmo2",
2255	"qwen2_5_vl",
2256	"qwen3_vl",
2257	"qwen3_vl_moe",
2258	"qwen3_5",
2259	"qwen3_5_moe",
2260	]
2261
2262
2263	MODELS_SUPPORT_VIT_CUDA_GRAPH = [
2264	"llama4",
2265	"gemma4",
2266	"qwen2_vl",
2267	"qwen2_5_vl",
2268	"qwen3_vl",
2269	"qwen3_vl_moe",
2270	"kimi_vl",
2271	"qwen3_5",
2272	"qwen3_5_moe",
2273	"internvl_chat",
2274	"stepvl",
2275	"glm4_1v",
2276	"deepseek_ocr",
2277	"ernie45_vl",
2278	"minicpmv2_5_vl",
2279	"minicpmv2_6_vl",
2280	"minicpmv4_vl",
2281	]
2282
2283
2284	def get_multi_modal_input(args):
2285	"""Return {
2286	"data": image or video,
2287	"question": question,
2288	}
2289	"""
2290	if args.modality == "image":
2291	# Input image and question
2292	image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
2293	img_questions = [
2294	"What is the content of this image?",
2295	"Describe the content of this image in detail.",
2296	"What's in the image?",
2297	"Where is this image taken?",
2298	]
2299
2300	return {
2301	"data": image,
2302	"questions": img_questions,
2303	}
2304
2305	if args.modality == "video":
2306	# Input video and question
2307	needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
2308	video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
2309	metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
2310	vid_questions = ["Why is this video funny?"]
2311
2312	return {
2313	"data": ([(video, metadata)] if needs_metadata else video),
2314	"questions": vid_questions,
2315	}
2316
2317	if args.modality == "vision_chunk":
2318	# Input vision chunks and question
2319	image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
2320	vision_chunk_questions = [
2321	"What is the content of this image chunk?",
2322	"Describe the content of this image chunk in detail.",
2323	]
2324
2325	return {
2326	"data": {"type": "image", "image": image},
2327	"questions": vision_chunk_questions,
2328	}
2329
2330	if args.modality == "image+video":
2331	image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
2332	needs_metadata = args.model_type in MODELS_NEED_VIDEO_METADATA
2333	video = VideoAsset(name="baby_reading", num_frames=args.num_frames).np_ndarrays
2334	metadata = VideoAsset(name="baby_reading", num_frames=args.num_frames).metadata
2335	img_video_questions = [
2336	"What is shown in the image? What happens in the video?",
2337	"Describe both the image and the video content.",
2338	]
2339
2340	return {
2341	"data": {
2342	"image": image,
2343	"video": ([(video, metadata)] if needs_metadata else video),
2344	},
2345	"questions": img_video_questions,
2346	}
2347
2348	msg = f"Modality {args.modality} is not supported."
2349	raise ValueError(msg)
2350
2351
2352	def apply_image_repeat(
2353	image_repeat_prob, num_prompts, data, prompts: list[str], modality
2354	):
2355	"""Repeats images with provided probability of "image_repeat_prob".
2356	Used to simulate hit/miss for the MM preprocessor cache.
2357	"""
2358	assert image_repeat_prob <= 1.0 and image_repeat_prob >= 0
2359	no_yes = [0, 1]
2360	probs = [1.0 - image_repeat_prob, image_repeat_prob]
2361
2362	inputs = []
2363	inputs_with_empty_media = []
2364	cur_image = data
2365	for i in range(num_prompts):
2366	if image_repeat_prob is not None:
2367	res = random.choices(no_yes, probs)[0]
2368	if res == 0:
2369	# No repeat => Modify one pixel
2370	cur_image = cur_image.copy()
2371	new_val = (i // 256 // 256, i // 256, i % 256)
2372	cur_image.putpixel((0, 0), new_val)
2373
2374	uuid = "uuid_{}".format(i)
2375
2376	inputs.append(
2377	{
2378	"prompt": prompts[i % len(prompts)],
2379	"multi_modal_data": {modality: cur_image},
2380	"multi_modal_uuids": {modality: uuid},
2381	}
2382	)
2383
2384	inputs_with_empty_media.append(
2385	{
2386	"prompt": prompts[i % len(prompts)],
2387	"multi_modal_data": {modality: None},
2388	"multi_modal_uuids": {modality: uuid},
2389	}
2390	)
2391
2392	return inputs, inputs_with_empty_media
2393
2394
2395	def maybe_add_vit_cuda_graph_compilation_config(args, engine_args):
2396	model = args.model_type
2397	modality = args.modality
2398	enable_vit_cuda_graph = args.enable_vit_cuda_graph
2399
2400	if enable_vit_cuda_graph and model in MODELS_SUPPORT_VIT_CUDA_GRAPH:
2401	if modality == "image" or modality == "video":
2402	vision_items_per_batch = 1
2403	elif modality == "image+video":
2404	vision_items_per_batch = 2
2405	else:
2406	raise ValueError(
2407	f"modality={modality} is not supported for vit cuda graph."
2408	)
2409
2410	engine_args.compilation_config = {
2411	"cudagraph_mm_encoder": True,
2412	"encoder_cudagraph_max_vision_items_per_batch": vision_items_per_batch,
2413	}
2414
2415	return engine_args
2416
2417
2418	@contextmanager
2419	def time_counter(enable: bool):
2420	if enable:
2421	import time
2422
2423	start_time = time.time()
2424	yield
2425	elapsed_time = time.time() - start_time
2426	print("-" * 50)
2427	print("-- generate time = {}".format(elapsed_time))
2428	print("-" * 50)
2429	else:
2430	yield
2431
2432
2433	def parse_args():
2434	parser = FlexibleArgumentParser(
2435	description="Demo on using vLLM for offline inference with "
2436	"vision language models for text generation"
2437	)
2438	parser.add_argument(
2439	"--model-type",
2440	"-m",
2441	type=str,
2442	default="llava",
2443	choices=model_example_map.keys(),
2444	help='Huggingface "model_type".',
2445	)
2446	parser.add_argument(
2447	"--num-prompts", type=int, default=4, help="Number of prompts to run."
2448	)
2449	parser.add_argument(
2450	"--modality",
2451	type=str,
2452	default="image",
2453	choices=["image", "video", "image+video", "vision_chunk"],
2454	help="Modality of the input.",
2455	)
2456	parser.add_argument(
2457	"--num-frames",
2458	type=int,
2459	default=16,
2460	help="Number of frames to extract from the video.",
2461	)
2462	parser.add_argument(
2463	"--seed",
2464	type=int,
2465	default=0,
2466	help="Set the seed when initializing `vllm.LLM`.",
2467	)
2468	parser.add_argument(
2469	"--image-repeat-prob",
2470	type=float,
2471	default=None,
2472	help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)",
2473	)
2474	parser.add_argument(
2475	"--disable-mm-processor-cache",
2476	action="store_true",
2477	help="If True, disables caching of multi-modal processor.",
2478	)
2479	parser.add_argument(
2480	"--time-generate",
2481	action="store_true",
2482	help="If True, then print the total generate() call time",
2483	)
2484	parser.add_argument(
2485	"--use-different-prompt-per-request",
2486	action="store_true",
2487	help="If True, then use different prompt (with the same multi-modal "
2488	"data) for each request.",
2489	)
2490	parser.add_argument(
2491	"--verify-mm-cache-hit-with-uuids",
2492	action="store_true",
2493	help="If True, will send all requests in a second batch with empty mm "
2494	"data to verify cache hits with UUIDs.",
2495	)
2496	parser.add_argument(
2497	"--tensor-parallel-size",
2498	"-tp",
2499	type=int,
2500	default=None,
2501	help="Tensor parallel size to override the model's default setting. ",
2502	)
2503	parser.add_argument(
2504	"--enable-vit-cuda-graph",
2505	action="store_true",
2506	help="If True, will enable vit cuda graph capture and replay for the model.",
2507	)
2508	return parser.parse_args()
2509
2510
2511	def main(args):
2512	model = args.model_type
2513	if model not in model_example_map:
2514	raise ValueError(f"Model type {model} is not supported.")
2515
2516	if args.tensor_parallel_size is not None and args.tensor_parallel_size < 1:
2517	raise ValueError(
2518	f"tensor_parallel_size must be a positive integer, "
2519	f"got {args.tensor_parallel_size}"
2520	)
2521
2522	modality = args.modality
2523	mm_input = get_multi_modal_input(args)
2524	data = mm_input["data"]
2525	questions = mm_input["questions"]
2526
2527	req_data = model_example_map[model](questions, modality)
2528
2529	# Disable other modalities to save memory
2530	default_limits = {"image": 0, "video": 0, "audio": 0, "vision_chunk": 0}
2531	req_data.engine_args.limit_mm_per_prompt = default_limits \| dict(
2532	req_data.engine_args.limit_mm_per_prompt or {}
2533	)
2534
2535	engine_args = req_data.engine_args
2536	engine_args.seed = args.seed
2537	mm_processor_cache_gb = 0 if args.disable_mm_processor_cache else 4
2538	engine_args.mm_processor_cache_gb = mm_processor_cache_gb
2539	if args.tensor_parallel_size is not None:
2540	engine_args.tensor_parallel_size = args.tensor_parallel_size
2541	engine_args = maybe_add_vit_cuda_graph_compilation_config(args, engine_args)
2542	if current_platform.is_rocm():
2543	os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
2544	llm = LLM.from_engine_args(engine_args)
2545
2546	# Don't want to check the flag multiple times, so just hijack `prompts`.
2547	prompts = (
2548	req_data.prompts
2549	if args.use_different_prompt_per_request
2550	else [req_data.prompts[0]]
2551	)
2552
2553	# We set temperature to 0.2 so that outputs can be different
2554	# even when all prompts are identical when running batch inference.
2555	sampling_params = (
2556	SamplingParams(
2557	temperature=0.2, max_tokens=64, stop_token_ids=req_data.stop_token_ids
2558	)
2559	if req_data.sampling_params is None
2560	else req_data.sampling_params
2561	)
2562
2563	def _mm_data(data, modality):
2564	if modality == "image+video":
2565	return {"image": data["image"], "video": data["video"]}
2566	return {modality: data}
2567
2568	def _mm_uuid(uuid, modality):
2569	if modality == "image+video":
2570	return {"image": uuid, "video": uuid + "v"}
2571	return {modality: uuid}
2572
2573	def _mm_empty(modality):
2574	if modality == "image+video":
2575	return {"image": None, "video": None}
2576	return {modality: None}
2577
2578	assert args.num_prompts > 0
2579	if args.num_prompts == 1:
2580	# Single inference
2581	uuid = "uuid_0"
2582	inputs = {
2583	"prompt": prompts[0],
2584	"multi_modal_data": _mm_data(data, modality),
2585	"multi_modal_uuids": _mm_uuid(uuid, modality),
2586	}
2587	inputs_with_empty_media = {
2588	"prompt": prompts[0],
2589	"multi_modal_data": _mm_empty(modality),
2590	"multi_modal_uuids": _mm_uuid(uuid, modality),
2591	}
2592	else:
2593	# Batch inference
2594	if args.image_repeat_prob is not None:
2595	if modality == "image+video":
2596	raise ValueError(
2597	"--image-repeat-prob is not supported for 'image+video' modality"
2598	)
2599	# Repeat images with specified probability of "image_repeat_prob"
2600	inputs, inputs_with_empty_media = apply_image_repeat(
2601	args.image_repeat_prob,
2602	args.num_prompts,
2603	data,
2604	prompts,
2605	modality,
2606	)
2607	else:
2608	# Use the same image/video for all prompts
2609	inputs = []
2610	inputs_with_empty_media = []
2611	for i in range(args.num_prompts):
2612	uuid = "uuid_{}".format(i)
2613	inputs.append(
2614	{
2615	"prompt": prompts[i % len(prompts)],
2616	"multi_modal_data": _mm_data(data, modality),
2617	"multi_modal_uuids": _mm_uuid(uuid, modality),
2618	}
2619	)
2620	inputs_with_empty_media.append(
2621	{
2622	"prompt": prompts[i % len(prompts)],
2623	"multi_modal_data": _mm_empty(modality),
2624	"multi_modal_uuids": _mm_uuid(uuid, modality),
2625	}
2626	)
2627
2628	# Add LoRA request if applicable
2629	lora_request = (
2630	req_data.lora_requests * args.num_prompts if req_data.lora_requests else None
2631	)
2632
2633	with time_counter(args.time_generate):
2634	outputs = llm.generate(
2635	inputs,
2636	sampling_params=sampling_params,
2637	lora_request=lora_request,
2638	)
2639
2640	print("-" * 50)
2641	for o in outputs:
2642	generated_text = o.outputs[0].text
2643	print(generated_text)
2644	print("-" * 50)
2645
2646	if args.verify_mm_cache_hit_with_uuids:
2647	try:
2648	# Verify cache hits with UUIDs
2649	print(
2650	"Sending a second batch of requests with empty media"
2651	" and matching UUIDs."
2652	)
2653	outputs = llm.generate(
2654	inputs_with_empty_media,
2655	sampling_params=sampling_params,
2656	lora_request=lora_request,
2657	)
2658	print("-" * 50)
2659	for o in outputs:
2660	generated_text = o.outputs[0].text
2661	print(generated_text)
2662	print("-" * 50)
2663	except Exception as e:
2664	print(f"Failed to verify cache hits with UUIDs. Error: {e}")
2665
2666
2667	if __name__ == "__main__":
2668	args = parse_args()
2669	main(args)