@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d9fc8cd9da

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

[V1] Enable multi-input by default (#15799)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Apr 12, 2025 at 16:52 UTC d9fc8cd9da4a69cb4171efb7cb5a46308680c83c

21 files changed +214 -105

docs/source/models/supported_models.md

+3 -1

@@ -759,7 +759,7 @@ On the other hand, modalities separated by `/` are mutually exclusive.

759 See [this page](#multimodal-inputs) on how to pass multi-modal inputs to the model.

760

761 :::{important}

762 -To enable multiple multi-modal items per text prompt, you have to set `limit_mm_per_prompt` (offline inference)

762 +**To enable multiple multi-modal items per text prompt in vLLM V0**, you have to set `limit_mm_per_prompt` (offline inference)

763 or `--limit-mm-per-prompt` (online serving). For example, to enable passing up to 4 images per text prompt:

764

765 Offline inference:

@@ -777,6 +777,8 @@ Online serving:

777 vllm serve Qwen/Qwen2-VL-7B-Instruct --limit-mm-per-prompt image=4

778 ```

779

780 +**This is no longer required if you are using vLLM V1.**

781 +

782 :::

783

784 :::{note}

docs/source/serving/offline_inference.md

+24

@@ -110,6 +110,30 @@ If you run out of CPU RAM, try the following options:

110 - (Multi-modal models only) you can set the size of multi-modal input cache using `VLLM_MM_INPUT_CACHE_GIB` environment variable (default 4 GiB).

111 - (CPU backend only) you can set the size of KV cache using `VLLM_CPU_KVCACHE_SPACE` environment variable (default 4 GiB).

112

113 +#### Disable unused modalities

114 +

115 +You can disable unused modalities (except for text) by setting its limit to zero.

116 +

117 +For example, if your application only accepts image input, there is no need to allocate any memory for videos.

118 +

119 +```python

120 +from vllm import LLM

121 +

122 +# Accept images but not videos

123 +llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct",

124 + limit_mm_per_prompt={"video": 0})

125 +```

126 +

127 +You can even run a multi-modal model for text-only inference:

128 +

129 +```python

130 +from vllm import LLM

131 +

132 +# Don't accept images. Just text.

133 +llm = LLM(model="google/gemma-3-27b-it",

134 + limit_mm_per_prompt={"image": 0})

135 +```

136 +

137 ### Performance optimization and tuning

138

139 You can potentially improve the performance of vLLM by finetuning various options.

examples/offline_inference/audio_language.py

@@ -196,6 +196,11 @@ def main(args):

196 req_data = model_example_map[model](question_per_audio_count[audio_count],

197 audio_count)

198

199 + # Disable other modalities to save memory

200 + default_limits = {"image": 0, "video": 0, "audio": 0}

201 + req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

202 + req_data.engine_args.limit_mm_per_prompt or {})

203 +

204 engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

205 llm = LLM(**engine_args)

206

examples/offline_inference/encoder_decoder_multimodal.py

@@ -133,6 +133,11 @@ def main(args):

133

134 req_data = model_example_map[model]()

135

136 + # Disable other modalities to save memory

137 + default_limits = {"image": 0, "video": 0, "audio": 0}

138 + req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

139 + req_data.engine_args.limit_mm_per_prompt or {})

140 +

141 engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

142 llm = LLM(**engine_args)

143

examples/offline_inference/vision_language.py

+45 -34

@@ -45,7 +45,7 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:

45 max_model_len=4096,

46 max_num_seqs=2,

47 dtype="bfloat16",

48 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

48 + limit_mm_per_prompt={"image": 1},

49 )

51 prompts = [(f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"

@@ -71,7 +71,7 @@ def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:

71 max_model_len=2048,

72 max_num_seqs=2,

73 mm_processor_kwargs={"crop_to_patches": True},

74 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

74 + limit_mm_per_prompt={"image": 1},

75 )

76 prompts = [

77 f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"

@@ -92,7 +92,7 @@ def run_blip2(questions: list[str], modality: str) -> ModelRequestData:

92 prompts = [f"Question: {question} Answer:" for question in questions]

93 engine_args = EngineArgs(

94 model="Salesforce/blip2-opt-6.7b",

95 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

95 + limit_mm_per_prompt={"image": 1},

96 )

98 return ModelRequestData(

@@ -110,7 +110,7 @@ def run_chameleon(questions: list[str], modality: str) -> ModelRequestData:

110 model="facebook/chameleon-7b",

111 max_model_len=4096,

112 max_num_seqs=2,

113 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

113 + limit_mm_per_prompt={"image": 1},

114 )

115

116 return ModelRequestData(

@@ -129,8 +129,8 @@ def run_deepseek_vl2(questions: list[str], modality: str) -> ModelRequestData:

129 model=model_name,

130 max_model_len=4096,

131 max_num_seqs=2,

132 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

132 hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},

133 + limit_mm_per_prompt={"image": 1},

134 )

135

136 prompts = [

@@ -155,7 +155,7 @@ def run_florence2(questions: list[str], modality: str) -> ModelRequestData:

155 max_num_seqs=2,

156 trust_remote_code=True,

157 dtype="bfloat16",

158 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

158 + limit_mm_per_prompt={"image": 1},

159 )

160

161 prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]

@@ -175,7 +175,7 @@ def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:

175 model="adept/fuyu-8b",

176 max_model_len=2048,

177 max_num_seqs=2,

178 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

178 + limit_mm_per_prompt={"image": 1},

179 )

180

181 return ModelRequestData(

@@ -194,7 +194,7 @@ def run_gemma3(questions: list[str], modality: str) -> ModelRequestData:

194 max_model_len=2048,

195 max_num_seqs=2,

196 mm_processor_kwargs={"do_pan_and_scan": True},

197 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

197 + limit_mm_per_prompt={"image": 1},

198 )

199

200 prompts = [("<bos><start_of_turn>user\n"

@@ -219,7 +219,7 @@ def run_glm4v(questions: list[str], modality: str) -> ModelRequestData:

219 trust_remote_code=True,

220 enforce_eager=True,

221 hf_overrides={"architectures": ["GLM4VForCausalLM"]},

222 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

222 + limit_mm_per_prompt={"image": 1},

223 )

224

225 prompts = [

@@ -246,7 +246,7 @@ def run_h2ovl(questions: list[str], modality: str) -> ModelRequestData:

246 model=model_name,

247 trust_remote_code=True,

248 max_model_len=8192,

249 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

249 + limit_mm_per_prompt={"image": 1},

250 )

251

252 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -287,7 +287,7 @@ def run_idefics3(questions: list[str], modality: str) -> ModelRequestData:

287 "longest_edge": 3 * 364

288 },

289 },

290 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

290 + limit_mm_per_prompt={"image": 1},

291 )

292 prompts = [(

293 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"

@@ -314,7 +314,7 @@ def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:

314 "longest_edge": 384

315 },

316 },

317 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

317 + limit_mm_per_prompt={"image": 1},

318 )

319 prompts = [

320 (f"<|im_start|>User:<image>{question}<end_of_utterance>\nAssistant:")

@@ -337,7 +337,7 @@ def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

337 model=model_name,

338 trust_remote_code=True,

339 max_model_len=4096,

340 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

340 + limit_mm_per_prompt={"image": 1},

341 )

342

343 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -375,7 +375,7 @@ def run_llava(questions: list[str], modality: str) -> ModelRequestData:

375 engine_args = EngineArgs(

376 model="llava-hf/llava-1.5-7b-hf",

377 max_model_len=4096,

378 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

378 + limit_mm_per_prompt={"image": 1},

379 )

380

381 return ModelRequestData(

@@ -392,7 +392,7 @@ def run_llava_next(questions: list[str], modality: str) -> ModelRequestData:

392 engine_args = EngineArgs(

393 model="llava-hf/llava-v1.6-mistral-7b-hf",

394 max_model_len=8192,

395 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

395 + limit_mm_per_prompt={"image": 1},

396 )

397

398 return ModelRequestData(

@@ -414,7 +414,7 @@ def run_llava_next_video(questions: list[str],

414 model="llava-hf/LLaVA-NeXT-Video-7B-hf",

415 max_model_len=8192,

416 max_num_seqs=2,

417 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

417 + limit_mm_per_prompt={"image": 1},

418 )

419

420 return ModelRequestData(

@@ -442,7 +442,7 @@ def run_llava_onevision(questions: list[str],

442 engine_args = EngineArgs(

443 model="llava-hf/llava-onevision-qwen2-7b-ov-hf",

444 max_model_len=16384,

445 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

445 + limit_mm_per_prompt={"image": 1},

446 )

447

448 return ModelRequestData(

@@ -465,7 +465,7 @@ def run_mantis(questions: list[str], modality: str) -> ModelRequestData:

465 model="TIGER-Lab/Mantis-8B-siglip-llama3",

466 max_model_len=4096,

467 hf_overrides={"architectures": ["MantisForConditionalGeneration"]},

468 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

468 + limit_mm_per_prompt={"image": 1},

469 )

470 stop_token_ids = [128009]

471

@@ -506,7 +506,7 @@ def run_minicpmv_base(questions: list[str], modality: str, model_name):

506 max_model_len=4096,

507 max_num_seqs=2,

508 trust_remote_code=True,

509 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

509 + limit_mm_per_prompt={"image": 1},

510 )

511 # NOTE The stop_token_ids are different for various versions of MiniCPM-V

512 # 2.0

@@ -561,7 +561,7 @@ def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:

561 max_model_len=8192,

562 max_num_seqs=2,

563 tensor_parallel_size=2,

564 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

564 + limit_mm_per_prompt={"image": 1},

565 )

566

567 prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

@@ -587,7 +587,7 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:

587 model=model_name,

588 max_model_len=8192,

589 max_num_seqs=2,

590 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

590 + limit_mm_per_prompt={"image": 1},

591 )

592

593 tokenizer = AutoTokenizer.from_pretrained(model_name)

@@ -611,7 +611,7 @@ def run_mllama(questions: list[str], modality: str) -> ModelRequestData:

611 )

612

613

614 -def run_llama4(questions: list[str], modality: str):

614 +def run_llama4(questions: list[str], modality: str) -> ModelRequestData:

615 assert modality == "image"

616

617 model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"

@@ -621,8 +621,8 @@ def run_llama4(questions: list[str], modality: str):

621 max_model_len=8192,

622 max_num_seqs=4,

623 tensor_parallel_size=8,

624 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

624 gpu_memory_utilization=0.4,

625 + limit_mm_per_prompt={"image": 1},

626 )

627

628 tokenizer = AutoTokenizer.from_pretrained(model_name)

@@ -657,7 +657,7 @@ def run_molmo(questions: list[str], modality: str) -> ModelRequestData:

657 model=model_name,

658 trust_remote_code=True,

659 dtype="bfloat16",

660 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

660 + limit_mm_per_prompt={"image": 1},

661 )

662

663 prompts = [

@@ -683,7 +683,7 @@ def run_nvlm_d(questions: list[str], modality: str) -> ModelRequestData:

683 trust_remote_code=True,

684 max_model_len=4096,

685 tensor_parallel_size=4,

686 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

686 + limit_mm_per_prompt={"image": 1},

687 )

688

689 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -710,7 +710,8 @@ def run_paligemma(questions: list[str], modality: str) -> ModelRequestData:

710 prompts = ["caption en" for _ in questions]

711 engine_args = EngineArgs(

712 model="google/paligemma-3b-mix-224",

713 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)

713 + limit_mm_per_prompt={"image": 1},

714 + )

715

716 return ModelRequestData(

717 engine_args=engine_args,

@@ -726,7 +727,8 @@ def run_paligemma2(questions: list[str], modality: str) -> ModelRequestData:

727 prompts = ["caption en" for _ in questions]

728 engine_args = EngineArgs(

729 model="google/paligemma2-3b-ft-docci-448",

729 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)

730 + limit_mm_per_prompt={"image": 1},

731 + )

732

733 return ModelRequestData(

734 engine_args=engine_args,

@@ -762,7 +764,7 @@ def run_phi3v(questions: list[str], modality: str) -> ModelRequestData:

764 max_num_seqs=2,

765 # Note - mm_processor_kwargs can also be passed to generate/chat calls

766 mm_processor_kwargs={"num_crops": 16},

765 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

767 + limit_mm_per_prompt={"image": 1},

768 )

769

770 return ModelRequestData(

@@ -793,6 +795,7 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:

795 max_num_seqs=2,

796 enable_lora=True,

797 max_lora_rank=320,

798 + limit_mm_per_prompt={"image": 1},

799 )

800

801 return ModelRequestData(

@@ -813,7 +816,7 @@ def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:

816 model=model_name,

817 max_model_len=6144,

818 max_num_seqs=2,

816 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

819 + limit_mm_per_prompt={"image": 1},

820 )

821

822 prompts = [f"<s>[INST]{question}\n[IMG][/INST]" for question in questions]

@@ -834,7 +837,7 @@ def run_qwen_vl(questions: list[str], modality: str) -> ModelRequestData:

837 max_model_len=1024,

838 max_num_seqs=2,

839 hf_overrides={"architectures": ["QwenVLForConditionalGeneration"]},

837 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

840 + limit_mm_per_prompt={"image": 1},

841 )

842

843 prompts = [f"{question}Picture 1: <img></img>\n" for question in questions]

@@ -859,7 +862,7 @@ def run_qwen2_vl(questions: list[str], modality: str) -> ModelRequestData:

862 "min_pixels": 28 * 28,

863 "max_pixels": 1280 * 28 * 28,

864 },

862 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

865 + limit_mm_per_prompt={"image": 1},

866 )

867

868 if modality == "image":

@@ -894,7 +897,7 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:

897 "max_pixels": 1280 * 28 * 28,

898 "fps": 1,

899 },

897 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

900 + limit_mm_per_prompt={"image": 1},

901 )

902

903 if modality == "image":

@@ -925,7 +928,7 @@ def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:

928 model=model_name,

929 trust_remote_code=True,

930 max_model_len=4096,

928 - disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,

931 + limit_mm_per_prompt={"image": 1},

932 )

933

934 tokenizer = AutoTokenizer.from_pretrained(model_name,

@@ -1082,7 +1085,15 @@ def main(args):

1085

1086 req_data = model_example_map[model](questions, modality)

1087

1085 - engine_args = asdict(req_data.engine_args) | {"seed": args.seed}

1088 + # Disable other modalities to save memory

1089 + default_limits = {"image": 0, "video": 0, "audio": 0}

1090 + req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

1091 + req_data.engine_args.limit_mm_per_prompt or {})

1092 +

1093 + engine_args = asdict(req_data.engine_args) | {

1094 + "seed": args.seed,

1095 + "disable_mm_preprocessor_cache": args.disable_mm_preprocessor_cache,

1096 + }

1097 llm = LLM(**engine_args)

1098

1099 # To maintain code compatibility in this script, we add LoRA here.

examples/offline_inference/vision_language_embedding.py

@@ -63,6 +63,7 @@ def run_e5_v(query: Query) -> ModelRequestData:

63 model="royokong/e5-v",

64 task="embed",

65 max_model_len=4096,

66 + limit_mm_per_prompt={"image": 1},

67 )

69 return ModelRequestData(

@@ -93,6 +94,7 @@ def run_vlm2vec(query: Query) -> ModelRequestData:

94 task="embed",

95 trust_remote_code=True,

96 mm_processor_kwargs={"num_crops": 4},

97 + limit_mm_per_prompt={"image": 1},

98 )

100 return ModelRequestData(

@@ -131,6 +133,11 @@ def run_encode(model: str, modality: QueryModality, seed: Optional[int]):

133 query = get_query(modality)

134 req_data = model_example_map[model](query)

135

136 + # Disable other modalities to save memory

137 + default_limits = {"image": 0, "video": 0, "audio": 0}

138 + req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

139 + req_data.engine_args.limit_mm_per_prompt or {})

140 +

141 engine_args = asdict(req_data.engine_args) | {"seed": seed}

142 llm = LLM(**engine_args)

143

examples/offline_inference/vision_language_multi_image.py

@@ -687,6 +687,11 @@ def run_chat(model: str, question: str, image_urls: list[str],

687 seed: Optional[int]):

688 req_data = model_example_map[model](question, image_urls)

689

690 + # Disable other modalities to save memory

691 + default_limits = {"image": 0, "video": 0, "audio": 0}

692 + req_data.engine_args.limit_mm_per_prompt = default_limits | dict(

693 + req_data.engine_args.limit_mm_per_prompt or {})

694 +

695 engine_args = asdict(req_data.engine_args) | {"seed": seed}

696 llm = LLM(**engine_args)

697

tests/entrypoints/openai/test_audio.py

+34 -27

@@ -12,7 +12,9 @@ from ...utils import RemoteOpenAIServer

12 MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"

13 TEST_AUDIO_URLS = [

14 AudioAsset("winning_call").url,

15 + AudioAsset("mary_had_lamb").url,

16 ]

17 +MAXIMUM_AUDIOS = 2

20 @pytest.fixture(scope="module")

@@ -24,6 +26,8 @@ def server():

26 "5",

27 "--enforce-eager",

28 "--trust-remote-code",

29 + "--limit-mm-per-prompt",

30 + f"audio={MAXIMUM_AUDIOS}",

31 ]

33 with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

@@ -46,7 +50,7 @@ def base64_encoded_audio() -> dict[str, str]:

51 @pytest.mark.asyncio

52 @pytest.mark.parametrize("model_name", [MODEL_NAME])

49 -@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

53 +@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])

54 async def test_single_chat_session_audio(client: openai.AsyncOpenAI,

55 model_name: str, audio_url: str):

56 messages = [{

@@ -100,7 +104,7 @@ async def test_single_chat_session_audio(client: openai.AsyncOpenAI,

104

105 @pytest.mark.asyncio

106 @pytest.mark.parametrize("model_name", [MODEL_NAME])

103 -@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

107 +@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])

108 async def test_single_chat_session_audio_base64encoded(

109 client: openai.AsyncOpenAI, model_name: str, audio_url: str,

110 base64_encoded_audio: dict[str, str]):

@@ -158,7 +162,7 @@ async def test_single_chat_session_audio_base64encoded(

162

163 @pytest.mark.asyncio

164 @pytest.mark.parametrize("model_name", [MODEL_NAME])

161 -@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

165 +@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])

166 async def test_single_chat_session_input_audio(

167 client: openai.AsyncOpenAI, model_name: str, audio_url: str,

168 base64_encoded_audio: dict[str, str]):

@@ -330,28 +334,21 @@ async def test_chat_streaming_input_audio(client: openai.AsyncOpenAI,

334

335 @pytest.mark.asyncio

336 @pytest.mark.parametrize("model_name", [MODEL_NAME])

333 -@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)

337 +@pytest.mark.parametrize(

338 + "audio_urls", [TEST_AUDIO_URLS, TEST_AUDIO_URLS + [TEST_AUDIO_URLS[0]]])

339 async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,

335 - audio_url: str,

336 - base64_encoded_audio: dict[str, str]):

340 + audio_urls: list[str]):

341

342 messages = [{

343 "role":

344 "user",

345 "content": [

342 - {

346 + *({

347 "type": "audio_url",

348 "audio_url": {

349 "url": audio_url

350 }

347 - },

348 - {

349 - "type": "input_audio",

350 - "input_audio": {

351 - "data": base64_encoded_audio[audio_url],

352 - "format": "wav"

353 - }

354 - },

351 + } for audio_url in audio_urls),

352 {

353 "type": "text",

354 "text": "What's happening in this audio?"

@@ -359,20 +356,30 @@ async def test_multi_audio_input(client: openai.AsyncOpenAI, model_name: str,

356 ],

357 }]

358

362 - with pytest.raises(openai.BadRequestError): # test multi-audio input

363 - await client.chat.completions.create(

359 + if len(audio_urls) > MAXIMUM_AUDIOS:

360 + with pytest.raises(openai.BadRequestError): # test multi-audio input

361 + await client.chat.completions.create(

362 + model=model_name,

363 + messages=messages,

364 + max_completion_tokens=10,

365 + temperature=0.0,

366 + )

367 +

368 + # the server should still work afterwards

369 + completion = await client.completions.create(

370 + model=model_name,

371 + prompt=[0, 0, 0, 0, 0],

372 + max_tokens=5,

373 + temperature=0.0,

374 + )

375 + completion = completion.choices[0].text

376 + assert completion is not None and len(completion) >= 0

377 + else:

378 + chat_completion = await client.chat.completions.create(

379 model=model_name,

380 messages=messages,

381 max_completion_tokens=10,

382 temperature=0.0,

383 )

369 -

370 - # the server should still work afterwards

371 - completion = await client.completions.create(

372 - model=model_name,

373 - prompt=[0, 0, 0, 0, 0],

374 - max_tokens=5,

375 - temperature=0.0,

376 - )

377 - completion = completion.choices[0].text

378 - assert completion is not None and len(completion) >= 0

384 + message = chat_completion.choices[0].message

385 + assert message.content is not None and len(message.content) >= 0

tests/models/decoder_only/vision_language/vlm_utils/core.py

@@ -51,6 +51,10 @@ def run_test(

51 model_info.check_available_online(on_fail="skip")

52 model_info.check_transformers_version(on_fail="skip")

54 + # Disable other modalities to save memory

55 + default_limits = {"image": 0, "video": 0, "audio": 0}

56 + limit_mm_per_prompt = default_limits | limit_mm_per_prompt

57 +

58 vllm_outputs_per_mm = []

59 hf_outputs_per_mm = []

tests/models/test_oot_registration.py

@@ -90,6 +90,7 @@ def test_oot_registration_multimodal(

90 max_model_len=4096,

91 enforce_eager=True,

92 limit_mm_per_prompt={"image": 1})

93 +

94 first_token = llm.get_tokenizer().decode(0)

95 outputs = llm.generate(prompts, sampling_params)

tests/multimodal/test_processing.py

+5 -2

@@ -972,10 +972,13 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):

972 if is_valid:

973 exc_ctx = nullcontext()

974 else:

975 - exc_ctx = pytest.raises(ValueError, match="this model only supports")

975 + exc_ctx = pytest.raises(ValueError, match="The model only supports")

976

977 with exc_ctx:

978 - profiler.get_decoder_dummy_data(model_config.max_model_len)

978 + profiler.get_decoder_dummy_data(

979 + model_config.max_model_len,

980 + mm_counts=limit_mm_per_prompt,

981 + )

982

983

984 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])

vllm/config.py

+9 -3

@@ -2667,14 +2667,20 @@ class MultiModalConfig:

2667 usedforsecurity=False).hexdigest()

2668 return hash_str

2669

2670 + def get_default_limit_per_prompt(self) -> int:

2671 + """

2672 + Return the default number of input items allowed per prompt

2673 + for any modality if not specified by the user.

2674 + """

2675 + return 999 if envs.VLLM_USE_V1 else 1

2676 +

2677 def get_limit_per_prompt(self, modality: str) -> int:

2678 """

2679 Get the maximum number of input items allowed per prompt

2680 for the given modality.

2674 -

2675 - If not set by the user, this defaults to `1`.

2681 """

2677 - return self.limit_per_prompt.get(modality, 1)

2682 + default = self.get_default_limit_per_prompt()

2683 + return self.limit_per_prompt.get(modality, default)

2684

2685 # TODO: Add configs to init vision tower or not.

2686

vllm/engine/arg_utils.py

+3 -3

@@ -671,13 +671,13 @@ class EngineArgs:

671 type=nullable_kvs,

672 default=EngineArgs.limit_mm_per_prompt,

673 # The default value is given in

674 - # MultiModalConfig.get_limit_per_prompt

674 + # MultiModalConfig.get_default_limit_per_prompt

675 help=('For each multimodal plugin, limit how many '

676 'input instances to allow for each prompt. '

677 'Expects a comma-separated list of items, '

678 'e.g.: `image=16,video=2` allows a maximum of 16 '

679 - 'images and 2 videos per prompt. Defaults to 1 for '

680 - 'each modality.'))

679 + 'images and 2 videos per prompt. Defaults to '

680 + '1 (V0) or 999 (V1) for each modality.'))

681 parser.add_argument(

682 '--mm-processor-kwargs',

683 default=None,

vllm/entrypoints/chat_utils.py

+24 -5

@@ -35,7 +35,7 @@ from typing_extensions import Required, TypeAlias, TypedDict

36 from vllm.config import ModelConfig

37 from vllm.logger import init_logger

38 -from vllm.multimodal import MultiModalDataDict

38 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

39 from vllm.multimodal.utils import MediaConnector

40 from vllm.transformers_utils.processor import cached_get_processor

41 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

@@ -452,8 +452,6 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

452

453 self._model_config = model_config

454 self._tokenizer = tokenizer

455 - self._allowed_items = (model_config.multimodal_config.limit_per_prompt

456 - if model_config.multimodal_config else {})

455

456 self._items_by_modality = defaultdict[str, list[_T]](list)

457

@@ -465,6 +463,10 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

463 def allowed_local_media_path(self):

464 return self._model_config.allowed_local_media_path

465

466 + @property

467 + def mm_registry(self):

468 + return MULTIMODAL_REGISTRY

469 +

470 @staticmethod

471 @cache

472 def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:

@@ -540,12 +542,29 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

542 Add a multi-modal item to the current prompt and returns the

543 placeholder string to use, if any.

544 """

543 - allowed_count = self._allowed_items.get(modality, 1)

545 + mm_registry = self.mm_registry

546 + model_config = self.model_config

547 +

548 + input_modality = modality.replace("_embeds", "")

549 +

550 + if mm_registry.has_processor(model_config):

551 + mm_processor = mm_registry.create_processor(model_config)

552 + allowed_counts = mm_processor.info.get_allowed_mm_limits()

553 + allowed_count = allowed_counts.get(input_modality, 0)

554 + else:

555 + mm_config = model_config.multimodal_config

556 + if mm_config is None:

557 + msg = "This model does not support multi-modal inputs"

558 + raise ValueError(msg)

559 +

560 + allowed_count = mm_config.get_limit_per_prompt(input_modality)

561 +

562 current_count = len(self._items_by_modality[modality]) + 1

563 if current_count > allowed_count:

564 raise ValueError(

565 f"At most {allowed_count} {modality}(s) may be provided in "

548 - "one request.")

566 + "one request. You can set `--limit-mm-per-prompt` to "

567 + "increase this limit if the model supports it.")

568

569 self._items_by_modality[modality].append(item)

570

vllm/model_executor/models/minicpmo.py

+1 -1

@@ -126,7 +126,7 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):

126 def _parse_audio_data(

127 self,

128 data: Union[dict[str, torch.Tensor], ModalityData[AudioItem]],

129 - ) -> ModalityDataItems[Any, Any]:

129 + ) -> Optional[ModalityDataItems[Any, Any]]:

130 if isinstance(data, dict):

131 return MiniCPMOAudioEmbeddingItems(

132 data,

vllm/model_executor/models/minicpmv.py

+2 -2

@@ -290,7 +290,7 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):

290 def _parse_image_data(

291 self,

292 data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],

293 - ) -> ModalityDataItems[Any, Any]:

293 + ) -> Optional[ModalityDataItems[Any, Any]]:

294 if isinstance(data, dict):

295 return MiniCPMVImageEmbeddingItems(

296 data,

@@ -302,7 +302,7 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):

302 def _parse_video_data(

303 self,

304 data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],

305 - ) -> ModalityDataItems[Any, Any]:

305 + ) -> Optional[ModalityDataItems[Any, Any]]:

306 if isinstance(data, dict):

307 return MiniCPMVVideoEmbeddingItems(

308 data,

vllm/model_executor/models/qwen2_vl.py

+2 -2

@@ -720,7 +720,7 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):

720 def _parse_image_data(

721 self,

722 data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],

723 - ) -> ModalityDataItems[Any, Any]:

723 + ) -> Optional[ModalityDataItems[Any, Any]]:

724 if isinstance(data, dict):

725 return DictEmbeddingItems(

726 data,

@@ -734,7 +734,7 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):

734 def _parse_video_data(

735 self,

736 data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],

737 - ) -> ModalityDataItems[Any, Any]:

737 + ) -> Optional[ModalityDataItems[Any, Any]]:

738 if isinstance(data, dict):

739 return DictEmbeddingItems(

740 data,

vllm/multimodal/processing.py

+29 -5

@@ -1034,6 +1034,20 @@ class BaseProcessingInfo:

1034 """

1035 raise NotImplementedError

1036

1037 + def get_allowed_mm_limits(self) -> Mapping[str, int]:

1038 + """Return the maximum allowed number of items for each modality."""

1039 + supported_mm_limits = self.get_supported_mm_limits()

1040 + mm_config = self.ctx.get_mm_config()

1041 +

1042 + allowed_limits = dict[str, int]()

1043 + for modality, supported_limit in supported_mm_limits.items():

1044 + user_limit = mm_config.get_limit_per_prompt(modality)

1045 +

1046 + allowed_limits[modality] = (user_limit if supported_limit is None

1047 + else min(user_limit, supported_limit))

1048 +

1049 + return allowed_limits

1050 +

1051

1052 _I = TypeVar("_I", bound=BaseProcessingInfo)

1053

@@ -1087,14 +1101,24 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1101 before passing them to :meth:`_get_hf_mm_data`.

1102 """

1103 mm_items = self.data_parser.parse_mm_data(mm_data)

1090 - mm_config = self.info.ctx.get_mm_config()

1104 + supported_mm_limits = self.info.get_supported_mm_limits()

1105 + allowed_mm_limits = self.info.get_allowed_mm_limits()

1106

1107 for modality, items in mm_items.items():

1093 - limit = mm_config.get_limit_per_prompt(modality)

1094 - if len(items) > limit:

1108 + supported_limit = supported_mm_limits.get(modality, 0)

1109 + allowed_limit = allowed_mm_limits.get(modality, 0)

1110 + num_items = len(items)

1111 +

1112 + if supported_limit is not None and num_items > supported_limit:

1113 + raise ValueError(

1114 + f"The model only supports at most {supported_limit} "

1115 + f"{modality} items, but you passed {num_items} "

1116 + f"{modality} items in the same prompt.")

1117 +

1118 + if num_items > allowed_limit:

1119 raise ValueError(

1096 - f"You set {modality}={limit} (or defaulted to 1) in "

1097 - f"`--limit-mm-per-prompt`, but passed {len(items)} "

1120 + f"You set or defaulted to {modality}={allowed_limit} "

1121 + f"in --limit-mm-per-prompt`, but passed {num_items} "

1122 f"{modality} items in the same prompt.")

1123

1124 return mm_items

vllm/multimodal/profiling.py

+1 -17

@@ -162,23 +162,7 @@ class MultiModalProfiler(Generic[_I]):

162 return self.processor.dummy_inputs

163

164 def get_mm_limits(self) -> Mapping[str, int]:

165 - mm_config = self.processing_info.ctx.get_mm_config()

166 - supported_mm_limits = self.processing_info.get_supported_mm_limits()

167 -

168 - mm_limits = {

169 - modality: mm_config.get_limit_per_prompt(modality)

170 - for modality in supported_mm_limits

171 - }

172 -

173 - for modality, supported_limit in supported_mm_limits.items():

174 - limit = mm_limits[modality]

175 - if supported_limit is not None and supported_limit < limit:

176 - raise ValueError(

177 - f"You set {modality}={limit} (or defaulted to 1) in "

178 - f"`--limit-mm-per-prompt`, but this model only supports "

179 - f"at most {supported_limit} {modality} items.")

180 -

181 - return mm_limits

165 + return self.processing_info.get_allowed_mm_limits()

166

167 def _get_dummy_mm_inputs(

168 self,

vllm/multimodal/registry.py

+4 -2

@@ -265,8 +265,10 @@ class MultiModalRegistry:

265

266 return profiler.get_mm_max_tokens(

267 seq_len,

268 - {modality: 1

269 - for modality in mm_limits},

268 + {

269 + modality: 1

270 + for modality, limit in mm_limits.items() if limit > 0

271 + },

272 )

273

274 return {

vllm/multimodal/utils.py

+1 -1

@@ -264,7 +264,7 @@ fetch_video = global_media_connector.fetch_video

264

265 def encode_audio_base64(

266 audio: np.ndarray,

267 - sampling_rate: int,

267 + sampling_rate: float,

268 ) -> str:

269 """Encode audio as base64."""

270 audio_io = AudioMediaIO()