A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 15:15 UTC
[Doc] Update usage of `--limit-mm-per-prompt` (#34148)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 10, 2026 at 13:12 UTC
25e48a3aae35849fd777f8a48c3c494337c11d83
6 files changed
+10
-10
docs/features/multimodal_inputs.md
+1
-1
@@ -521,7 +521,7 @@ First, launch the OpenAI-compatible server:
521
522
```bash
523
vllm serve microsoft/Phi-3.5-vision-instruct --runner generate \
524
- --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'
524
+ --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt.image 2
525
```
526
527
Then, you can use the OpenAI client as follows:
docs/models/supported_models.md
+1
-1
@@ -658,7 +658,7 @@ On the other hand, modalities separated by `/` are mutually exclusive.
658
See [this page](../features/multimodal_inputs.md) on how to pass multi-modal inputs to the model.
659
660
!!! tip
661
- For hybrid-only models such as Llama-4, Step3 and Mistral-3, a text-only mode can be enabled by setting all supported multimodal modalities to 0 (e.g, `--limit-mm-per-prompt '{"image":0}`) so that their multimodal modules will not be loaded to free up more GPU memory for KV cache.
661
+ For hybrid-only models such as Llama-4, Step3, Mistral-3 and Qwen-3.5, a text-only mode can be enabled by setting all supported multimodal modalities to 0 (`--language-model-only`) so that their multimodal modules will not be loaded to free up more GPU memory for KV cache.
662
663
!!! note
664
vLLM currently supports adding LoRA adapters to the language backbone for most multimodal models. Additionally, vLLM now experimentally supports adding LoRA to the tower and connector modules for some multimodal models. See [this page](../features/lora.md).
examples/offline_inference/mistral-small.py
+2
-2
@@ -18,11 +18,11 @@ from vllm.assets.image import ImageAsset
18
# # Mistral format
19
# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
20
# --tokenizer-mode mistral --config-format mistral --load-format mistral \
21
-# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384
21
+# --limit-mm-per-prompt.image 4 --max-model-len 16384
22
#
23
# # HF format
24
# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
25
-# --limit-mm-per-prompt '{"image":4}' --max-model-len 16384
25
+# --limit-mm-per-prompt.image 4 --max-model-len 16384
26
# ```
27
#
28
# - Client:
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+1
-1
@@ -10,7 +10,7 @@ vllm serve llava-hf/llava-1.5-7b-hf
10
11
(multi-image inference with Phi-3.5-vision-instruct)
12
vllm serve microsoft/Phi-3.5-vision-instruct --runner generate \
13
- --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'
13
+ --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt.image 2
14
15
(audio inference with Ultravox)
16
vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b \
examples/pooling/classify/vision_classification_online.py
+1
-1
@@ -7,7 +7,7 @@ NOTE:
7
vllm serve muziyongshixin/Qwen2.5-VL-7B-for-VideoCls \
8
--runner pooling \
9
--max-model-len 5000 \
10
- --limit-mm-per-prompt '{"video": 1}' \
10
+ --limit-mm-per-prompt.video 1 \
11
--hf-overrides '{"text_config": {"architectures": ["Qwen2_5_VLForSequenceClassification"]}}'
12
"""
13
vllm/config/multimodal.py
+4
-4
@@ -55,12 +55,12 @@ class MultiModalConfig:
55
"""Controls the behavior of multimodal models."""
56
57
language_model_only: bool = False
58
- """If True, disables all multimodal inputs by setting all modality limits
59
- to 0. Equivalent to setting --limit-mm-per-prompt to 0 for every
60
- modality."""
58
+ """If True, disables all multimodal inputs by setting all modality limits to 0.
59
+ Equivalent to setting `--limit-mm-per-prompt` to 0 for every modality."""
60
limit_per_prompt: dict[str, DummyOptions] = Field(default_factory=dict)
61
"""The maximum number of input items and options allowed per
63
- prompt for each modality.
62
+ prompt for each modality.
63
+
64
Defaults to 999 for each modality.
65
66
Legacy format (count only):