@tduncan / Nvidia-Optimization-Stack-VLLM / commits / c44e191b01

[Rust Frontend] Add --limit-mm-per-prompt support (#49604)

Signed-off-by: Maria Guevara <kawaiiplush14@gmail.com>

Maria Guevara committed Jul 29, 2026 at 10:21 UTC c44e191b014db0619bd51921e94c86b901ab952e
14 files changed +413 -16
rust/src/chat/src/backend/hf.rs
+2
@@ -57,6 +57,7 @@ impl HfChatBackend {
57 processor_config: files.processor_config_path.as_deref(),
58 },
59 tokenizer.clone(),
60 + options.limit_mm_per_prompt.clone(),
61 )?
62 };
63 let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
@@ -231,6 +232,7 @@ mod tests {
232 chat_template_content_format: Default::default(),
233 chat_template: None,
234 default_chat_template_kwargs: HashMap::new(),
235 + limit_mm_per_prompt: HashMap::new(),
236 },
237 test_tokenizer(),
238 )
rust/src/chat/src/backend/mod.rs
+4 -1
@@ -8,7 +8,7 @@ use serde_json::Value;
8 use vllm_text::{DynTextBackend, TextBackend};
9
10 use crate::error::Result;
11 -use crate::multimodal::MultimodalModelInfo;
11 +use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
12 use crate::output::DynChatOutputProcessor;
13 use crate::renderer::DynChatRenderer;
14 use crate::request::ChatRequest;
@@ -74,6 +74,9 @@ pub struct LoadModelBackendsOptions {
74 /// Optional server-default keyword arguments merged into every
75 /// chat-template render before request-level `chat_template_kwargs`.
76 pub default_chat_template_kwargs: HashMap<String, Value>,
77 + /// Maximum number of input items allowed per prompt for each modality.
78 + /// Unspecified modalities are unlimited.
79 + pub limit_mm_per_prompt: MmLimitPerPrompt,
80 }
81
82 /// Shared backends loaded from a model id.
rust/src/chat/src/error.rs
+4 -1
@@ -23,6 +23,8 @@ pub enum Error {
23 UnsupportedMultimodalContent(&'static str),
24 #[error("`{modality}` input is not supported by this model")]
25 UnsupportedModality { modality: String },
26 + #[error("At most {limit} {modality}(s) may be provided in one prompt.")]
27 + MmLimitExceeded { modality: String, limit: usize },
28 #[error("multimodal preprocessing error: {0}")]
29 Multimodal(#[message] String),
30 #[error("{kind} parsing is not available for model `{model_id}`")]
@@ -87,7 +89,8 @@ impl Error {
89 Self::Text(error) => error.is_request_validation_error(),
90 Self::UnsupportedMultimodalRenderer
91 | Self::UnsupportedMultimodalContent(_)
90 - | Self::UnsupportedModality { .. } => true,
92 + | Self::UnsupportedModality { .. }
93 + | Self::MmLimitExceeded { .. } => true,
94
95 _ => false,
96 }
rust/src/chat/src/multimodal.rs
+262 -9
@@ -11,7 +11,7 @@
11 //! Raw media stays above `vllm-text`; this module lowers it into token IDs and
12 //! opaque tensor payloads before the request is handed to text generation.
13
14 -use std::collections::HashSet;
14 +use std::collections::{BTreeMap, HashMap, HashSet};
15 use std::fs;
16 use std::path::Path;
17 use std::sync::{Arc, LazyLock};
@@ -24,6 +24,7 @@ use llm_multimodal::{
24 PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
25 VisionProcessorRegistry,
26 };
27 +use serde::{Deserialize, Serialize};
28 use thiserror_ext::AsReport as _;
29 use tracing::warn;
30 use vllm_engine_core_client::protocol::dtype::ModelDtype;
@@ -52,6 +53,71 @@ pub struct MultimodalModelInfo {
53 video: Option<ModalitySupport>,
54 audio: Option<AudioModalitySupport>,
55 media_connector: Arc<MediaConnector>,
56 + /// Maximum number of input items allowed per prompt for each modality.
57 + limit_mm_per_prompt: MmLimitPerPrompt,
58 +}
59 +
60 +/// Per-modality item-count limits configured by `--limit-mm-per-prompt`.
61 +///
62 +/// Modalities absent from the map are unlimited.
63 +pub type MmLimitPerPrompt = HashMap<MmLimitModality, MmLimitSpec>;
64 +
65 +/// Modalities that `--limit-mm-per-prompt` can be keyed by.
66 +///
67 +/// Closed on purpose: these are exactly the keys Python accepts, per
68 +/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`.
69 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
70 +#[serde(rename_all = "snake_case")]
71 +pub enum MmLimitModality {
72 + Image,
73 + Audio,
74 + Video,
75 +}
76 +
77 +impl MmLimitModality {
78 + /// The wire name, matching Python's modality strings.
79 + pub fn as_str(self) -> &'static str {
80 + match self {
81 + Self::Image => "image",
82 + Self::Audio => "audio",
83 + Self::Video => "video",
84 + }
85 + }
86 +}
87 +
88 +/// One modality's limit, in either of the two shapes Python accepts.
89 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
90 +#[serde(
91 + untagged,
92 + expecting = "an item count, or an object with an optional `count` field"
93 +)]
94 +pub enum MmLimitSpec {
95 + /// Legacy form: `"image": 16`
96 + Count(usize),
97 +
98 + /// Configurable form:
99 + /// `"video": {"count": 1, "num_frames": 32}`
100 + Options {
101 + /// Absent means unlimited, matching an absent modality.
102 + #[serde(default, skip_serializing_if = "Option::is_none")]
103 + count: Option<usize>,
104 +
105 + /// Preserve Python-owned options for forwarding. Never interpreted
106 + /// here: they size the engine's dummy-profiling encoder cache, which
107 + /// has no Rust counterpart.
108 + #[serde(flatten)]
109 + extra: BTreeMap<String, serde_json::Value>,
110 + },
111 +}
112 +
113 +impl MmLimitSpec {
114 + /// The configured item count, or `None` when this modality is unlimited.
115 + pub fn count(&self) -> Option<usize> {
116 + match self {
117 + Self::Count(count) => Some(*count),
118 + Self::Options { count, .. } => *count,
119 + }
120 + }
121 }
122
123 /// Model metadata and tokenizer access shared by all multimodal specs.
@@ -287,6 +353,7 @@ impl MultimodalModelInfo {
353 model_type: Option<String>,
354 files: MultimodalConfigFiles<'_>,
355 tokenizer: DynTokenizer,
356 + limit_mm_per_prompt: MmLimitPerPrompt,
357 ) -> Result<Option<Self>> {
358 let config = match files.config {
359 Some(path) => {
@@ -319,7 +386,12 @@ impl MultimodalModelInfo {
386 tokenizer: TokenizerResolver(tokenizer),
387 };
388
322 - Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
389 + Self::from_loaded(
390 + context,
391 + preprocessor_config,
392 + video_preprocessor_config,
393 + limit_mm_per_prompt,
394 + )
395 }
396
397 /// Resolve multimodal support from an assembled context and parsed
@@ -328,6 +400,7 @@ impl MultimodalModelInfo {
400 context: MultimodalModelContext,
401 preprocessor_config: PreProcessorConfig,
402 video_preprocessor_config: PreProcessorConfig,
403 + limit_mm_per_prompt: MmLimitPerPrompt,
404 ) -> Result<Option<Self>> {
405 let (image, video) = Self::resolve_vision_lanes(
406 &context,
@@ -356,6 +429,7 @@ impl MultimodalModelInfo {
429 video,
430 audio,
431 media_connector,
432 + limit_mm_per_prompt,
433 }))
434 }
435
@@ -567,7 +641,55 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
641 Ok(format!("data:{mime_type};base64,{data}"))
642 }
643
644 +/// The modality a content part counts against, or `None` for plain text.
645 +///
646 +/// Embedding inputs share their base modality's budget rather than getting one
647 +/// of their own, matching Python's `modality.replace("_embeds", "")` in
648 +/// `vllm/entrypoints/chat_utils.py`.
649 +fn media_part_limit_modality(part: &MediaContentPart) -> Option<MmLimitModality> {
650 + match part {
651 + MediaContentPart::Text { .. } => None,
652 + MediaContentPart::ImageUrl { .. }
653 + | MediaContentPart::ImageData { .. }
654 + | MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image),
655 + MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => {
656 + Some(MmLimitModality::Audio)
657 + }
658 + MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => {
659 + Some(MmLimitModality::Video)
660 + }
661 + }
662 +}
663 +
664 impl MultimodalModelInfo {
665 + /// Reject requests exceeding `--limit-mm-per-prompt`'s configured
666 + /// per-modality item count, before any fetch/decode work is spent on them.
667 + ///
668 + /// Modalities without a configured count are unlimited.
669 + fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
670 + let mut counts: HashMap<MmLimitModality, usize> = HashMap::new();
671 + for part in media_parts {
672 + if let Some(modality) = media_part_limit_modality(part) {
673 + *counts.entry(modality).or_default() += 1;
674 + }
675 + }
676 +
677 + for (modality, count) in counts {
678 + let Some(limit) = self.limit_mm_per_prompt.get(&modality).and_then(MmLimitSpec::count)
679 + else {
680 + continue;
681 + };
682 + if count > limit {
683 + return Err(Error::MmLimitExceeded {
684 + modality: modality.as_str().to_string(),
685 + limit,
686 + });
687 + }
688 + }
689 +
690 + Ok(())
691 + }
692 +
693 /// Run media fetch, per-modality preprocessing, prompt expansion, and
694 /// feature build.
695 ///
@@ -584,8 +706,7 @@ impl MultimodalModelInfo {
706 if media_parts_len == 0 {
707 return Ok(Vec::new());
708 }
587 - // TODO: enforce per-modality item-count limits, aligned with the
588 - // engine's `--limit-mm-per-prompt` semantics.
709 + self.validate_mm_limits(&media_parts)?;
710 let fetched = self.fetch_media(media_parts).await?;
711
712 let mut prepared = Vec::new();
@@ -753,10 +874,11 @@ mod tests {
874 .with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID)
875 }
876
756 - fn test_info(
877 + fn test_info_with_limits(
878 model_type: &str,
879 config: serde_json::Value,
880 tokenizer: TestTokenizer,
881 + limit_mm_per_prompt: MmLimitPerPrompt,
882 ) -> MultimodalModelInfo {
883 let context = MultimodalModelContext {
884 model_id: format!("{model_type}-test"),
@@ -769,11 +891,20 @@ mod tests {
891 context,
892 PreProcessorConfig::default(),
893 PreProcessorConfig::default(),
894 + limit_mm_per_prompt,
895 )
896 .unwrap()
897 .unwrap_or_else(|| panic!("{model_type} multimodal support should resolve"))
898 }
899
900 + fn test_info(
901 + model_type: &str,
902 + config: serde_json::Value,
903 + tokenizer: TestTokenizer,
904 + ) -> MultimodalModelInfo {
905 + test_info_with_limits(model_type, config, tokenizer, HashMap::new())
906 + }
907 +
908 fn llama4_info() -> MultimodalModelInfo {
909 let config = serde_json::json!({
910 "model_type": "llama4",
@@ -783,16 +914,19 @@ mod tests {
914 test_info("llama4", config, llama4_tokenizer())
915 }
916
786 - pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
787 - let config = serde_json::json!({
917 + fn qwen3_vl_config() -> serde_json::Value {
918 + serde_json::json!({
919 "model_type": "qwen3_vl",
920 "image_token_id": QWEN3_IMAGE_PAD_ID,
921 "video_token_id": QWEN3_VIDEO_PAD_ID,
922 "vision_start_token_id": 151652,
923 "vision_end_token_id": 151653,
924 "vision_config": {"patch_size": 16}
794 - });
795 - test_info("qwen3_vl", config, qwen3_vl_tokenizer())
925 + })
926 + }
927 +
928 + pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
929 + test_info("qwen3_vl", qwen3_vl_config(), qwen3_vl_tokenizer())
930 }
931
932 #[test]
@@ -853,4 +987,123 @@ mod tests {
987 );
988 assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
989 }
990 +
991 + fn image_url_part() -> MediaContentPart {
992 + MediaContentPart::ImageUrl {
993 + url: "https://example.com/image.png".to_string(),
994 + detail: None,
995 + uuid: None,
996 + }
997 + }
998 +
999 + fn qwen3_vl_info_with_limits(limit_mm_per_prompt: MmLimitPerPrompt) -> MultimodalModelInfo {
1000 + test_info_with_limits(
1001 + "qwen3_vl",
1002 + qwen3_vl_config(),
1003 + qwen3_vl_tokenizer(),
1004 + limit_mm_per_prompt,
1005 + )
1006 + }
1007 +
1008 + #[test]
1009 + fn validate_mm_limits_ignores_text_parts() {
1010 + let info = qwen3_vl_info();
1011 + let parts = vec![
1012 + MediaContentPart::Text {
1013 + text: "hello".to_string(),
1014 + },
1015 + MediaContentPart::Text {
1016 + text: "world".to_string(),
1017 + },
1018 + ];
1019 + assert!(info.validate_mm_limits(&parts).is_ok());
1020 + }
1021 +
1022 + #[test]
1023 + fn validate_mm_limits_leaves_unconfigured_modalities_unlimited() {
1024 + let info = qwen3_vl_info();
1025 + let parts: Vec<_> = std::iter::repeat_with(image_url_part).take(1_000).collect();
1026 +
1027 + assert!(info.validate_mm_limits(&parts).is_ok());
1028 + }
1029 +
1030 + #[test]
1031 + fn validate_mm_limits_enforces_configured_limit_at_the_boundary() {
1032 + let info = qwen3_vl_info_with_limits(HashMap::from([(
1033 + MmLimitModality::Image,
1034 + MmLimitSpec::Count(1),
1035 + )]));
1036 + assert!(info.validate_mm_limits(&[image_url_part()]).is_ok());
1037 +
1038 + let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err();
1039 + assert_eq!(
1040 + error.to_report_string(),
1041 + "At most 1 image(s) may be provided in one prompt."
1042 + );
1043 + // Confirms the HTTP-mapping bug found during implementation stays fixed:
1044 + // this must map to 400, not 500.
1045 + assert!(error.is_request_validation_error());
1046 + }
1047 +
1048 + #[test]
1049 + fn validate_mm_limits_counts_image_embeds_against_the_image_limit() {
1050 + let info = qwen3_vl_info_with_limits(HashMap::from([(
1051 + MmLimitModality::Image,
1052 + MmLimitSpec::Count(1),
1053 + )]));
1054 + let image_embeds_part = MediaContentPart::ImageEmbeds {
1055 + payload: serde_json::Value::String("AAAA".to_string()),
1056 + uuid: None,
1057 + };
1058 +
1059 + let error = info.validate_mm_limits(&[image_url_part(), image_embeds_part]).unwrap_err();
1060 + assert_eq!(
1061 + error.to_report_string(),
1062 + "At most 1 image(s) may be provided in one prompt."
1063 + );
1064 + }
1065 +
1066 + /// An options object without a `count` carries only profiling keys, which
1067 + /// say nothing about how many items are allowed.
1068 + #[test]
1069 + fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() {
1070 + let info = qwen3_vl_info_with_limits(HashMap::from([(
1071 + MmLimitModality::Image,
1072 + MmLimitSpec::Options {
1073 + count: None,
1074 + extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]),
1075 + },
1076 + )]));
1077 +
1078 + assert!(info.validate_mm_limits(&[image_url_part(), image_url_part()]).is_ok());
1079 + }
1080 +
1081 + fn parse_limits(json: &str) -> MmLimitPerPrompt {
1082 + serde_json::from_str(json).expect("limit map should parse")
1083 + }
1084 +
1085 + #[test]
1086 + fn limit_map_parses_both_shapes_python_accepts() {
1087 + let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#);
1088 +
1089 + assert_eq!(limits[&MmLimitModality::Image].count(), Some(16));
1090 + assert_eq!(limits[&MmLimitModality::Video].count(), Some(1));
1091 + assert_eq!(limits.get(&MmLimitModality::Audio), None);
1092 + }
1093 +
1094 + #[test]
1095 + fn limit_map_rejects_keys_python_does_not_accept() {
1096 + assert!(serde_json::from_str::<MmLimitPerPrompt>(r#"{"image_embeds": 1}"#).is_err());
1097 + }
1098 +
1099 + /// Managed mode forwards this map back to Python as JSON, where
1100 + /// `BaseDummyOptions.count` is a non-optional `int` under `extra="forbid"`.
1101 + /// Emitting `"count": null` would make the engine subprocess fail to start.
1102 + #[test]
1103 + fn limit_map_round_trips_without_emitting_a_null_count() {
1104 + let source = r#"{"video":{"num_frames":32}}"#;
1105 + let encoded = serde_json::to_string(&parse_limits(source)).expect("map should serialize");
1106 +
1107 + assert_eq!(encoded, source);
1108 + }
1109 }
rust/src/chat/src/multimodal/audio.rs
+2
@@ -107,6 +107,7 @@ mod tests {
107 context,
108 PreProcessorConfig::default(),
109 PreProcessorConfig::default(),
110 + HashMap::new(),
111 )
112 .unwrap()
113 .expect("Inkling multimodal support")
@@ -126,6 +127,7 @@ mod tests {
127 context,
128 PreProcessorConfig::default(),
129 PreProcessorConfig::default(),
130 + HashMap::new(),
131 )
132 .unwrap()
133 .expect("Qwen3-ASR multimodal support")
rust/src/chat/src/multimodal/video.rs
+1
@@ -207,6 +207,7 @@ mod tests {
207 Some("qwen3_vl".to_string()),
208 files,
209 Arc::new(qwen3_vl_tokenizer()),
210 + std::collections::HashMap::new(),
211 )
212 };
213
rust/src/cmd/src/cli.rs
+27
@@ -23,6 +23,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
23 use thiserror_ext::AsReport as _;
24 use uuid::Uuid;
25 use vllm_chat::ReasoningParserFactory;
26 +use vllm_chat::multimodal::MmLimitPerPrompt;
27 use vllm_engine_core_client::TransportMode;
28 use vllm_managed_engine::ManagedEngineConfig;
29 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
@@ -186,6 +187,17 @@ pub struct SharedRuntimeArgs {
187 #[serde(default)]
188 pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
189
190 + /// The maximum number of input items allowed per prompt for each
191 + /// modality, as a JSON object (e.g. `{"image": 16, "video": 2}`).
192 + ///
193 + /// Also accepts the engine's configurable form
194 + /// (e.g. `{"video": {"count": 1, "num_frames": 32}}`); the extra
195 + /// profiling options are forwarded to the engine untouched.
196 + /// Unspecified modalities are unlimited.
197 + #[arg(long, value_parser = parse_json::<MmLimitPerPrompt>, value_name = "JSON", default_value = "{}")]
198 + #[serde(default)]
199 + pub limit_mm_per_prompt: MmLimitPerPrompt,
200 +
201 /// The format to render message content within a chat template.
202 ///
203 /// * "auto" detects the format from the template
@@ -348,6 +360,18 @@ impl SharedRuntimeArgs {
360 .expect("profiler config serialization should not fail")
361 }
362
363 + /// Return the per-modality limits as JSON for managed Python engine
364 + /// forwarding, or `None` when nothing is configured.
365 + ///
366 + /// Round-tripping the parsed map rather than the raw argument keeps the
367 + /// engine's own profiling options (`num_frames`, `width`, ...) intact.
368 + pub fn limit_mm_per_prompt_json(&self) -> Option<String> {
369 + (!self.limit_mm_per_prompt.is_empty()).then(|| {
370 + serde_json::to_string(&self.limit_mm_per_prompt)
371 + .expect("limit-mm-per-prompt serialization should not fail")
372 + })
373 + }
374 +
375 /// Apply fallback logic for API key configuration from env variables.
376 fn apply_env_api_key_fallback(&mut self) {
377 if self.api_key.is_empty()
@@ -399,6 +423,7 @@ impl SharedRuntimeArgs {
423 language_model_only: self.language_model_only,
424 chat_template: self.chat_template,
425 default_chat_template_kwargs: self.default_chat_template_kwargs,
426 + limit_mm_per_prompt: self.limit_mm_per_prompt,
427 chat_template_content_format: self.chat_template_content_format,
428 max_logprobs: self.max_logprobs,
429 api_server_options,
@@ -451,6 +476,7 @@ impl SharedRuntimeArgs {
476 language_model_only: self.language_model_only,
477 chat_template: self.chat_template,
478 default_chat_template_kwargs: self.default_chat_template_kwargs,
479 + limit_mm_per_prompt: self.limit_mm_per_prompt,
480 chat_template_content_format: self.chat_template_content_format,
481 max_logprobs: self.max_logprobs,
482 api_server_options,
@@ -666,6 +692,7 @@ impl ServeArgs {
692 self.runtime.disable_log_stats,
693 self.runtime.shutdown_timeout,
694 handshake_port,
695 + self.runtime.limit_mm_per_prompt_json(),
696 )
697 }
698 }
rust/src/cmd/src/cli/tests.rs
+24
@@ -64,6 +64,7 @@ fn serve_args_forward_python_flags_with_separator() {
64 http_timeout_keep_alive: None,
65 chat_template: None,
66 default_chat_template_kwargs: None,
67 + limit_mm_per_prompt: {},
68 chat_template_content_format: Auto,
69 enable_log_requests: false,
70 enable_prompt_tokens_details: false,
@@ -762,6 +763,7 @@ fn frontend_args_accept_json() {
763 http_timeout_keep_alive: None,
764 chat_template: None,
765 default_chat_template_kwargs: None,
766 + limit_mm_per_prompt: {},
767 chat_template_content_format: Auto,
768 enable_log_requests: false,
769 enable_prompt_tokens_details: false,
@@ -1117,6 +1119,24 @@ fn frontend_args_json_rejects_malformed_json() {
1119 "#]].assert_eq(&error.to_string());
1120 }
1121
1122 +#[test]
1123 +fn serve_args_reject_unsupported_modality_in_limit_mm_per_prompt() {
1124 + let error = Cli::try_parse_from([
1125 + "vllm-rs",
1126 + "serve",
1127 + "Qwen/Qwen3-0.6B",
1128 + "--limit-mm-per-prompt",
1129 + r#"{"unsupported_modality": 1}"#,
1130 + ])
1131 + .unwrap_err();
1132 +
1133 + expect![[r#"
1134 + error: invalid value '{"unsupported_modality": 1}' for '--limit-mm-per-prompt <JSON>': invalid JSON object: unknown variant `unsupported_modality`, expected one of `image`, `audio`, `video` at line 1 column 23
1135 +
1136 + For more information, try '--help'.
1137 + "#]].assert_eq(&error.to_string());
1138 +}
1139 +
1140 #[test]
1141 fn serve_args_reject_flags_before_model() {
1142 let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
@@ -1331,6 +1351,7 @@ fn serve_args_accept_handshake_aliases() {
1351 http_timeout_keep_alive: None,
1352 chat_template: None,
1353 default_chat_template_kwargs: None,
1354 + limit_mm_per_prompt: {},
1355 chat_template_content_format: Auto,
1356 enable_log_requests: false,
1357 enable_prompt_tokens_details: false,
@@ -1474,6 +1495,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1495 language_model_only: false,
1496 chat_template: None,
1497 default_chat_template_kwargs: None,
1498 + limit_mm_per_prompt: {},
1499 chat_template_content_format: Auto,
1500 max_logprobs: None,
1501 api_server_options: ApiServerOptions {
@@ -1558,6 +1580,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1580 language_model_only: false,
1581 chat_template: None,
1582 default_chat_template_kwargs: None,
1583 + limit_mm_per_prompt: {},
1584 chat_template_content_format: Auto,
1585 max_logprobs: None,
1586 api_server_options: ApiServerOptions {
@@ -1660,6 +1683,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
1683 language_model_only: false,
1684 chat_template: None,
1685 default_chat_template_kwargs: None,
1686 + limit_mm_per_prompt: {},
1687 chat_template_content_format: Auto,
1688 max_logprobs: None,
1689 api_server_options: ApiServerOptions {
rust/src/cmd/src/cli/unsupported.rs
-5
@@ -299,11 +299,6 @@ pub struct EngineUnsupportedArgs {
299 )]
300 pub kv_sharing_fast_prefill: Option<Unsupported>,
301
302 - /// The maximum number of input items and options allowed per
303 - /// prompt for each modality.
304 - #[arg(long)]
305 - pub limit_mm_per_prompt: Option<Unsupported>,
306 -
302 /// Additional args passed to process media inputs, keyed by modalities.
303 #[arg(long)]
304 pub media_io_kwargs: Option<Unsupported>,
rust/src/managed-engine/src/cli.rs
+5
@@ -91,6 +91,7 @@ impl ManagedEngineArgs {
91 disable_log_stats: bool,
92 shutdown_timeout: u64,
93 handshake_port: u16,
94 + limit_mm_per_prompt: Option<String>,
95 ) -> ManagedEngineConfig {
96 let mut python_args = self.python_args;
97 // Manually forward some args to the Python engine.
@@ -126,6 +127,10 @@ impl ManagedEngineArgs {
127 python_args.push("--data-parallel-size-local".to_string());
128 python_args.push(data_parallel_size_local.to_string());
129 }
130 + if let Some(limit_mm_per_prompt) = limit_mm_per_prompt {
131 + python_args.push("--limit-mm-per-prompt".to_string());
132 + python_args.push(limit_mm_per_prompt);
133 + }
134
135 ManagedEngineConfig {
136 python: self.python,
rust/src/server/examples/external_engine_openai_qwen.rs
+2
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +use std::collections::HashMap;
5 use std::time::Duration;
6
7 use anyhow::{Context, Result, bail};
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
71 language_model_only: false,
72 chat_template: None,
73 default_chat_template_kwargs: None,
74 + limit_mm_per_prompt: HashMap::new(),
75 chat_template_content_format: ChatTemplateContentFormatOption::Auto,
76 max_logprobs: None,
77 api_server_options: ApiServerOptions::default(),
rust/src/server/src/config.rs
+4
@@ -10,6 +10,7 @@ use axum::http::{HeaderName, HeaderValue, Method};
10 use educe::Educe;
11 use serde::Serialize;
12 use serde_json::Value;
13 +use vllm_chat::multimodal::MmLimitPerPrompt;
14 use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
15 use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
16
@@ -184,6 +185,9 @@ pub struct Config {
185 pub chat_template: Option<String>,
186 /// Server-default keyword arguments merged into every chat-template render.
187 pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
188 + /// Maximum number of input items allowed per prompt for each modality.
189 + /// Unspecified modalities are unlimited.
190 + pub limit_mm_per_prompt: MmLimitPerPrompt,
191 /// How to serialize `message.content` for chat-template rendering.
192 pub chat_template_content_format: ChatTemplateContentFormatOption,
193 /// Optional maximum number of top log probabilities accepted by the
rust/src/server/src/lib.rs
+1
@@ -101,6 +101,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
101 .default_chat_template_kwargs
102 .clone()
103 .unwrap_or_default(),
104 + limit_mm_per_prompt: config.limit_mm_per_prompt.clone(),
105 },
106 )
107 .await
rust/src/server/src/routes/tests.rs
+75
@@ -577,6 +577,12 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
577 }
578
579 fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
580 + qwen_multimodal_model_info_with_limits(std::collections::HashMap::new())
581 +}
582 +
583 +fn qwen_multimodal_model_info_with_limits(
584 + limit_mm_per_prompt: vllm_chat::multimodal::MmLimitPerPrompt,
585 +) -> vllm_chat::multimodal::MultimodalModelInfo {
586 let config_path = std::env::temp_dir().join(format!(
587 "vllm-server-qwen-config-{}.json",
588 uuid::Uuid::new_v4()
@@ -594,6 +600,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
600 ..Default::default()
601 },
602 Arc::new(fake_chat_tokenizer()),
603 + limit_mm_per_prompt,
604 )
605 .expect("load multimodal info")
606 .expect("qwen multimodal info is registered");
@@ -2307,6 +2314,74 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() {
2314 assert_eq!(json["choices"][0]["message"]["content"], "hi");
2315 }
2316
2317 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2318 +#[serial]
2319 +async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() {
2320 + // The request is rejected by `--limit-mm-per-prompt` validation before
2321 + // ever reaching the engine, so the mock engine task is never awaited.
2322 + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
2323 + b"engine-openai-mm-limit",
2324 + default_stream_output_specs(),
2325 + Arc::new(FakeChatBackend::with_multimodal_model_info(
2326 + qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([(
2327 + vllm_chat::multimodal::MmLimitModality::Image,
2328 + vllm_chat::multimodal::MmLimitSpec::Count(1),
2329 + )])),
2330 + )),
2331 + )
2332 + .await;
2333 + let app = build_router(Arc::new(AppState::new(
2334 + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
2335 + chat,
2336 + )));
2337 +
2338 + let response = app
2339 + .clone()
2340 + .call(
2341 + Request::builder()
2342 + .method("POST")
2343 + .uri("/v1/chat/completions")
2344 + .header("content-type", "application/json")
2345 + .body(Body::from(
2346 + json!({
2347 + "model": "Qwen/Qwen1.5-0.5B-Chat",
2348 + "stream": false,
2349 + "messages": [{
2350 + "role": "user",
2351 + "content": [
2352 + {"type": "text", "text": "describe "},
2353 + {
2354 + "type": "image_url",
2355 + "image_url": {
2356 + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
2357 + }
2358 + },
2359 + {
2360 + "type": "image_url",
2361 + "image_url": {
2362 + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
2363 + }
2364 + }
2365 + ]
2366 + }]
2367 + })
2368 + .to_string(),
2369 + ))
2370 + .expect("build request"),
2371 + )
2372 + .await
2373 + .expect("call app");
2374 +
2375 + assert_eq!(response.status(), StatusCode::BAD_REQUEST);
2376 + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
2377 + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
2378 + assert_eq!(json["error"]["type"], "invalid_request_error");
2379 + assert_eq!(
2380 + json["error"]["message"],
2381 + "At most 1 image(s) may be provided in one prompt."
2382 + );
2383 +}
2384 +
2385 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2386 #[serial]
2387 async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {