A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 22:30 UTC
[Rust Frontend] Skip loading multimodal processor if `--language-model-only` is specified (#44500)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Jun 5, 2026 at 08:02 UTC
62d6f06e3db276030ae46b2e0b3915674515ca74
8 files changed
+87
-8
rust/src/chat/src/backend/hf.rs
+60
-7
@@ -38,13 +38,17 @@ impl HfChatBackend {
38
) -> Result<Self> {
39
let model_config = load_model_config(files.config_path.as_deref())?;
40
let model_type = model_config.model_type().unwrap_or_default();
41
- let multimodal_model_info = MultimodalModelInfo::from_paths(
42
- model_id.clone(),
43
- (!model_type.is_empty()).then_some(model_type.to_string()),
44
- files.config_path.as_deref(),
45
- files.preprocessor_config_path.as_deref(),
46
- tokenizer.clone(),
47
- )?;
41
+ let multimodal_model_info = if options.language_model_only {
42
+ None
43
+ } else {
44
+ MultimodalModelInfo::from_paths(
45
+ model_id.clone(),
46
+ (!model_type.is_empty()).then_some(model_type.to_string()),
47
+ files.config_path.as_deref(),
48
+ files.preprocessor_config_path.as_deref(),
49
+ tokenizer.clone(),
50
+ )?
51
+ };
52
let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());
53
54
let renderer = options.renderer.resolve(model_type);
@@ -225,6 +229,7 @@ mod tests {
229
"test-model".to_string(),
230
LoadModelBackendsOptions {
231
renderer,
232
+ language_model_only: false,
233
chat_template_content_format: Default::default(),
234
chat_template: None,
235
default_chat_template_kwargs: HashMap::new(),
@@ -267,6 +272,54 @@ mod tests {
272
assert_eq!(prompt, "hello");
273
}
274
275
+ #[test]
276
+ fn language_model_only_skips_multimodal_preprocessor_config() {
277
+ let mut files = resolved_files(
278
+ r#"{"model_type":"deepseek_v0_vl"}"#,
279
+ r#"{"chat_template":"{{ messages[0].content }}"}"#,
280
+ );
281
+ let preprocessor_config_path = files
282
+ .config_path
283
+ .as_ref()
284
+ .unwrap()
285
+ .parent()
286
+ .unwrap()
287
+ .join("preprocessor_config.json");
288
+ write_json(&preprocessor_config_path, r#"{"size":[672,672]}"#);
289
+ files.preprocessor_config_path = Some(preprocessor_config_path);
290
+
291
+ let backend = HfChatBackend::from_resolved_model_files(
292
+ files.clone(),
293
+ "test-model".to_string(),
294
+ LoadModelBackendsOptions {
295
+ language_model_only: true,
296
+ chat_template_content_format: Default::default(),
297
+ chat_template: None,
298
+ default_chat_template_kwargs: HashMap::new(),
299
+ ..Default::default()
300
+ },
301
+ test_tokenizer(),
302
+ )
303
+ .unwrap();
304
+
305
+ assert!(backend.multimodal_model_info().is_none());
306
+
307
+ let error = HfChatBackend::from_resolved_model_files(
308
+ files,
309
+ "test-model".to_string(),
310
+ LoadModelBackendsOptions {
311
+ chat_template_content_format: Default::default(),
312
+ chat_template: None,
313
+ default_chat_template_kwargs: HashMap::new(),
314
+ ..Default::default()
315
+ },
316
+ test_tokenizer(),
317
+ )
318
+ .err()
319
+ .expect("invalid preprocessor config should fail without language_model_only");
320
+ assert!(error.to_string().contains("failed to parse preprocessor_config.json"));
321
+ }
322
+
323
#[test]
324
fn explicit_deepseek_renderer_overrides_generic_model_type() {
325
let prompt = render_prompt(
rust/src/chat/src/backend/mod.rs
+3
@@ -60,6 +60,9 @@ pub type DynChatTextBackend = Arc<dyn ChatTextBackend>;
60
pub struct LoadModelBackendsOptions {
61
/// Which chat renderer implementation to use.
62
pub renderer: RendererSelection,
63
+ /// Disable frontend-side multimodal preprocessing and render the model as
64
+ /// language-only.
65
+ pub language_model_only: bool,
66
/// How to serialize `message.content` when rendering the chat template.
67
pub chat_template_content_format: ChatTemplateContentFormatOption,
68
/// Optional server-default chat template override, provided either as an
rust/src/cmd/src/cli.rs
+7
@@ -116,6 +116,10 @@ pub struct SharedRuntimeArgs {
116
#[arg(long = "tokenizer-mode", default_value_t)]
117
#[serde(default, rename = "tokenizer_mode")]
118
pub renderer: RendererSelection,
119
+ /// Disable multimodal inputs and treat the model as language-only.
120
+ #[arg(long)]
121
+ #[serde(default)]
122
+ pub language_model_only: bool,
123
/// Override the maximum model context length. When set, the frontend uses
124
/// this value instead of the model's `max_position_embeddings` from
125
/// `config.json`.
@@ -243,6 +247,7 @@ impl SharedRuntimeArgs {
247
tool_call_parser: self.tool_call_parser,
248
reasoning_parser: self.reasoning_parser,
249
renderer: self.renderer,
250
+ language_model_only: self.language_model_only,
251
chat_template: self.chat_template,
252
default_chat_template_kwargs: self.default_chat_template_kwargs,
253
chat_template_content_format: self.chat_template_content_format,
@@ -284,6 +289,7 @@ impl SharedRuntimeArgs {
289
tool_call_parser: self.tool_call_parser,
290
reasoning_parser: self.reasoning_parser,
291
renderer: self.renderer,
292
+ language_model_only: self.language_model_only,
293
chat_template: self.chat_template,
294
default_chat_template_kwargs: self.default_chat_template_kwargs,
295
chat_template_content_format: self.chat_template_content_format,
@@ -419,6 +425,7 @@ impl ServeArgs {
425
self.managed_engine.clone().into_config(
426
self.runtime.model.clone(),
427
self.runtime.max_model_len,
428
+ self.runtime.language_model_only,
429
handshake_port,
430
)
431
}
rust/src/cmd/src/cli/tests.rs
+8
-1
@@ -34,6 +34,7 @@ fn serve_args_forward_python_flags_with_separator() {
34
tool_call_parser: Auto,
35
reasoning_parser: Auto,
36
renderer: Auto,
37
+ language_model_only: false,
38
max_model_len: Some(
39
512,
40
),
@@ -263,6 +264,7 @@ fn frontend_args_accept_json() {
264
tool_call_parser: Auto,
265
reasoning_parser: Auto,
266
renderer: Auto,
267
+ language_model_only: false,
268
max_model_len: None,
269
grpc_port: None,
270
shutdown_timeout: 0,
@@ -321,7 +323,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
323
"--output-address",
324
"ipc:///tmp/output.sock",
325
"--args-json",
324
- r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","max_model_len":8192,"shutdown_timeout":3}"#,
326
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"shutdown_timeout":3}"#,
327
])
328
.unwrap();
329
@@ -338,6 +340,7 @@ fn frontend_args_json_accepts_supported_non_default_fields() {
340
ParserSelection::Explicit("qwen3_thinking".to_string())
341
);
342
assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
343
+ assert!(args.runtime.language_model_only);
344
assert_eq!(args.runtime.max_model_len, Some(8192));
345
assert_eq!(args.runtime.shutdown_timeout, 3);
346
}
@@ -662,6 +665,7 @@ fn serve_args_accept_handshake_aliases() {
665
tool_call_parser: Auto,
666
reasoning_parser: Auto,
667
renderer: Auto,
668
+ language_model_only: false,
669
max_model_len: None,
670
grpc_port: None,
671
shutdown_timeout: 0,
@@ -783,6 +787,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
787
tool_call_parser: Auto,
788
reasoning_parser: Auto,
789
renderer: Auto,
790
+ language_model_only: false,
791
chat_template: None,
792
default_chat_template_kwargs: None,
793
chat_template_content_format: Auto,
@@ -846,6 +851,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
851
tool_call_parser: Auto,
852
reasoning_parser: Auto,
853
renderer: Auto,
854
+ language_model_only: false,
855
chat_template: None,
856
default_chat_template_kwargs: None,
857
chat_template_content_format: Auto,
@@ -924,6 +930,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
930
tool_call_parser: Auto,
931
reasoning_parser: Auto,
932
renderer: Auto,
933
+ language_model_only: false,
934
chat_template: None,
935
default_chat_template_kwargs: None,
936
chat_template_content_format: Auto,
rust/src/managed-engine/src/cli.rs
+4
@@ -71,6 +71,7 @@ impl ManagedEngineArgs {
71
self,
72
model: String,
73
max_model_len: Option<u32>,
74
+ language_model_only: bool,
75
handshake_port: u16,
76
) -> ManagedEngineConfig {
77
let mut python_args = self.python_args;
@@ -79,6 +80,9 @@ impl ManagedEngineArgs {
80
python_args.push("--max-model-len".to_string());
81
python_args.push(max_model_len.to_string());
82
}
83
+ if language_model_only {
84
+ python_args.push("--language-model-only".to_string());
85
+ }
86
if let Some(data_parallel_size_local) = self.data_parallel_size_local {
87
python_args.push("--data-parallel-size-local".to_string());
88
python_args.push(data_parallel_size_local.to_string());
rust/src/server/examples/external_engine_openai_qwen.rs
+1
@@ -64,6 +64,7 @@ async fn main() -> Result<()> {
64
tool_call_parser: ParserSelection::Auto,
65
reasoning_parser: ParserSelection::Auto,
66
renderer: RendererSelection::Auto,
67
+ language_model_only: false,
68
chat_template: None,
69
default_chat_template_kwargs: None,
70
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
rust/src/server/src/config.rs
+3
@@ -53,6 +53,9 @@ pub struct Config {
53
pub reasoning_parser: ParserSelection,
54
/// Chat renderer selection.
55
pub renderer: RendererSelection,
56
+ /// Disable frontend-side multimodal preprocessing and render the model as
57
+ /// language-only.
58
+ pub language_model_only: bool,
59
/// Server-default chat template override, as a file path or inline
60
/// template.
61
pub chat_template: Option<String>,
rust/src/server/src/lib.rs
+1
@@ -42,6 +42,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
42
&config.model,
43
LoadModelBackendsOptions {
44
renderer: config.renderer,
45
+ language_model_only: config.language_model_only,
46
chat_template: config.chat_template.clone(),
47
chat_template_content_format: config.chat_template_content_format,
48
default_chat_template_kwargs: config