[Rust Frontend] Make --max-model-len optional for the render server (#55328)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Wen Zhou <wenzhou@redhat.com>
Wen Zhou committed
Sep 9, 2026 at 10:54 UTC
68dcc4fd86b11d1cfa228cfc2c9d2b9706ba7af3
5 files changed
+59
-8
rust/src/cmd/src/cli.rs
+3
-2
@@ -139,9 +139,10 @@ pub struct RenderArgs {
139
/// How message content is exposed to the chat template.
140
#[arg(long, default_value_t)]
141
chat_template_content_format: ChatTemplateContentFormatOption,
142
- /// Maximum model context length used for request validation.
142
+ /// Maximum model context length used for request validation. When not set,
143
+ /// prompt-length validation is skipped and the engine enforces its own limit in later stage.
144
#[arg(long)]
144
- max_model_len: u32,
145
+ max_model_len: Option<u32>,
146
/// Maximum accepted logprobs count; -1 disables the cap.
147
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
148
max_logprobs: Option<i32>,
rust/src/cmd/src/cli/tests.rs
+13
-1
@@ -57,7 +57,7 @@ fn render_args_build_config_without_tls() {
57
assert_eq!(config.model, "Qwen/Qwen2.5-0.5B-Instruct");
58
assert_eq!(config.host, "127.0.0.1");
59
assert_eq!(config.port, 8080);
60
- assert_eq!(config.max_model_len, 32768);
60
+ assert_eq!(config.max_model_len, Some(32768));
61
assert_eq!(config.served_model_name, ["qwen"]);
62
assert_eq!(config.tool_call_parser, ParserSelection::Auto);
63
assert_eq!(config.reasoning_parser, ParserSelection::Auto);
@@ -66,6 +66,18 @@ fn render_args_build_config_without_tls() {
66
assert!(config.tls.is_none());
67
}
68
69
+#[test]
70
+fn render_args_allow_omitted_max_model_len() {
71
+ let cli = Cli::try_parse_from(["vllm-rs", "render", "Qwen/Qwen2.5-0.5B-Instruct"]).unwrap();
72
+
73
+ let Command::Render(args) = cli.command else {
74
+ panic!("expected render args");
75
+ };
76
+ let config = args.into_config();
77
+
78
+ assert_eq!(config.max_model_len, None);
79
+}
80
+
81
#[test]
82
fn render_args_build_config_with_tls() {
83
let cli = Cli::try_parse_from([
rust/src/server/src/render.rs
+8
-3
@@ -33,7 +33,7 @@ pub struct RenderConfig {
33
pub chat_template: Option<String>,
34
pub default_chat_template_kwargs: HashMap<String, Value>,
35
pub chat_template_content_format: ChatTemplateContentFormatOption,
36
- pub max_model_len: u32,
36
+ pub max_model_len: Option<u32>,
37
pub max_logprobs: Option<i32>,
38
pub tls: Option<TlsConfig>,
39
}
@@ -56,6 +56,9 @@ impl RenderConfig {
56
pub(crate) struct RenderState {
57
pub(crate) model: String,
58
pub(crate) served_model_names: Vec<String>,
59
+ /// Unlike `text.max_model_len()` this stays `None` when unset,
60
+ /// so model cards advertise `null` instead of the `u32::MAX`.
61
+ pub(crate) max_model_len: Option<u32>,
62
pub(crate) text: TextRequestProcessor,
63
pub(crate) chat: ChatRequestProcessor,
64
}
@@ -77,7 +80,8 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
80
.context("failed to load renderer/tokenizer backends")?;
81
let served_model_names =
82
crate::effective_served_model_names(&config.model, &config.served_model_name);
80
- let text = TextRequestProcessor::new(loaded.text_backend, config.max_model_len)
83
+ let max_model_len = config.max_model_len.unwrap_or(u32::MAX);
84
+ let text = TextRequestProcessor::new(loaded.text_backend, max_model_len)
85
.with_max_logprobs(config.max_logprobs);
86
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
87
config.tool_call_parser.clone(),
@@ -86,6 +90,7 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
90
Ok(Arc::new(RenderState {
91
model: config.model.clone(),
92
served_model_names,
93
+ max_model_len: config.max_model_len,
94
text,
95
chat,
96
}))
@@ -154,7 +159,7 @@ mod tests {
159
chat_template: None,
160
default_chat_template_kwargs: HashMap::new(),
161
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
157
- max_model_len: 128,
162
+ max_model_len: Some(128),
163
max_logprobs: None,
164
tls: None,
165
},
rust/src/server/src/routes/render.rs
+1
-1
@@ -55,7 +55,7 @@ async fn list_models(State(state): State<Arc<RenderState>>) -> Json<ListModelsRe
55
owned_by: "vllm".to_string(),
56
root: Some(state.model.clone()),
57
parent: None,
58
- max_model_len: Some(state.text.max_model_len()),
58
+ max_model_len: state.max_model_len,
59
})
60
.collect(),
61
})
rust/src/server/src/routes/tests.rs
+34
-1
@@ -691,12 +691,25 @@ fn test_render_app() -> axum::Router {
691
fn test_render_app_with_parser_selections(
692
tool_call_parser: ParserSelection,
693
reasoning_parser: ParserSelection,
694
+) -> axum::Router {
695
+ test_render_app_with(Some(128), tool_call_parser, reasoning_parser)
696
+}
697
+
698
+fn test_render_app_with_max_model_len(max_model_len: Option<u32>) -> axum::Router {
699
+ test_render_app_with(max_model_len, ParserSelection::Auto, ParserSelection::Auto)
700
+}
701
+
702
+fn test_render_app_with(
703
+ max_model_len: Option<u32>,
704
+ tool_call_parser: ParserSelection,
705
+ reasoning_parser: ParserSelection,
706
) -> axum::Router {
707
let backend = Arc::new(FakeChatBackend::new());
708
build_render_router(Arc::new(RenderState {
709
model: "backend-model".to_string(),
710
served_model_names: vec!["render-model".to_string()],
699
- text: TextRequestProcessor::new(backend.clone(), 128),
711
+ max_model_len,
712
+ text: TextRequestProcessor::new(backend.clone(), max_model_len.unwrap_or(u32::MAX)),
713
chat: ChatRequestProcessor::render_only(backend)
714
.with_parser_selections(tool_call_parser, reasoning_parser),
715
}))
@@ -1414,6 +1427,26 @@ async fn render_completion_returns_generate_request_with_body_request_id() {
1427
assert!(json[0].get("mm_features").is_none());
1428
}
1429
1430
+#[tokio::test]
1431
+async fn render_list_models_reports_configured_max_model_len() {
1432
+ for (configured, expected) in [(Some(128), json!(128)), (None, serde_json::Value::Null)] {
1433
+ let mut app = test_render_app_with_max_model_len(configured);
1434
+ let response = app
1435
+ .call(Request::builder().uri("/v1/models").body(Body::empty()).expect("build request"))
1436
+ .await
1437
+ .expect("call app");
1438
+ assert_eq!(response.status(), StatusCode::OK);
1439
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
1440
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
1441
+ let card = json["data"][0].as_object().expect("card object");
1442
+ assert!(card.contains_key("max_model_len"));
1443
+ assert_eq!(
1444
+ card["max_model_len"], expected,
1445
+ "configured: {configured:?}"
1446
+ );
1447
+ }
1448
+}
1449
+
1450
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1451
#[serial]
1452
async fn render_chat_response_can_be_submitted_to_generate_unchanged() {