[Rust Frontend] Raise frontend JSON body limit (#46582)
Signed-off-by: esmeetu <jasonailu87@gmail.com>
Roy Wang committed
Jun 24, 2026 at 20:15 UTC
160c80a34ca2f94ca22886ec807e66d875d27b66
2 files changed
+44
rust/src/server/src/routes.rs
+4
@@ -17,6 +17,7 @@ mod world_size;
17
use std::sync::Arc;
18
19
use axum::Router;
20
+use axum::extract::DefaultBodyLimit;
21
use axum::middleware::{from_fn, from_fn_with_state};
22
use axum::routing::{get, post};
23
use tower_http::trace::TraceLayer;
@@ -24,6 +25,8 @@ use tower_http::trace::TraceLayer;
25
use crate::middleware;
26
use crate::state::AppState;
27
28
+const DEFAULT_JSON_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
29
+
30
fn server_dev_mode_enabled() -> bool {
31
std::env::var("VLLM_SERVER_DEV_MODE")
32
.ok()
@@ -108,6 +111,7 @@ fn build_router_with_options(
111
let enable_api_key_auth = state.has_api_keys();
112
let mut router = router
113
.with_state(state.clone())
114
+ .layer(DefaultBodyLimit::max(DEFAULT_JSON_BODY_LIMIT_BYTES))
115
.layer(middleware::request_runtime_layer(state.clone()))
116
.layer(from_fn_with_state(
117
state.clone(),
rust/src/server/src/routes/tests.rs
+40
@@ -3510,6 +3510,46 @@ async fn non_stream_chat_completions_still_succeed() {
3510
engine_task.await.expect("mock engine task");
3511
}
3512
3513
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3514
+#[serial]
3515
+async fn chat_completions_accepts_request_body_larger_than_axum_default() {
3516
+ let (chat, engine_task) = test_chat_with_engine_outputs(
3517
+ b"engine-openai-chat-large-body",
3518
+ default_stream_output_specs(),
3519
+ )
3520
+ .await;
3521
+ let mut app = build_router(Arc::new(AppState::new(
3522
+ vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
3523
+ chat,
3524
+ )));
3525
+
3526
+ let large_template_arg = "a".repeat(2 * 1024 * 1024);
3527
+ let response = app
3528
+ .call(
3529
+ Request::builder()
3530
+ .method("POST")
3531
+ .uri("/v1/chat/completions")
3532
+ .header("content-type", "application/json")
3533
+ .body(Body::from(
3534
+ json!({
3535
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
3536
+ "stream": false,
3537
+ "messages": [{"role": "user", "content": "hello"}],
3538
+ "chat_template_kwargs": {"large": large_template_arg}
3539
+ })
3540
+ .to_string(),
3541
+ ))
3542
+ .expect("build request"),
3543
+ )
3544
+ .await
3545
+ .expect("call app");
3546
+
3547
+ let status = response.status();
3548
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
3549
+ assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body));
3550
+ engine_task.await.expect("mock engine task");
3551
+}
3552
+
3553
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3554
#[serial]
3555
async fn non_stream_completions_still_succeed() {