A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix][Rust Frontend] Return 400 for prompt-validation submit errors (#45286)
Signed-off-by: xiaguan <751080330@qq.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
JinYan Su committed
Jun 12, 2026 at 15:53 UTC
1ae1051b4bf6e7e98d61b15527040f63eda73a0b
4 files changed
+89
-18
rust/src/server/src/error.rs
+82
@@ -1,6 +1,7 @@
1
use axum::Json;
2
use axum::http::StatusCode;
3
use axum::response::{IntoResponse, Response};
4
+use thiserror_ext::AsReport as _;
5
use thiserror_ext::{Construct, Macro};
6
7
use crate::routes::openai::utils::types::{ErrorDetail, ErrorResponse};
@@ -72,3 +73,84 @@ impl IntoResponse for ApiError {
73
(self.status_code(), Json(self.to_error_response())).into_response()
74
}
75
}
76
+
77
+/// Classify a text-pipeline submit failure: tokenized-prompt validation
78
+/// failures (the prompt is too long for the model, or empty after
79
+/// tokenization) are the client's fault and map to HTTP 400, mirroring the
80
+/// Python frontend. Everything else stays an internal 500.
81
+pub fn text_submit_error(context: &'static str, error: vllm_text::Error) -> ApiError {
82
+ if is_prompt_validation_error(&error) {
83
+ return invalid_request!("{error}");
84
+ }
85
+ server_error!("{}: {}", context, error.to_report_string())
86
+}
87
+
88
+/// Like [`text_submit_error`], for the chat pipeline (which both wraps the
89
+/// text errors and raises its own prompt-length variant).
90
+pub fn chat_submit_error(context: &'static str, error: vllm_chat::Error) -> ApiError {
91
+ match &error {
92
+ vllm_chat::Error::PromptTooLong { .. } => invalid_request!("{error}"),
93
+ vllm_chat::Error::Text(text_error) if is_prompt_validation_error(text_error) => {
94
+ invalid_request!("{error}")
95
+ }
96
+ _ => server_error!("{}: {}", context, error.to_report_string()),
97
+ }
98
+}
99
+
100
+fn is_prompt_validation_error(error: &vllm_text::Error) -> bool {
101
+ matches!(
102
+ error,
103
+ vllm_text::Error::PromptTooLong { .. }
104
+ | vllm_text::Error::EmptyPromptTokenIds { .. }
105
+ // An empty tokenized prompt detected later, at request prepare
106
+ // time, surfaces through the transparent Llm wrapper.
107
+ | vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { .. })
108
+ )
109
+}
110
+
111
+#[cfg(test)]
112
+mod tests {
113
+ use super::*;
114
+
115
+ #[test]
116
+ fn prompt_too_long_maps_to_invalid_request() {
117
+ let error = vllm_text::Error::PromptTooLong {
118
+ max_model_len: 8192,
119
+ prompt_len: 9000,
120
+ };
121
+ let api_error = text_submit_error("failed to submit completion request", error);
122
+ assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
123
+ let response = api_error.to_error_response();
124
+ assert_eq!(response.error.error_type, "invalid_request_error");
125
+ assert!(response.error.message.contains("8192"));
126
+ assert!(response.error.message.contains("9000"));
127
+ }
128
+
129
+ #[test]
130
+ fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
131
+ let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {
132
+ max_model_len: 8192,
133
+ prompt_len: 9000,
134
+ });
135
+ let api_error = chat_submit_error("failed to submit chat request", error);
136
+ assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
137
+ }
138
+
139
+ #[test]
140
+ fn llm_wrapped_empty_prompt_maps_to_invalid_request() {
141
+ let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds {
142
+ request_id: "req-1".to_string(),
143
+ });
144
+ let api_error = text_submit_error("failed to submit completion request", error);
145
+ assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
146
+ }
147
+
148
+ #[test]
149
+ fn other_submit_errors_stay_internal() {
150
+ let error = vllm_text::Error::Tokenizer("backend exploded".to_string());
151
+ let api_error = text_submit_error("failed to submit completion request", error);
152
+ assert_eq!(api_error.status_code(), StatusCode::INTERNAL_SERVER_ERROR);
153
+ let response = api_error.to_error_response();
154
+ assert!(response.error.message.starts_with("failed to submit completion request:"));
155
+ }
156
+}
rust/src/server/src/routes/inference/generate.rs
+3
-6
@@ -28,7 +28,7 @@ use self::types::{
28
GenerateResponseStreamChoice, GenerateStreamResponse,
29
};
30
use crate::config::ApiServerOptions;
31
-use crate::error::{ApiError, bail_server_error, server_error};
31
+use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
32
use crate::routes::openai::utils::logprobs::clamp_logprob;
33
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
34
use crate::routes::openai::utils::validated_json::ValidatedJson;
@@ -65,11 +65,8 @@ pub async fn generate(
65
{
66
Ok(stream) => stream,
67
Err(error) => {
68
- return server_error!(
69
- "failed to submit raw generate request: {}",
70
- error.to_report_string()
71
- )
72
- .into_response();
68
+ return text_submit_error("failed to submit raw generate request", error)
69
+ .into_response();
70
}
71
};
72
rust/src/server/src/routes/openai/chat_completions.rs
+2
-6
@@ -25,7 +25,7 @@ use vllm_engine_core_client::protocol::StopReason;
25
26
use self::convert::{ResponseOptions, prepare_chat_request};
27
use crate::config::ApiServerOptions;
28
-use crate::error::{ApiError, bail_server_error, server_error};
28
+use crate::error::{ApiError, bail_server_error, chat_submit_error, server_error};
29
use crate::routes::openai::chat_completions::types::{
30
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
31
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
@@ -77,11 +77,7 @@ pub async fn chat_completions(
77
match state.chat.chat(prepared.chat_request).instrument(request_span.clone()).await {
78
Ok(stream) => stream,
79
Err(error) => {
80
- return server_error!(
81
- "failed to submit chat request: {}",
82
- error.to_report_string()
83
- )
84
- .into_response();
80
+ return chat_submit_error("failed to submit chat request", error).into_response();
81
}
82
};
83
rust/src/server/src/routes/openai/completions.rs
+2
-6
@@ -25,7 +25,7 @@ use super::utils::logprobs::{
25
};
26
use super::utils::types::Usage;
27
use crate::config::ApiServerOptions;
28
-use crate::error::{ApiError, bail_server_error, server_error};
28
+use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
29
use crate::routes::openai::completions::types::{
30
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
31
CompletionStreamChoice, CompletionStreamResponse,
@@ -75,11 +75,7 @@ pub async fn completions(
75
{
76
Ok(stream) => stream,
77
Err(error) => {
78
- return server_error!(
79
- "failed to submit completion request: {}",
80
- error.to_report_string()
81
- )
82
- .into_response();
78
+ return text_submit_error("failed to submit completion request", error).into_response();
79
}
80
};
81