[Bugfix][Rust Frontend] Reject n > 1 in the `/inference/v1/generate` route (#52844)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Quentin Gallouédec committed
Aug 20, 2026 at 03:41 UTC
16cfe728d8d0bc3cd4a8397db0f392dd52a2c109
6 files changed
+85
-9
rust/src/server/src/routes/inference/generate.rs
+1
-1
@@ -26,11 +26,11 @@ use vllm_llm::{
26
};
27
28
use self::convert::{ResponseOptions, prepare_generate_request};
29
-pub(crate) use self::types::GenerateRequest;
29
use self::types::{
30
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
31
GenerateStreamResponse,
32
};
33
+pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
34
pub(crate) use self::validate::validate_request_compat;
35
use crate::config::ApiServerOptions;
36
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
rust/src/server/src/routes/inference/generate/convert.rs
+3
-3
@@ -54,9 +54,9 @@ pub(super) fn prepare_generate_request(
54
.as_ref()
55
.and_then(|options| options.continuous_usage_stats)
56
.unwrap_or(false);
57
- let include_logprobs = request.sampling_params.logprobs.is_some();
58
- let include_prompt_logprobs = request.sampling_params.prompt_logprobs.is_some();
59
- let mut sampling_params = request.sampling_params;
57
+ let include_logprobs = request.sampling_params.inner.logprobs.is_some();
58
+ let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
59
+ let mut sampling_params = request.sampling_params.inner;
60
sampling_params.vllm_xargs = merge_kv_transfer_params(
61
sampling_params.vllm_xargs,
62
request.kv_transfer_params.as_ref(),
rust/src/server/src/routes/inference/generate/types.rs
+18
-1
@@ -11,6 +11,23 @@ use vllm_text::SamplingParams;
11
12
use crate::routes::openai::utils::types::{ChatLogProbs, Normalizable, StreamOptions, Usage};
13
14
+/// Sampling parameters for the token-in/token-out generate API.
15
+///
16
+/// Wraps [`SamplingParams`] to additionally capture `n`, which the shared
17
+/// northbound type intentionally omits (parallel sampling is handled by
18
+/// higher layers, and the Rust frontend does not implement it). Capturing it
19
+/// here lets validation reject `n > 1` explicitly instead of silently
20
+/// dropping the key and returning a single choice.
21
+#[serde_with::skip_serializing_none]
22
+#[derive(Debug, Clone, Deserialize, Serialize)]
23
+pub struct GenerateSamplingParams {
24
+ /// Number of output sequences to generate. Only `1` is supported.
25
+ pub n: Option<u32>,
26
+ /// The supported sampling parameters, lowered to the engine.
27
+ #[serde(flatten)]
28
+ pub inner: SamplingParams,
29
+}
30
+
31
/// vLLM-compatible request type for the token-in/token-out generate API.
32
#[serde_with::skip_serializing_none]
33
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
@@ -18,7 +35,7 @@ pub struct GenerateRequest {
35
pub request_id: Option<String>,
36
pub model: Option<String>,
37
pub token_ids: Vec<u32>,
21
- pub sampling_params: SamplingParams,
38
+ pub sampling_params: GenerateSamplingParams,
39
#[serde(default)]
40
pub stream: bool,
41
pub stream_options: Option<StreamOptions>,
rust/src/server/src/routes/inference/generate/validate.rs
+28
-2
@@ -23,6 +23,10 @@ pub(crate) fn validate_request_compat(
23
);
24
}
25
26
+ if request.sampling_params.n.unwrap_or(1) != 1 {
27
+ bail_invalid_request!(param = "n", "Only n=1 is supported.");
28
+ }
29
+
30
if request.token_ids.is_empty() {
31
bail_invalid_request!(
32
param = "token_ids",
@@ -30,14 +34,14 @@ pub(crate) fn validate_request_compat(
34
);
35
}
36
33
- if request.sampling_params.max_tokens == Some(0) {
37
+ if request.sampling_params.inner.max_tokens == Some(0) {
38
bail_invalid_request!(
39
param = "sampling_params",
40
"max_tokens must be greater than 0."
41
);
42
}
43
40
- if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
44
+ if let Some(prompt_logprobs) = request.sampling_params.inner.prompt_logprobs {
45
if prompt_logprobs < 0 && prompt_logprobs != -1 {
46
bail_invalid_request!(
47
param = "sampling_params",
@@ -98,6 +102,28 @@ mod tests {
102
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
103
}
104
105
+ #[test]
106
+ fn validate_request_compat_rejects_parallel_sampling() {
107
+ let request: GenerateRequest = serde_json::from_value(json!({
108
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
109
+ "token_ids": [11, 22],
110
+ "sampling_params": {"n": 4}
111
+ }))
112
+ .expect("parse request");
113
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
114
+ }
115
+
116
+ #[test]
117
+ fn validate_request_compat_accepts_explicit_n_one() {
118
+ let request: GenerateRequest = serde_json::from_value(json!({
119
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
120
+ "token_ids": [11, 22],
121
+ "sampling_params": {"n": 1}
122
+ }))
123
+ .expect("parse request");
124
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
125
+ }
126
+
127
#[test]
128
fn validate_request_compat_rejects_empty_token_ids() {
129
let request = GenerateRequest {
rust/src/server/src/routes/render.rs
+5
-2
@@ -17,7 +17,7 @@ use crate::error::{ApiError, text_submit_error};
17
use crate::lora::LoraModelResolution;
18
use crate::render::RenderState;
19
use crate::routes::inference::generate::{
20
- GenerateRequest, validate_request_compat as validate_generate_request,
20
+ GenerateRequest, GenerateSamplingParams, validate_request_compat as validate_generate_request,
21
};
22
use crate::routes::openai::utils::types::{ListModelsResponse, ModelObject, StreamOptions};
23
use crate::routes::openai::utils::validated_json::ValidatedJson;
@@ -86,7 +86,10 @@ fn lower_render_request(
86
request_id: Some(text_request.request_id),
87
model: Some(model),
88
token_ids,
89
- sampling_params: text_request.sampling_params,
89
+ sampling_params: GenerateSamplingParams {
90
+ n: None,
91
+ inner: text_request.sampling_params,
92
+ },
93
stream,
94
stream_options,
95
cache_salt: text_request.cache_salt,
rust/src/server/src/routes/tests.rs
+30
@@ -4316,6 +4316,36 @@ async fn raw_generate_rejects_empty_token_ids() {
4316
assert_eq!(json["error"]["param"], "token_ids");
4317
}
4318
4319
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4320
+#[serial]
4321
+async fn raw_generate_rejects_parallel_sampling() {
4322
+ let mut app = test_app().await;
4323
+
4324
+ let response = app
4325
+ .call(
4326
+ Request::builder()
4327
+ .method("POST")
4328
+ .uri("/inference/v1/generate")
4329
+ .header("content-type", "application/json")
4330
+ .body(Body::from(
4331
+ json!({
4332
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
4333
+ "token_ids": [11, 22],
4334
+ "sampling_params": {"n": 4}
4335
+ })
4336
+ .to_string(),
4337
+ ))
4338
+ .expect("build request"),
4339
+ )
4340
+ .await
4341
+ .expect("call app");
4342
+
4343
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST);
4344
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
4345
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
4346
+ assert_eq!(json["error"]["param"], "n");
4347
+}
4348
+
4349
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4350
#[serial]
4351
async fn raw_generate_rejects_streaming_prompt_logprobs() {