[Bugfix][Rust Frontend] Reject prompt_logprobs for streaming generate (#46839)
Signed-off-by: reidliu41 <reid201711@gmail.com>
Reid committed
Jun 30, 2026 at 13:10 UTC
930f8dc0a1f2d19a0a61df3c0ab9b67dde36214e
2 files changed
+103
-8
rust/src/server/src/routes/inference/generate/validate.rs
+64
-8
@@ -34,14 +34,20 @@ pub(super) fn validate_request_compat(
34
);
35
}
36
37
- if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
38
- && prompt_logprobs < 0
39
- && prompt_logprobs != -1
40
- {
41
- bail_invalid_request!(
42
- param = "sampling_params",
43
- "`prompt_logprobs` must be a non-negative value or -1."
44
- );
37
+ if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
38
+ if prompt_logprobs < 0 && prompt_logprobs != -1 {
39
+ bail_invalid_request!(
40
+ param = "sampling_params",
41
+ "`prompt_logprobs` must be a non-negative value or -1."
42
+ );
43
+ }
44
+
45
+ if request.stream {
46
+ bail_invalid_request!(
47
+ param = "sampling_params",
48
+ "`prompt_logprobs` are not available when `stream=true`."
49
+ );
50
+ }
51
}
52
53
Ok(())
@@ -97,4 +103,54 @@ mod tests {
103
};
104
assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
105
}
106
+
107
+ #[test]
108
+ fn validate_request_compat_rejects_streaming_prompt_logprobs() {
109
+ let request: GenerateRequest = serde_json::from_value(json!({
110
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
111
+ "token_ids": [11, 22],
112
+ "stream": true,
113
+ "sampling_params": {
114
+ "prompt_logprobs": 0
115
+ }
116
+ }))
117
+ .expect("parse request");
118
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
119
+
120
+ let request: GenerateRequest = serde_json::from_value(json!({
121
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
122
+ "token_ids": [11, 22],
123
+ "stream": true,
124
+ "sampling_params": {
125
+ "prompt_logprobs": 1
126
+ }
127
+ }))
128
+ .expect("parse request");
129
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
130
+
131
+ let request: GenerateRequest = serde_json::from_value(json!({
132
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
133
+ "token_ids": [11, 22],
134
+ "stream": true,
135
+ "sampling_params": {
136
+ "prompt_logprobs": -1
137
+ }
138
+ }))
139
+ .expect("parse request");
140
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
141
+ }
142
+
143
+ #[test]
144
+ fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
145
+ let request: GenerateRequest = serde_json::from_value(json!({
146
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
147
+ "token_ids": [11, 22],
148
+ "stream": false,
149
+ "sampling_params": {
150
+ "prompt_logprobs": 1
151
+ }
152
+ }))
153
+ .expect("parse request");
154
+ assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
155
+ }
156
}
rust/src/server/src/routes/tests.rs
+39
@@ -4129,6 +4129,45 @@ async fn raw_generate_rejects_empty_token_ids() {
4129
assert_eq!(json["error"]["param"], "token_ids");
4130
}
4131
4132
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4133
+#[serial]
4134
+async fn raw_generate_rejects_streaming_prompt_logprobs() {
4135
+ let mut app = test_app().await;
4136
+
4137
+ for prompt_logprobs in [0, 1] {
4138
+ let response = app
4139
+ .call(
4140
+ Request::builder()
4141
+ .method("POST")
4142
+ .uri("/inference/v1/generate")
4143
+ .header("content-type", "application/json")
4144
+ .body(Body::from(
4145
+ json!({
4146
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
4147
+ "token_ids": [11, 22],
4148
+ "stream": true,
4149
+ "sampling_params": {
4150
+ "prompt_logprobs": prompt_logprobs
4151
+ }
4152
+ })
4153
+ .to_string(),
4154
+ ))
4155
+ .expect("build request"),
4156
+ )
4157
+ .await
4158
+ .expect("call app");
4159
+
4160
+ assert_eq!(response.status(), StatusCode::BAD_REQUEST);
4161
+ let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
4162
+ let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
4163
+ assert_eq!(json["error"]["param"], "sampling_params");
4164
+ assert_eq!(
4165
+ json["error"]["message"],
4166
+ "`prompt_logprobs` are not available when `stream=true`."
4167
+ );
4168
+ }
4169
+}
4170
+
4171
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
4172
#[serial]
4173
async fn raw_generate_rejects_wrong_model() {