A high-throughput and memory-efficient inference and serving engine for LLMs
[Rust Frontend] Support prompt-only completions (#44938)
Signed-off-by: reidliu41 <reid201711@gmail.com>
Reid committed
Jun 17, 2026 at 14:38 UTC
56e434522671890bdbe7362bcec46956521f951c
3 files changed
+550
-27
rust/src/server/src/routes/openai/completions.rs
+452
-19
@@ -2,6 +2,7 @@ mod convert;
2
mod types;
3
mod validate;
4
5
+use std::collections::HashMap;
6
use std::convert::Infallible;
7
use std::result::Result;
8
use std::sync::Arc;
@@ -16,12 +17,15 @@ use futures::{Stream, StreamExt as _, pin_mut};
17
use thiserror_ext::AsReport as _;
18
use tracing::{debug, error, info, trace};
19
use tracing_futures::Instrument as _;
19
-use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
20
+use vllm_text::{
21
+ DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
22
+ TextOutputStreamExt as _,
23
+};
24
25
use self::convert::{ResponseOptions, prepare_completion_request};
26
use super::utils::logprobs::{
27
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
24
- text_len,
28
+ decoded_prompt_logprobs_to_openai, text_len,
29
};
30
use super::utils::types::Usage;
31
use crate::config::ApiServerOptions;
@@ -119,6 +123,7 @@ async fn collect_completion(
123
include_usage: _,
124
// Ignored: non-streaming responses are collected before usage is attached.
125
include_continuous_usage: _,
126
+ prompt_only,
127
echo,
128
requested_logprobs,
129
include_prompt_logprobs,
@@ -136,17 +141,17 @@ async fn collect_completion(
141
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
142
143
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
139
- let prompt_logprobs = if include_prompt_logprobs {
140
- let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
141
- server_error!(
142
- "completion response requested prompt_logprobs but generation returned none"
143
- )
144
+ let logprobs = if requested_logprobs.is_some() && prompt_only {
145
+ let prompt = echo.as_deref().ok_or_else(|| {
146
+ server_error!("prompt-only completion response missing echoed prompt")
147
})?;
145
- Some(prompt_logprobs)
146
- } else {
147
- None
148
- };
149
- let logprobs = if requested_logprobs.is_some() {
148
+ Some(prompt_only_logprobs_to_openai(
149
+ collected.prompt_logprobs.as_ref(),
150
+ prompt,
151
+ collected.prompt_token_ids.as_ref(),
152
+ return_tokens_as_token_ids,
153
+ )?)
154
+ } else if requested_logprobs.is_some() {
155
Some(collected_logprobs_to_openai(
156
&collected,
157
echo.is_some(),
@@ -156,10 +161,18 @@ async fn collect_completion(
161
} else {
162
None
163
};
159
- let prompt_logprobs =
160
- prompt_logprobs.map(|lp| decoded_prompt_logprobs_to_maps(lp, return_tokens_as_token_ids));
164
+ let prompt_logprobs = if include_prompt_logprobs {
165
+ Some(prompt_logprobs_to_maps(
166
+ collected.prompt_logprobs.as_ref(),
167
+ collected.prompt_token_ids.as_ref(),
168
+ return_tokens_as_token_ids,
169
+ )?)
170
+ } else {
171
+ None
172
+ };
173
let text = match &echo {
174
None => collected.text,
175
+ Some(prompt) if prompt_only => prompt.clone(),
176
Some(prompt) => format!("{prompt}{}", collected.text),
177
};
178
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
@@ -211,6 +224,7 @@ async fn completion_chunk_stream(
224
ResponseOptions {
225
include_usage,
226
include_continuous_usage,
227
+ prompt_only,
228
echo,
229
requested_logprobs,
230
// Ignored: streaming prompt logprobs are rejected for Python parity.
@@ -239,14 +253,30 @@ async fn completion_chunk_stream(
253
while let Some(next) = stream.next().await {
254
match next {
255
Ok(DecodedTextEvent::Start {
242
- prompt_token_ids, ..
256
+ prompt_token_ids,
257
+ prompt_logprobs,
258
}) => {
259
debug!("completion stream started");
260
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
261
if let Some(prompt) = echo.as_ref() {
262
visible_text_len = text_len(prompt);
248
- let mut chunk =
249
- delta_chunk(&request_id, &response_model, created, prompt.clone(), None);
263
+ let logprobs = if prompt_only && requested_logprobs.is_some() {
264
+ Some(prompt_only_logprobs_to_openai(
265
+ prompt_logprobs.as_ref(),
266
+ prompt,
267
+ prompt_token_ids.as_ref(),
268
+ return_tokens_as_token_ids,
269
+ )?)
270
+ } else {
271
+ None
272
+ };
273
+ let mut chunk = delta_chunk(
274
+ &request_id,
275
+ &response_model,
276
+ created,
277
+ prompt.clone(),
278
+ logprobs,
279
+ );
280
if return_token_ids && first_chunk {
281
if let Some(choice) = chunk.choices.first_mut() {
282
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
@@ -271,6 +301,48 @@ async fn completion_chunk_stream(
301
logprobs,
302
finished,
303
}) => {
304
+ // Prompt-only streaming already emitted the echoed prompt in the Start chunk.
305
+ // The one generated token is only used to drive the engine to a finished event,
306
+ // so hide its delta and forward only the terminal finish/usage metadata.
307
+ if prompt_only {
308
+ if let Some(finished) = finished {
309
+ if enable_log_requests {
310
+ info!(
311
+ stream = true,
312
+ model = %response_model,
313
+ prompt_tokens = finished.usage.prompt_token_count,
314
+ output_tokens = finished.usage.output_token_count,
315
+ finish_reason = finished.finish_reason.as_str(),
316
+ "completion finished"
317
+ );
318
+ }
319
+ continuous_usage.set_final_counts(
320
+ finished.usage.prompt_token_count,
321
+ finished.usage.output_token_count,
322
+ );
323
+ let final_chunk = final_chunk(
324
+ &request_id,
325
+ &response_model,
326
+ created,
327
+ finished.finish_reason,
328
+ )?;
329
+ yield_chunk!(final_chunk);
330
+
331
+ if include_usage {
332
+ y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
333
+ &request_id,
334
+ &response_model,
335
+ created,
336
+ Usage::from_token_usage(
337
+ finished.usage,
338
+ enable_prompt_tokens_details,
339
+ ),
340
+ )))
341
+ .await;
342
+ }
343
+ }
344
+ continue;
345
+ }
346
let delta_text_len = text_len(&delta);
347
let logprobs = if requested_logprobs.is_some() {
348
let decoded_logprobs = logprobs.as_ref().ok_or_else(|| {
@@ -386,6 +458,57 @@ fn completion_finish_reason_to_openai(
458
}
459
}
460
461
+fn prompt_only_logprobs_to_openai(
462
+ prompt_logprobs: Option<&DecodedPromptLogprobs>,
463
+ prompt: &str,
464
+ prompt_token_ids: &[u32],
465
+ return_tokens_as_token_ids: bool,
466
+) -> Result<LogProbs, ApiError> {
467
+ if let Some(prompt_logprobs) = prompt_logprobs {
468
+ return decoded_prompt_logprobs_to_openai(prompt_logprobs, 0, return_tokens_as_token_ids);
469
+ }
470
+
471
+ if let [token_id] = prompt_token_ids {
472
+ let token = if return_tokens_as_token_ids {
473
+ format!("token_id:{token_id}")
474
+ } else {
475
+ prompt.to_string()
476
+ };
477
+
478
+ return Ok(LogProbs {
479
+ tokens: vec![token],
480
+ token_logprobs: vec![None],
481
+ top_logprobs: vec![None],
482
+ text_offset: vec![0],
483
+ });
484
+ }
485
+
486
+ Err(server_error!(
487
+ "prompt-only completion requested logprobs but generation returned none"
488
+ ))
489
+}
490
+
491
+fn prompt_logprobs_to_maps(
492
+ prompt_logprobs: Option<&DecodedPromptLogprobs>,
493
+ prompt_token_ids: &[u32],
494
+ return_tokens_as_token_ids: bool,
495
+) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
496
+ if let Some(prompt_logprobs) = prompt_logprobs {
497
+ return Ok(decoded_prompt_logprobs_to_maps(
498
+ prompt_logprobs,
499
+ return_tokens_as_token_ids,
500
+ ));
501
+ }
502
+
503
+ if let [_token_id] = prompt_token_ids {
504
+ return Ok(vec![None]);
505
+ }
506
+
507
+ Err(server_error!(
508
+ "completion response requested prompt_logprobs but generation returned none"
509
+ ))
510
+}
511
+
512
fn usage_chunk(
513
request_id: &str,
514
response_model: &str,
@@ -449,8 +572,8 @@ mod tests {
572
use futures::{StreamExt as _, stream};
573
use itertools::Itertools as _;
574
use vllm_text::{
452
- DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob,
453
- FinishReason, Finished,
575
+ DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
576
+ DecodedTokenLogprob, FinishReason, Finished,
577
};
578
579
use super::{
@@ -613,4 +736,314 @@ mod tests {
736
CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
737
}
738
}
739
+
740
+ #[tokio::test]
741
+ async fn collect_completion_hides_internal_prompt_only_token() {
742
+ let stream = stream::iter(vec![
743
+ Ok(DecodedTextEvent::Start {
744
+ prompt_token_ids: vec![1, 2].into(),
745
+ prompt_logprobs: None,
746
+ }),
747
+ Ok(DecodedTextEvent::TextDelta {
748
+ delta: " leaked".to_string(),
749
+ token_ids: vec![3],
750
+ logprobs: None,
751
+ finished: Some(Finished {
752
+ usage: vllm_llm::TokenUsage {
753
+ prompt_token_count: 2,
754
+ output_token_count: 1,
755
+ cached_token_count: 0,
756
+ },
757
+ finish_reason: FinishReason::Length,
758
+ kv_transfer_params: None,
759
+ }),
760
+ }),
761
+ ]);
762
+
763
+ let response = super::collect_completion(
764
+ stream,
765
+ "cmpl-1".to_string(),
766
+ "model".to_string(),
767
+ 1,
768
+ ApiServerOptions::default(),
769
+ ResponseOptions {
770
+ prompt_only: true,
771
+ echo: Some("hello".to_string()),
772
+ return_token_ids: true,
773
+ ..Default::default()
774
+ },
775
+ )
776
+ .await
777
+ .expect("collect completion");
778
+
779
+ assert_eq!(response.choices[0].text, "hello");
780
+ assert_eq!(response.choices[0].token_ids.as_deref(), Some(&[3][..]));
781
+ assert_eq!(
782
+ response.choices[0].prompt_token_ids.as_deref(),
783
+ Some(&[1, 2][..])
784
+ );
785
+ let usage = response.usage.expect("usage");
786
+ assert_eq!(usage.prompt_tokens, 2);
787
+ assert_eq!(usage.completion_tokens, Some(1));
788
+ assert_eq!(usage.total_tokens, 3);
789
+ }
790
+
791
+ #[tokio::test]
792
+ async fn collect_completion_maps_prompt_logprobs_for_single_token_prompt() {
793
+ let stream = stream::iter(vec![
794
+ Ok(DecodedTextEvent::Start {
795
+ prompt_token_ids: vec![9707].into(),
796
+ prompt_logprobs: None,
797
+ }),
798
+ Ok(DecodedTextEvent::TextDelta {
799
+ delta: " leaked".to_string(),
800
+ token_ids: vec![3],
801
+ logprobs: None,
802
+ finished: Some(Finished {
803
+ usage: vllm_llm::TokenUsage {
804
+ prompt_token_count: 1,
805
+ output_token_count: 1,
806
+ cached_token_count: 0,
807
+ },
808
+ finish_reason: FinishReason::Length,
809
+ kv_transfer_params: None,
810
+ }),
811
+ }),
812
+ ]);
813
+
814
+ let response = super::collect_completion(
815
+ stream,
816
+ "cmpl-1".to_string(),
817
+ "model".to_string(),
818
+ 1,
819
+ ApiServerOptions::default(),
820
+ ResponseOptions {
821
+ prompt_only: true,
822
+ echo: Some("Hello".to_string()),
823
+ requested_logprobs: Some(1),
824
+ include_prompt_logprobs: true,
825
+ ..Default::default()
826
+ },
827
+ )
828
+ .await
829
+ .expect("collect completion");
830
+
831
+ let choice = &response.choices[0];
832
+ assert_eq!(choice.text, "Hello");
833
+ assert_eq!(choice.prompt_logprobs, Some(vec![None]));
834
+ let logprobs = choice.logprobs.as_ref().expect("logprobs");
835
+ assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
836
+ assert_eq!(logprobs.token_logprobs, vec![None]);
837
+ assert_eq!(logprobs.top_logprobs, vec![None]);
838
+ assert_eq!(logprobs.text_offset, vec![0]);
839
+ let usage = response.usage.expect("usage");
840
+ assert_eq!(usage.prompt_tokens, 1);
841
+ assert_eq!(usage.completion_tokens, Some(1));
842
+ assert_eq!(usage.total_tokens, 2);
843
+ }
844
+
845
+ #[tokio::test]
846
+ async fn completion_chunk_stream_hides_internal_prompt_only_token() {
847
+ let stream = stream::iter(vec![
848
+ Ok(DecodedTextEvent::Start {
849
+ prompt_token_ids: vec![1, 2].into(),
850
+ prompt_logprobs: None,
851
+ }),
852
+ Ok(DecodedTextEvent::TextDelta {
853
+ delta: " leaked".to_string(),
854
+ token_ids: vec![3],
855
+ logprobs: None,
856
+ finished: Some(Finished {
857
+ usage: vllm_llm::TokenUsage {
858
+ prompt_token_count: 2,
859
+ output_token_count: 1,
860
+ cached_token_count: 0,
861
+ },
862
+ finish_reason: FinishReason::Length,
863
+ kv_transfer_params: None,
864
+ }),
865
+ }),
866
+ ]);
867
+
868
+ let chunks = completion_chunk_stream(
869
+ stream,
870
+ "cmpl-1".to_string(),
871
+ "model".to_string(),
872
+ 1,
873
+ ApiServerOptions::default(),
874
+ ResponseOptions {
875
+ include_usage: true,
876
+ prompt_only: true,
877
+ echo: Some("hello".to_string()),
878
+ return_token_ids: true,
879
+ ..Default::default()
880
+ },
881
+ )
882
+ .collect::<Vec<_>>()
883
+ .await;
884
+
885
+ let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
886
+ assert_eq!(chunks.len(), 3);
887
+
888
+ match &chunks[0] {
889
+ CompletionSseChunk::Chunk(chunk) => {
890
+ assert_eq!(chunk.choices[0].text, "hello");
891
+ assert_eq!(
892
+ chunk.choices[0].prompt_token_ids.as_deref(),
893
+ Some(&[1, 2][..])
894
+ );
895
+ }
896
+ CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
897
+ }
898
+ match &chunks[1] {
899
+ CompletionSseChunk::Chunk(chunk) => {
900
+ assert_eq!(chunk.choices[0].text, "");
901
+ assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
902
+ }
903
+ CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
904
+ }
905
+ match &chunks[2] {
906
+ CompletionSseChunk::Usage(chunk) => {
907
+ let usage = chunk.usage.as_ref().expect("usage");
908
+ assert_eq!(usage.prompt_tokens, 2);
909
+ assert_eq!(usage.completion_tokens, Some(1));
910
+ assert_eq!(usage.total_tokens, 3);
911
+ }
912
+ CompletionSseChunk::Chunk(_) => panic!("expected usage chunk"),
913
+ }
914
+ }
915
+
916
+ #[tokio::test]
917
+ async fn completion_chunk_stream_maps_prompt_logprobs_for_single_token_prompt() {
918
+ let stream = stream::iter(vec![
919
+ Ok(DecodedTextEvent::Start {
920
+ prompt_token_ids: vec![9707].into(),
921
+ prompt_logprobs: None,
922
+ }),
923
+ Ok(DecodedTextEvent::TextDelta {
924
+ delta: " leaked".to_string(),
925
+ token_ids: vec![3],
926
+ logprobs: None,
927
+ finished: Some(Finished {
928
+ usage: vllm_llm::TokenUsage {
929
+ prompt_token_count: 1,
930
+ output_token_count: 1,
931
+ cached_token_count: 0,
932
+ },
933
+ finish_reason: FinishReason::Length,
934
+ kv_transfer_params: None,
935
+ }),
936
+ }),
937
+ ]);
938
+
939
+ let chunks = completion_chunk_stream(
940
+ stream,
941
+ "cmpl-1".to_string(),
942
+ "model".to_string(),
943
+ 1,
944
+ ApiServerOptions::default(),
945
+ ResponseOptions {
946
+ prompt_only: true,
947
+ echo: Some("Hello".to_string()),
948
+ requested_logprobs: Some(1),
949
+ ..Default::default()
950
+ },
951
+ )
952
+ .collect::<Vec<_>>()
953
+ .await;
954
+
955
+ let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
956
+ assert_eq!(chunks.len(), 2);
957
+
958
+ match &chunks[0] {
959
+ CompletionSseChunk::Chunk(chunk) => {
960
+ assert_eq!(chunk.choices[0].text, "Hello");
961
+ let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
962
+ assert_eq!(logprobs.tokens, vec!["Hello".to_string()]);
963
+ assert_eq!(logprobs.token_logprobs, vec![None]);
964
+ assert_eq!(logprobs.top_logprobs, vec![None]);
965
+ assert_eq!(logprobs.text_offset, vec![0]);
966
+ }
967
+ CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
968
+ }
969
+ match &chunks[1] {
970
+ CompletionSseChunk::Chunk(chunk) => {
971
+ assert_eq!(chunk.choices[0].text, "");
972
+ assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
973
+ }
974
+ CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
975
+ }
976
+ }
977
+
978
+ #[tokio::test]
979
+ async fn completion_chunk_stream_maps_prompt_only_logprobs() {
980
+ let stream = stream::iter(vec![
981
+ Ok(DecodedTextEvent::Start {
982
+ prompt_token_ids: vec![1, 2].into(),
983
+ prompt_logprobs: Some(DecodedPromptLogprobs {
984
+ first_token_id: 1,
985
+ first_token: "he".to_string(),
986
+ scored_positions: vec![DecodedPositionLogprobs {
987
+ entries: vec![DecodedTokenLogprob {
988
+ token_id: 2,
989
+ token: "llo".to_string(),
990
+ logprob: -0.2,
991
+ rank: 1,
992
+ }],
993
+ }],
994
+ }),
995
+ }),
996
+ Ok(DecodedTextEvent::TextDelta {
997
+ delta: " leaked".to_string(),
998
+ token_ids: vec![3],
999
+ logprobs: None,
1000
+ finished: Some(Finished {
1001
+ usage: vllm_llm::TokenUsage {
1002
+ prompt_token_count: 2,
1003
+ output_token_count: 1,
1004
+ cached_token_count: 0,
1005
+ },
1006
+ finish_reason: FinishReason::Length,
1007
+ kv_transfer_params: None,
1008
+ }),
1009
+ }),
1010
+ ]);
1011
+
1012
+ let chunks = completion_chunk_stream(
1013
+ stream,
1014
+ "cmpl-1".to_string(),
1015
+ "model".to_string(),
1016
+ 1,
1017
+ ApiServerOptions::default(),
1018
+ ResponseOptions {
1019
+ prompt_only: true,
1020
+ echo: Some("hello".to_string()),
1021
+ requested_logprobs: Some(1),
1022
+ ..Default::default()
1023
+ },
1024
+ )
1025
+ .collect::<Vec<_>>()
1026
+ .await;
1027
+
1028
+ let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
1029
+ assert_eq!(chunks.len(), 2);
1030
+
1031
+ match &chunks[0] {
1032
+ CompletionSseChunk::Chunk(chunk) => {
1033
+ assert_eq!(chunk.choices[0].text, "hello");
1034
+ let logprobs = chunk.choices[0].logprobs.as_ref().expect("logprobs");
1035
+ assert_eq!(logprobs.tokens, vec!["he".to_string(), "llo".to_string()]);
1036
+ assert_eq!(logprobs.token_logprobs, vec![None, Some(-0.2)]);
1037
+ assert_eq!(logprobs.text_offset, vec![0, 2]);
1038
+ }
1039
+ CompletionSseChunk::Usage(_) => panic!("expected prompt chunk"),
1040
+ }
1041
+ match &chunks[1] {
1042
+ CompletionSseChunk::Chunk(chunk) => {
1043
+ assert_eq!(chunk.choices[0].text, "");
1044
+ assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
1045
+ }
1046
+ CompletionSseChunk::Usage(_) => panic!("expected final chunk"),
1047
+ }
1048
+ }
1049
}
rust/src/server/src/routes/openai/completions/convert.rs
+67
-6
@@ -27,6 +27,8 @@ pub(super) struct ResponseOptions {
27
pub include_usage: bool,
28
/// Whether every streamed chunk should carry cumulative usage.
29
pub include_continuous_usage: bool,
30
+ /// Whether the caller requested prompt-only echo via `max_tokens=0`.
31
+ pub prompt_only: bool,
32
/// Original text prompt that should be echoed back northbound when
33
/// `echo=true`.
34
pub echo: Option<String>,
@@ -68,11 +70,13 @@ pub(super) fn prepare_completion_request(
70
})?),
71
None => None,
72
};
71
- let prompt_logprobs = request.prompt_logprobs.or(if request.echo && !request.stream {
72
- logprobs
73
- } else {
74
- None
75
- });
73
+ let prompt_only = request.echo && request.max_tokens == Some(0);
74
+ let prompt_logprobs =
75
+ request.prompt_logprobs.or(if request.echo && (!request.stream || prompt_only) {
76
+ logprobs
77
+ } else {
78
+ None
79
+ });
80
let include_usage = (request.stream_options.as_ref())
81
.and_then(|options| options.include_usage)
82
.unwrap_or(false);
@@ -83,6 +87,11 @@ pub(super) fn prepare_completion_request(
87
.and_then(|options| options.continuous_usage_stats)
88
.unwrap_or(false);
89
let include_prompt_logprobs = prompt_logprobs.is_some();
90
+ let max_tokens = if prompt_only {
91
+ Some(1)
92
+ } else {
93
+ request.max_tokens
94
+ };
95
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
96
97
let structured_outputs =
@@ -97,7 +106,7 @@ pub(super) fn prepare_completion_request(
106
top_p: request.top_p,
107
top_k: request.top_k,
108
seed: request.seed,
100
- max_tokens: request.max_tokens,
109
+ max_tokens,
110
min_tokens: request.min_tokens,
111
logprobs,
112
prompt_logprobs,
@@ -138,6 +147,7 @@ pub(super) fn prepare_completion_request(
147
options: ResponseOptions {
148
include_usage,
149
include_continuous_usage,
150
+ prompt_only,
151
echo,
152
requested_logprobs: request.logprobs,
153
include_prompt_logprobs,
@@ -325,6 +335,57 @@ mod tests {
335
336
assert_eq!(prepared.options.echo, Some("hello".to_string()));
337
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
338
+ assert!(!prepared.options.prompt_only);
339
+ }
340
+
341
+ #[test]
342
+ fn prepare_completion_request_lowers_prompt_only_echo_as_one_internal_token() {
343
+ let request: CompletionRequest = serde_json::from_value(json!({
344
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
345
+ "prompt": "hello",
346
+ "stream": false,
347
+ "echo": true,
348
+ "max_tokens": 0
349
+ }))
350
+ .expect("parse request");
351
+
352
+ let prepared = prepare_completion_request(
353
+ request,
354
+ &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
355
+ ResolvedRequestContext::default(),
356
+ )
357
+ .expect("prepare");
358
+
359
+ assert!(prepared.options.prompt_only);
360
+ assert_eq!(prepared.options.echo, Some("hello".to_string()));
361
+ assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(1));
362
+ }
363
+
364
+ #[test]
365
+ fn prepare_completion_request_enables_prompt_logprobs_for_stream_prompt_only_echo() {
366
+ let request: CompletionRequest = serde_json::from_value(json!({
367
+ "model": "Qwen/Qwen1.5-0.5B-Chat",
368
+ "prompt": "hello",
369
+ "echo": true,
370
+ "stream": true,
371
+ "max_tokens": 0,
372
+ "logprobs": 3
373
+ }))
374
+ .expect("parse request");
375
+
376
+ let prepared = prepare_completion_request(
377
+ request,
378
+ &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
379
+ ResolvedRequestContext::default(),
380
+ )
381
+ .expect("prepare");
382
+
383
+ assert!(prepared.options.prompt_only);
384
+ assert_eq!(prepared.text_request.sampling_params.logprobs, Some(3));
385
+ assert_eq!(
386
+ prepared.text_request.sampling_params.prompt_logprobs,
387
+ Some(3)
388
+ );
389
}
390
391
#[test]
rust/src/server/src/routes/openai/completions/validate.rs
+31
-2
@@ -26,8 +26,11 @@ pub(super) fn validate_request_compat(
26
bail_invalid_request!(param = "n", "Only n=1 is supported.");
27
}
28
29
- if request.max_tokens == Some(0) {
30
- bail_invalid_request!(param = "max_tokens", "max_tokens must be greater than 0.");
29
+ if request.max_tokens == Some(0) && !request.echo {
30
+ bail_invalid_request!(
31
+ param = "max_tokens",
32
+ "max_tokens=0 is only supported when echo=true."
33
+ );
34
}
35
36
if request.echo && matches!(request.prompt, Prompt::TokenIds(_)) {
@@ -166,4 +169,30 @@ mod tests {
169
validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
170
);
171
}
172
+
173
+ #[test]
174
+ fn validate_request_compat_accepts_prompt_only_echo() {
175
+ let request = CompletionRequest {
176
+ stream: false,
177
+ echo: true,
178
+ max_tokens: Some(0),
179
+ ..base_request()
180
+ };
181
+ assert!(
182
+ validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok()
183
+ );
184
+ }
185
+
186
+ #[test]
187
+ fn validate_request_compat_rejects_prompt_only_without_echo() {
188
+ let request = CompletionRequest {
189
+ stream: false,
190
+ echo: false,
191
+ max_tokens: Some(0),
192
+ ..base_request()
193
+ };
194
+ assert!(
195
+ validate_request_compat(&request, &served_names(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
196
+ );
197
+ }
198
}