A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix] Fix include prompt in stream response when echo=true (#15233)
Signed-off-by: Yuan Fang <yuanfang@alauda.io>
fyuan1316 committed
Jul 1, 2025 at 09:30 UTC
e28533a16f73a4eae01c2b7b1b4ddf3fc1beedab
2 files changed
+71
-4
tests/entrypoints/openai/test_completion.py
+54
@@ -779,3 +779,57 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
779
prompt="Give an example string that fits this regex",
780
extra_body=dict(guided_regex=sample_regex,
781
guided_json=sample_json_schema))
782
+
783
+
784
+@pytest.mark.asyncio
785
+@pytest.mark.parametrize(
786
+ "model_name,stream,echo",
787
+ [
788
+ (MODEL_NAME, False, False),
789
+ (MODEL_NAME, False, True),
790
+ (MODEL_NAME, True, False),
791
+ (MODEL_NAME, True, True) # should not raise BadRequestError error
792
+ ],
793
+)
794
+async def test_echo_stream_completion(client: openai.AsyncOpenAI,
795
+ model_name: str, stream: bool,
796
+ echo: bool):
797
+ saying: str = "Hello, my name is"
798
+ result = await client.completions.create(model=model_name,
799
+ prompt=saying,
800
+ max_tokens=10,
801
+ temperature=0.0,
802
+ echo=echo,
803
+ stream=stream)
804
+
805
+ stop_reason = "length"
806
+
807
+ if not stream:
808
+ completion = result
809
+ assert completion.id is not None
810
+ assert completion.choices is not None and len(completion.choices) == 1
811
+
812
+ choice = completion.choices[0]
813
+ assert len(choice.text) >= 5
814
+ assert choice.finish_reason == stop_reason
815
+
816
+ if echo:
817
+ assert choice.text is not None and saying in choice.text
818
+ else:
819
+ assert choice.text is not None and saying not in choice.text
820
+
821
+ else:
822
+ chunks: list[str] = []
823
+ final_finish_reason = None
824
+ async for chunk in result:
825
+ if chunk.choices and chunk.choices[0].text:
826
+ chunks.append(chunk.choices[0].text)
827
+ if chunk.choices and chunk.choices[0].finish_reason:
828
+ final_finish_reason = chunk.choices[0].finish_reason
829
+
830
+ assert final_finish_reason == stop_reason
831
+ content = "".join(chunks)
832
+ if echo:
833
+ assert content is not None and saying in content
834
+ else:
835
+ assert content is not None and saying not in content
vllm/entrypoints/openai/serving_completion.py
+17
-4
@@ -25,10 +25,13 @@ from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
25
ErrorResponse,
26
RequestResponseMetadata,
27
UsageInfo)
28
-# yapf: enable
28
+from vllm.entrypoints.openai.serving_engine import (
29
+ EmbedsPrompt as ServingEngineEmbedsPrompt)
30
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
31
+ TextTokensPrompt,
32
clamp_prompt_logprobs,
33
is_text_tokens_prompt)
34
+# yapf: enable
35
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
36
from vllm.inputs.data import (EmbedsPrompt, TokensPrompt, is_embeds_prompt,
37
is_tokens_prompt)
@@ -223,6 +226,7 @@ class OpenAIServingCompletion(OpenAIServing):
226
if stream:
227
return self.completion_stream_generator(
228
request,
229
+ request_prompts,
230
result_generator,
231
request_id,
232
created_time,
@@ -285,6 +289,8 @@ class OpenAIServingCompletion(OpenAIServing):
289
async def completion_stream_generator(
290
self,
291
request: CompletionRequest,
292
+ request_prompts: list[Union[TextTokensPrompt,
293
+ ServingEngineEmbedsPrompt]],
294
result_generator: AsyncIterator[tuple[int, RequestOutput]],
295
request_id: str,
296
created_time: int,
@@ -313,7 +319,15 @@ class OpenAIServingCompletion(OpenAIServing):
319
async for prompt_idx, res in result_generator:
320
prompt_token_ids = res.prompt_token_ids
321
prompt_logprobs = res.prompt_logprobs
316
- prompt_text = res.prompt
322
+
323
+ if res.prompt is not None:
324
+ prompt_text = res.prompt
325
+ else:
326
+ request_prompt = request_prompts[prompt_idx]
327
+ if is_text_tokens_prompt(request_prompt):
328
+ prompt_text = request_prompt["prompt"]
329
+ else:
330
+ prompt_text = None
331
332
# Prompt details are excluded from later streamed outputs
333
if prompt_token_ids is not None:
@@ -336,14 +350,13 @@ class OpenAIServingCompletion(OpenAIServing):
350
delta_token_ids = prompt_token_ids
351
out_logprobs = prompt_logprobs
352
else:
339
- assert prompt_logprobs is not None
353
# echo the prompt and first token
354
delta_text = prompt_text + output.text
355
delta_token_ids = [
356
*prompt_token_ids, *output.token_ids
357
]
358
out_logprobs = [
346
- *prompt_logprobs,
359
+ *(prompt_logprobs or []),
360
*(output.logprobs or []),
361
]
362
has_echoed[i] = True