A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 10:15 UTC
[Bugfix] missing tokens occur in harmony streaming (#30437)
Signed-off-by: RioS <aa248424@gmail.com> Signed-off-by: Ri0S <aa248424@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
RioS committed
Jan 9, 2026 at 12:59 UTC
e2d49ec2a4818186b8bafda43f97c298ff91a35e
2 files changed
+13
-7
vllm/entrypoints/context.py
+6
@@ -824,6 +824,7 @@ class StreamingHarmonyContext(HarmonyContext):
824
self.encoding = get_encoding()
825
self.last_tok = None
826
self.first_tok_of_message = True
827
+ self.last_content_delta = None
828
829
@property
830
def messages(self) -> list:
@@ -832,6 +833,7 @@ class StreamingHarmonyContext(HarmonyContext):
833
def append_output(self, output: RequestOutput) -> None:
834
# append_output is called for each output token in streaming case,
835
# so we only want to add the prompt tokens once for each message.
836
+ self.last_content_delta = None
837
if self.first_tok_of_message:
838
self._update_prefill_token_usage(output)
839
# Reset self.first_tok_of_message if needed:
@@ -839,8 +841,12 @@ class StreamingHarmonyContext(HarmonyContext):
841
# (finished=True), then the next token processed will mark the
842
# beginning of a new message
843
self.first_tok_of_message = output.finished
844
+ last_delta_text = ""
845
for tok in output.outputs[0].token_ids:
846
self.parser.process(tok)
847
+ last_delta_text += self.parser.last_content_delta or ""
848
+ if last_delta_text:
849
+ self.last_content_delta = last_delta_text
850
self._update_decode_token_usage(output)
851
852
# For streaming, update previous turn when message is complete
vllm/entrypoints/openai/serving_responses.py
+7
-7
@@ -1811,7 +1811,7 @@ class OpenAIServingResponses(OpenAIServing):
1811
content_index=state.current_content_index,
1812
output_index=state.current_output_index,
1813
item_id=state.current_item_id,
1814
- delta=ctx.parser.last_content_delta,
1814
+ delta=ctx.last_content_delta,
1815
# TODO, use logprobs from ctx.last_request_output
1816
logprobs=[],
1817
)
@@ -1861,7 +1861,7 @@ class OpenAIServingResponses(OpenAIServing):
1861
item_id=state.current_item_id,
1862
output_index=state.current_output_index,
1863
content_index=state.current_content_index,
1864
- delta=ctx.parser.last_content_delta,
1864
+ delta=ctx.last_content_delta,
1865
sequence_number=-1,
1866
)
1867
)
@@ -1908,7 +1908,7 @@ class OpenAIServingResponses(OpenAIServing):
1908
sequence_number=-1,
1909
output_index=state.current_output_index,
1910
item_id=state.current_item_id,
1911
- delta=ctx.parser.last_content_delta,
1911
+ delta=ctx.last_content_delta,
1912
)
1913
)
1914
return events
@@ -1952,7 +1952,7 @@ class OpenAIServingResponses(OpenAIServing):
1952
sequence_number=-1,
1953
output_index=state.current_output_index,
1954
item_id=state.current_item_id,
1955
- delta=ctx.parser.last_content_delta,
1955
+ delta=ctx.last_content_delta,
1956
)
1957
)
1958
return events
@@ -1999,7 +1999,7 @@ class OpenAIServingResponses(OpenAIServing):
1999
sequence_number=-1,
2000
output_index=state.current_output_index,
2001
item_id=state.current_item_id,
2002
- delta=ctx.parser.last_content_delta,
2002
+ delta=ctx.last_content_delta,
2003
)
2004
)
2005
return events
@@ -2010,7 +2010,7 @@ class OpenAIServingResponses(OpenAIServing):
2010
state: HarmonyStreamingState,
2011
) -> list[StreamingResponsesResponse]:
2012
"""Emit events for content delta streaming based on channel type."""
2013
- if not ctx.parser.last_content_delta:
2013
+ if not ctx.last_content_delta:
2014
return []
2015
2016
if (
@@ -2364,7 +2364,7 @@ class OpenAIServingResponses(OpenAIServing):
2364
events.append(
2365
ResponseFunctionCallArgumentsDeltaEvent(
2366
item_id=state.current_item_id,
2367
- delta=ctx.parser.last_content_delta,
2367
+ delta=ctx.last_content_delta,
2368
output_index=state.current_output_index,
2369
sequence_number=-1,
2370
type="response.function_call_arguments.delta",