A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Fixed stream generator for gpt-oss + spec-decoding (#26027)
Signed-off-by: Aleksandr Samarin <astrlrd@nebius.com>
Aleksandr Samarin committed
Oct 3, 2025 at 15:43 UTC
73a99cc2a53ccabaa9fee8ebc583c2bcdc998b8e
1 file changed
+3
-1
vllm/entrypoints/openai/serving_chat.py
+3
-1
@@ -691,11 +691,13 @@ class OpenAIServingChat(OpenAIServing):
691
if self.use_harmony:
692
harmony_parser = harmony_parsers[i]
693
prev_recipient = harmony_parser.current_recipient
694
+ delta_text = ""
695
for token_id in output.token_ids:
696
harmony_parser.process(token_id)
697
+ delta_text += (harmony_parser.last_content_delta
698
+ or "")
699
cur_channel = harmony_parser.current_channel
700
cur_recipient = harmony_parser.current_recipient
698
- delta_text = harmony_parser.last_content_delta or ""
701
else:
702
delta_text = output.text
703