A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 04:00 UTC
[Bug] Fix a corner case in _process_simple_streaming_events (#34754)
Signed-off-by: Shiyan Deng <dsy842974287@meta.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Shiyan Deng committed
Mar 5, 2026 at 20:57 UTC
8e87cc57f1b071d69a93b5d5aa27a5841f817739
2 files changed
+334
tests/entrypoints/openai/test_serving_responses.py
+314
@@ -6,6 +6,13 @@ from unittest.mock import MagicMock
6
7
import pytest
8
import pytest_asyncio
9
+from openai.types.responses import (
10
+ ResponseOutputItemDoneEvent,
11
+ ResponseReasoningItem,
12
+ ResponseReasoningTextDeltaEvent,
13
+ ResponseReasoningTextDoneEvent,
14
+ ResponseTextDeltaEvent,
15
+)
16
from openai.types.responses.tool import (
17
CodeInterpreterContainerCodeInterpreterToolAuto,
18
LocalShell,
@@ -16,6 +23,7 @@ from openai.types.responses.tool import (
23
import vllm.envs as envs
24
from vllm.entrypoints.mcp.tool_server import ToolServer
25
from vllm.entrypoints.openai.engine.protocol import (
26
+ DeltaMessage,
27
ErrorResponse,
28
RequestResponseMetadata,
29
)
@@ -554,3 +562,309 @@ class TestHarmonyPreambleStreaming:
562
563
type_names = [e.type for e in events]
564
assert "response.output_text.done" not in type_names
565
+
566
+
567
+def _make_simple_context_with_output(text, token_ids):
568
+ """Create a SimpleContext with a RequestOutput containing the given text."""
569
+ ctx = SimpleContext()
570
+ completion = CompletionOutput(
571
+ index=0,
572
+ text=text,
573
+ token_ids=token_ids,
574
+ cumulative_logprob=0.0,
575
+ logprobs=None,
576
+ finish_reason=None,
577
+ stop_reason=None,
578
+ )
579
+ req_output = RequestOutput(
580
+ request_id="req",
581
+ prompt="hi",
582
+ prompt_token_ids=[7, 8],
583
+ prompt_logprobs=None,
584
+ outputs=[completion],
585
+ finished=False,
586
+ num_cached_tokens=0,
587
+ )
588
+ ctx.append_output(req_output)
589
+ return ctx
590
+
591
+
592
+def _make_serving_instance_with_reasoning():
593
+ """Create an OpenAIServingResponses with a mocked reasoning parser."""
594
+ engine_client = MagicMock()
595
+ model_config = MagicMock()
596
+ model_config.max_model_len = 100
597
+ model_config.hf_config.model_type = "test"
598
+ model_config.hf_text_config = MagicMock()
599
+ model_config.get_diff_sampling_param.return_value = {}
600
+ engine_client.model_config = model_config
601
+ engine_client.input_processor = MagicMock()
602
+ engine_client.io_processor = MagicMock()
603
+ engine_client.renderer = MagicMock()
604
+
605
+ models = MagicMock()
606
+
607
+ serving = OpenAIServingResponses(
608
+ engine_client=engine_client,
609
+ models=models,
610
+ request_logger=None,
611
+ chat_template=None,
612
+ chat_template_content_format="auto",
613
+ reasoning_parser="qwen3",
614
+ )
615
+ return serving
616
+
617
+
618
+def _identity_increment(event):
619
+ """Simple identity callable for _increment_sequence_number_and_return."""
620
+ seq = getattr(_identity_increment, "_counter", 0)
621
+ if hasattr(event, "sequence_number"):
622
+ event.sequence_number = seq
623
+ _identity_increment._counter = seq + 1 # type: ignore
624
+ return event
625
+
626
+
627
+class TestStreamingReasoningToContentTransition:
628
+ """Tests for _process_simple_streaming_events reasoning-to-content
629
+ transition, specifically the fix for mixed deltas that carry both
630
+ reasoning and content simultaneously."""
631
+
632
+ @pytest.mark.asyncio
633
+ async def test_mixed_delta_reasoning_and_content_emits_reasoning_delta(
634
+ self, monkeypatch
635
+ ):
636
+ """When the reasoning parser produces a delta with both reasoning
637
+ and content set (e.g. reasoning end and content start in the same
638
+ chunk), the trailing reasoning text must be emitted as a
639
+ ResponseReasoningTextDeltaEvent and included in the
640
+ ResponseReasoningTextDoneEvent text."""
641
+
642
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
643
+ serving = _make_serving_instance_with_reasoning()
644
+
645
+ # Sequence of DeltaMessages the mock reasoning parser will return
646
+ delta_sequence = [
647
+ DeltaMessage(reasoning="thinking..."),
648
+ DeltaMessage(reasoning=" end", content="hello"), # mixed delta
649
+ DeltaMessage(content=" world"),
650
+ ]
651
+ call_count = 0
652
+
653
+ def mock_extract_reasoning_streaming(**kwargs):
654
+ nonlocal call_count
655
+ result = delta_sequence[call_count]
656
+ call_count += 1
657
+ return result
658
+
659
+ # Mock the reasoning parser on the serving instance
660
+ mock_parser = MagicMock()
661
+ mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
662
+ serving.parser = MagicMock()
663
+ serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
664
+
665
+ # Create contexts for each streaming chunk
666
+ contexts = [
667
+ _make_simple_context_with_output("chunk1", [10]),
668
+ _make_simple_context_with_output("chunk2", [20]),
669
+ _make_simple_context_with_output("chunk3", [30]),
670
+ ]
671
+
672
+ async def result_generator():
673
+ for ctx in contexts:
674
+ yield ctx
675
+
676
+ request = ResponsesRequest(input="hi", tools=[], stream=True)
677
+ sampling_params = SamplingParams(max_tokens=64)
678
+ metadata = RequestResponseMetadata(request_id="req")
679
+ _identity_increment._counter = 0 # type: ignore
680
+
681
+ events = []
682
+ async for event in serving._process_simple_streaming_events(
683
+ request=request,
684
+ sampling_params=sampling_params,
685
+ result_generator=result_generator(),
686
+ context=SimpleContext(),
687
+ model_name="test-model",
688
+ tokenizer=MagicMock(),
689
+ request_metadata=metadata,
690
+ created_time=0,
691
+ _increment_sequence_number_and_return=_identity_increment,
692
+ ):
693
+ events.append(event)
694
+
695
+ # The first reasoning delta should be emitted
696
+ reasoning_deltas = [
697
+ e for e in events if isinstance(e, ResponseReasoningTextDeltaEvent)
698
+ ]
699
+ assert len(reasoning_deltas) == 2
700
+ assert reasoning_deltas[0].delta == "thinking..."
701
+ # The trailing reasoning from the mixed delta must also be emitted
702
+ assert reasoning_deltas[1].delta == " end"
703
+
704
+ # The done event must include both reasoning parts
705
+ reasoning_done = [
706
+ e for e in events if isinstance(e, ResponseReasoningTextDoneEvent)
707
+ ]
708
+ assert len(reasoning_done) == 1
709
+ assert reasoning_done[0].text == "thinking... end"
710
+
711
+ # Content deltas should be emitted for both the mixed delta's
712
+ # content and the pure content delta
713
+ text_deltas = [e for e in events if isinstance(e, ResponseTextDeltaEvent)]
714
+ assert len(text_deltas) == 2
715
+ assert text_deltas[0].delta == "hello"
716
+ assert text_deltas[1].delta == " world"
717
+
718
+ @pytest.mark.asyncio
719
+ async def test_transition_without_mixed_delta_no_extra_reasoning_event(
720
+ self, monkeypatch
721
+ ):
722
+ """When the transition from reasoning to content is clean (no mixed
723
+ delta), no extra reasoning delta event should be emitted."""
724
+
725
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
726
+ serving = _make_serving_instance_with_reasoning()
727
+
728
+ delta_sequence = [
729
+ DeltaMessage(reasoning="thinking"),
730
+ DeltaMessage(content="answer"),
731
+ ]
732
+ call_count = 0
733
+
734
+ def mock_extract_reasoning_streaming(**kwargs):
735
+ nonlocal call_count
736
+ result = delta_sequence[call_count]
737
+ call_count += 1
738
+ return result
739
+
740
+ mock_parser = MagicMock()
741
+ mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
742
+ serving.parser = MagicMock()
743
+ serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
744
+
745
+ contexts = [
746
+ _make_simple_context_with_output("chunk1", [10]),
747
+ _make_simple_context_with_output("chunk2", [20]),
748
+ ]
749
+
750
+ async def result_generator():
751
+ for ctx in contexts:
752
+ yield ctx
753
+
754
+ request = ResponsesRequest(input="hi", tools=[], stream=True)
755
+ sampling_params = SamplingParams(max_tokens=64)
756
+ metadata = RequestResponseMetadata(request_id="req")
757
+ _identity_increment._counter = 0 # type: ignore
758
+
759
+ events = []
760
+ async for event in serving._process_simple_streaming_events(
761
+ request=request,
762
+ sampling_params=sampling_params,
763
+ result_generator=result_generator(),
764
+ context=SimpleContext(),
765
+ model_name="test-model",
766
+ tokenizer=MagicMock(),
767
+ request_metadata=metadata,
768
+ created_time=0,
769
+ _increment_sequence_number_and_return=_identity_increment,
770
+ ):
771
+ events.append(event)
772
+
773
+ # Exactly one reasoning delta
774
+ reasoning_deltas = [
775
+ e for e in events if isinstance(e, ResponseReasoningTextDeltaEvent)
776
+ ]
777
+ assert len(reasoning_deltas) == 1
778
+ assert reasoning_deltas[0].delta == "thinking"
779
+
780
+ # Done event has just "thinking"
781
+ reasoning_done = [
782
+ e for e in events if isinstance(e, ResponseReasoningTextDoneEvent)
783
+ ]
784
+ assert len(reasoning_done) == 1
785
+ assert reasoning_done[0].text == "thinking"
786
+
787
+ # One content delta
788
+ text_deltas = [e for e in events if isinstance(e, ResponseTextDeltaEvent)]
789
+ assert len(text_deltas) == 1
790
+ assert text_deltas[0].delta == "answer"
791
+
792
+ @pytest.mark.asyncio
793
+ async def test_reasoning_only_stream_no_content(self, monkeypatch):
794
+ """When the stream has only reasoning deltas and no content, the
795
+ reasoning done event should be emitted at finalization with the
796
+ full accumulated text, and no text delta events should appear."""
797
+
798
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
799
+ serving = _make_serving_instance_with_reasoning()
800
+
801
+ delta_sequence = [
802
+ DeltaMessage(reasoning="step 1"),
803
+ DeltaMessage(reasoning=" step 2"),
804
+ ]
805
+ call_count = 0
806
+
807
+ def mock_extract_reasoning_streaming(**kwargs):
808
+ nonlocal call_count
809
+ result = delta_sequence[call_count]
810
+ call_count += 1
811
+ return result
812
+
813
+ mock_parser = MagicMock()
814
+ mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
815
+ serving.parser = MagicMock()
816
+ serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
817
+
818
+ contexts = [
819
+ _make_simple_context_with_output("chunk1", [10]),
820
+ _make_simple_context_with_output("chunk2", [20]),
821
+ ]
822
+
823
+ async def result_generator():
824
+ for ctx in contexts:
825
+ yield ctx
826
+
827
+ request = ResponsesRequest(input="hi", tools=[], stream=True)
828
+ sampling_params = SamplingParams(max_tokens=64)
829
+ metadata = RequestResponseMetadata(request_id="req")
830
+ _identity_increment._counter = 0 # type: ignore
831
+
832
+ events = []
833
+ async for event in serving._process_simple_streaming_events(
834
+ request=request,
835
+ sampling_params=sampling_params,
836
+ result_generator=result_generator(),
837
+ context=SimpleContext(),
838
+ model_name="test-model",
839
+ tokenizer=MagicMock(),
840
+ request_metadata=metadata,
841
+ created_time=0,
842
+ _increment_sequence_number_and_return=_identity_increment,
843
+ ):
844
+ events.append(event)
845
+
846
+ # Two reasoning deltas
847
+ reasoning_deltas = [
848
+ e for e in events if isinstance(e, ResponseReasoningTextDeltaEvent)
849
+ ]
850
+ assert len(reasoning_deltas) == 2
851
+ assert reasoning_deltas[0].delta == "step 1"
852
+ assert reasoning_deltas[1].delta == " step 2"
853
+
854
+ # Done event at finalization with accumulated text
855
+ reasoning_done = [
856
+ e for e in events if isinstance(e, ResponseReasoningTextDoneEvent)
857
+ ]
858
+ assert len(reasoning_done) == 1
859
+ assert reasoning_done[0].text == "step 1 step 2"
860
+
861
+ # No content text deltas
862
+ text_deltas = [e for e in events if isinstance(e, ResponseTextDeltaEvent)]
863
+ assert len(text_deltas) == 0
864
+
865
+ # Final item should be a reasoning item
866
+ item_done_events = [
867
+ e for e in events if isinstance(e, ResponseOutputItemDoneEvent)
868
+ ]
869
+ assert len(item_done_events) == 1
870
+ assert isinstance(item_done_events[0].item, ResponseReasoningItem)
vllm/entrypoints/openai/responses/serving.py
+20
@@ -1364,6 +1364,26 @@ class OpenAIServingResponses(OpenAIServing):
1364
for pm in previous_delta_messages
1365
if pm.reasoning is not None
1366
)
1367
+
1368
+ # delta message could have both reasoning and
1369
+ # content. Include current delta's reasoning in the
1370
+ # finalization since it may carry the tail end of
1371
+ # reasoning text (e.g. when reasoning end and
1372
+ # content start arrive in the same delta).
1373
+ if delta_message.reasoning is not None:
1374
+ yield _increment_sequence_number_and_return(
1375
+ ResponseReasoningTextDeltaEvent(
1376
+ type="response.reasoning_text.delta",
1377
+ sequence_number=-1,
1378
+ content_index=current_content_index,
1379
+ output_index=current_output_index,
1380
+ item_id=current_item_id,
1381
+ delta=delta_message.reasoning,
1382
+ )
1383
+ )
1384
+ reason_content += delta_message.reasoning
1385
+ delta_message = DeltaMessage(content=delta_message.content)
1386
+
1387
yield _increment_sequence_number_and_return(
1388
ResponseReasoningTextDoneEvent(
1389
type="response.reasoning_text.done",