A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Parser] Migrate response api streaming to unified parser (#38755)
Signed-off-by: sfeng33 <4florafeng@gmail.com> Signed-off-by: Andrew Xia <axia@meta.com>
Flora Feng committed
Apr 7, 2026 at 22:09 UTC
927975ead8ea8f2c818844680fd7b29d0f62d0e9
3 files changed
+154
-120
tests/entrypoints/openai/responses/test_serving_responses.py
+29
-44
@@ -628,6 +628,31 @@ def _identity_increment(event):
628
return event
629
630
631
+def _mock_parser_with_reasoning(serving, delta_sequence: list[DeltaMessage]):
632
+ """Set up serving.parser so that it returns a mock parser instance
633
+ with a reasoning parser that returns the given delta_sequence.
634
+
635
+ The mock has reasoning_parser set (truthy) but tool_parser as None,
636
+ so the parser's parse_delta enters the reasoning-only branch.
637
+ """
638
+ call_count = 0
639
+
640
+ def mock_parse_delta(**kwargs):
641
+ nonlocal call_count
642
+ if call_count >= len(delta_sequence):
643
+ return None
644
+ result = delta_sequence[call_count]
645
+ call_count += 1
646
+ return result
647
+
648
+ mock_parser_instance = MagicMock()
649
+ mock_parser_instance.reasoning_parser = MagicMock() # truthy
650
+ mock_parser_instance.tool_parser = None
651
+ mock_parser_instance.parse_delta = mock_parse_delta
652
+ mock_parser_instance.is_reasoning_end = MagicMock(return_value=False)
653
+ serving.parser = MagicMock(return_value=mock_parser_instance)
654
+
655
+
656
class TestStreamingReasoningToContentTransition:
657
"""Tests for _process_simple_streaming_events reasoning-to-content
658
transition, specifically the fix for mixed deltas that carry both
@@ -646,27 +671,13 @@ class TestStreamingReasoningToContentTransition:
671
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
672
serving = _make_serving_instance_with_reasoning()
673
649
- # Sequence of DeltaMessages the mock reasoning parser will return
674
+ # Sequence of DeltaMessages the mock orchestrator will return
675
delta_sequence = [
676
DeltaMessage(reasoning="thinking..."),
677
DeltaMessage(reasoning=" end", content="hello"), # mixed delta
678
DeltaMessage(content=" world"),
679
]
655
- call_count = 0
656
-
657
- def mock_extract_reasoning_streaming(**kwargs):
658
- nonlocal call_count
659
- result = delta_sequence[call_count]
660
- call_count += 1
661
- return result
662
-
663
- # Mock the reasoning parser on the serving instance
664
- mock_parser = MagicMock()
665
- mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
666
- mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
667
- serving.parser = MagicMock()
668
- serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
669
- serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
680
+ _mock_parser_with_reasoning(serving, delta_sequence)
681
# Create contexts for each streaming chunk
682
contexts = [
683
_make_simple_context_with_output("chunk1", [10]),
@@ -734,20 +745,7 @@ class TestStreamingReasoningToContentTransition:
745
DeltaMessage(reasoning="thinking"),
746
DeltaMessage(content="answer"),
747
]
737
- call_count = 0
738
-
739
- def mock_extract_reasoning_streaming(**kwargs):
740
- nonlocal call_count
741
- result = delta_sequence[call_count]
742
- call_count += 1
743
- return result
744
-
745
- mock_parser = MagicMock()
746
- mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
747
- mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
748
- serving.parser = MagicMock()
749
- serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
750
- serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
748
+ _mock_parser_with_reasoning(serving, delta_sequence)
749
750
contexts = [
751
_make_simple_context_with_output("chunk1", [10]),
@@ -809,20 +807,7 @@ class TestStreamingReasoningToContentTransition:
807
DeltaMessage(reasoning="step 1"),
808
DeltaMessage(reasoning=" step 2"),
809
]
812
- call_count = 0
813
-
814
- def mock_extract_reasoning_streaming(**kwargs):
815
- nonlocal call_count
816
- result = delta_sequence[call_count]
817
- call_count += 1
818
- return result
819
-
820
- mock_parser = MagicMock()
821
- mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
822
- mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
823
- serving.parser = MagicMock()
824
- serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
825
- serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
810
+ _mock_parser_with_reasoning(serving, delta_sequence)
811
812
contexts = [
813
_make_simple_context_with_output("chunk1", [10]),
vllm/entrypoints/openai/responses/serving.py
+6
-76
@@ -1339,101 +1339,31 @@ class OpenAIServingResponses(OpenAIServing):
1339
current_content_index = 0
1340
current_output_index = 0
1341
current_item_id = ""
1342
- reasoning_parser = None
1343
- if self.parser and self.parser.reasoning_parser_cls:
1344
- reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
1345
- tool_parser = None
1346
- if self.parser and self.parser.tool_parser_cls:
1347
- tool_parser = self.parser.tool_parser_cls(tokenizer, request.tools)
1348
- reasoning_ended = False
1349
- tool_call_text_started = False
1350
- previous_text = ""
1351
- previous_token_ids: list[int] = []
1352
- prompt_is_reasoning_end = None
1342
+ parser = self.parser(tokenizer, request.tools) if self.parser else None
1343
first_delta_sent = False
1344
previous_delta_messages: list[DeltaMessage] = []
1345
async for ctx in result_generator:
1346
assert isinstance(ctx, SimpleContext)
1347
if ctx.last_output is None:
1348
continue
1359
- if reasoning_parser and prompt_is_reasoning_end is None:
1360
- prompt_is_reasoning_end = reasoning_parser.is_reasoning_end(
1361
- ctx.last_output.prompt_token_ids
1362
- )
1349
if ctx.last_output.outputs:
1350
output = ctx.last_output.outputs[0]
1351
# finish_reason='error' indicates a retryable error
1352
self._raise_if_error(output.finish_reason, request.request_id)
1353
delta_text = output.text
1354
delta_token_ids = as_list(output.token_ids)
1369
- current_text = previous_text + delta_text
1370
- current_token_ids = previous_token_ids + delta_token_ids
1371
-
1372
- if reasoning_parser and tool_parser:
1373
- if prompt_is_reasoning_end:
1374
- reasoning_ended = True
1375
- if not reasoning_ended:
1376
- delta_message = reasoning_parser.extract_reasoning_streaming(
1377
- previous_text=previous_text,
1378
- current_text=current_text,
1379
- delta_text=delta_text,
1380
- previous_token_ids=previous_token_ids,
1381
- current_token_ids=current_token_ids,
1382
- delta_token_ids=delta_token_ids,
1383
- )
1384
- if reasoning_parser.is_reasoning_end(delta_token_ids):
1385
- reasoning_ended = True
1386
- current_token_ids = reasoning_parser.extract_content_ids(
1387
- delta_token_ids
1388
- )
1389
- if delta_message and delta_message.content:
1390
- current_text = delta_message.content
1391
- delta_message.content = None
1392
- else:
1393
- current_text = ""
1394
-
1395
- if reasoning_ended:
1396
- if not tool_call_text_started:
1397
- tool_call_text_started = True
1398
- previous_text = ""
1399
- previous_token_ids = []
1400
- delta_text = current_text
1401
- delta_token_ids = current_token_ids
1402
-
1403
- delta_message = tool_parser.extract_tool_calls_streaming(
1404
- previous_text=previous_text,
1405
- current_text=current_text,
1406
- delta_text=delta_text,
1407
- previous_token_ids=previous_token_ids,
1408
- current_token_ids=current_token_ids,
1409
- delta_token_ids=delta_token_ids,
1410
- request=request, # type: ignore[arg-type]
1411
- )
1412
- elif reasoning_parser:
1413
- delta_message = reasoning_parser.extract_reasoning_streaming(
1414
- previous_text=previous_text,
1415
- current_text=current_text,
1416
- delta_text=delta_text,
1417
- previous_token_ids=previous_token_ids,
1418
- current_token_ids=current_token_ids,
1419
- delta_token_ids=delta_token_ids,
1420
- )
1421
- elif tool_parser:
1422
- delta_message = tool_parser.extract_tool_calls_streaming(
1423
- previous_text=previous_text,
1424
- current_text=current_text,
1355
+
1356
+ if parser:
1357
+ delta_message = parser.parse_delta(
1358
delta_text=delta_text,
1426
- previous_token_ids=previous_token_ids,
1427
- current_token_ids=current_token_ids,
1359
delta_token_ids=delta_token_ids,
1429
- request=request, # type: ignore[arg-type]
1360
+ request=request,
1361
+ prompt_token_ids=ctx.last_output.prompt_token_ids,
1362
)
1363
else:
1364
delta_message = DeltaMessage(
1365
content=output.text,
1366
)
1435
- previous_text = current_text
1436
- previous_token_ids = current_token_ids
1367
if not delta_message:
1368
continue
1369
if not first_delta_sent:
vllm/parser/abstract_parser.py
+119
@@ -5,6 +5,7 @@ import contextlib
5
import json
6
from abc import abstractmethod
7
from collections.abc import Sequence
8
+from dataclasses import dataclass, field
9
from functools import cached_property
10
11
from openai.types.responses import (
@@ -43,6 +44,17 @@ from vllm.utils import random_uuid
44
logger = init_logger(__name__)
45
46
47
+@dataclass
48
+class StreamState:
49
+ """Mutable state for ``Parser.parse_delta()``. One per stream."""
50
+
51
+ reasoning_ended: bool = False
52
+ tool_call_text_started: bool = False
53
+ prompt_reasoning_checked: bool = False
54
+ previous_text: str = ""
55
+ previous_token_ids: list[int] = field(default_factory=list)
56
+
57
+
58
class Parser:
59
"""
60
Abstract Parser class that unifies ReasoningParser and ToolParser into
@@ -80,6 +92,7 @@ class Parser:
92
self.model_tokenizer = tokenizer
93
self._reasoning_parser: ReasoningParser | None = None
94
self._tool_parser: ToolParser | None = None
95
+ self._stream_state = StreamState()
96
97
@cached_property
98
def vocab(self) -> dict[str, int]:
@@ -291,6 +304,18 @@ class Parser:
304
A DeltaMessage with tool_calls field, or None.
305
"""
306
307
+ @abstractmethod
308
+ def parse_delta(
309
+ self,
310
+ delta_text: str,
311
+ delta_token_ids: list[int],
312
+ request: ChatCompletionRequest | ResponsesRequest,
313
+ prompt_token_ids: list[int] | None = None,
314
+ ) -> DeltaMessage | None:
315
+ """Parse a single streaming delta, orchestrating reasoning then
316
+ tool call extraction via internal stream state.
317
+ """
318
+
319
320
class DelegatingParser(Parser):
321
"""
@@ -524,6 +549,100 @@ class DelegatingParser(Parser):
549
request,
550
)
551
552
+ def is_reasoning_end(self, input_ids: list[int]) -> bool:
553
+ if self._reasoning_parser is None:
554
+ return False
555
+ return self._reasoning_parser.is_reasoning_end(input_ids)
556
+
557
+ def extract_content_ids(self, input_ids: list[int]) -> list[int]:
558
+ if self._reasoning_parser is None:
559
+ return input_ids
560
+ return self._reasoning_parser.extract_content_ids(input_ids)
561
+
562
+ def parse_delta(
563
+ self,
564
+ delta_text: str,
565
+ delta_token_ids: list[int],
566
+ request: ChatCompletionRequest | ResponsesRequest,
567
+ prompt_token_ids: list[int] | None = None,
568
+ ) -> DeltaMessage | None:
569
+ state = self._stream_state
570
+
571
+ if not state.prompt_reasoning_checked and prompt_token_ids is not None:
572
+ state.prompt_reasoning_checked = True
573
+ if self.is_reasoning_end(prompt_token_ids):
574
+ state.reasoning_ended = True
575
+
576
+ current_text = state.previous_text + delta_text
577
+ current_token_ids = state.previous_token_ids + delta_token_ids
578
+
579
+ delta_message: DeltaMessage | None = None
580
+
581
+ if self._reasoning_parser and self._tool_parser:
582
+ if not state.reasoning_ended:
583
+ delta_message = self.extract_reasoning_streaming(
584
+ previous_text=state.previous_text,
585
+ current_text=current_text,
586
+ delta_text=delta_text,
587
+ previous_token_ids=state.previous_token_ids,
588
+ current_token_ids=current_token_ids,
589
+ delta_token_ids=delta_token_ids,
590
+ )
591
+ if self.is_reasoning_end(delta_token_ids):
592
+ state.reasoning_ended = True
593
+ current_token_ids = self.extract_content_ids(delta_token_ids)
594
+ if delta_message and delta_message.content:
595
+ current_text = delta_message.content
596
+ delta_message.content = None
597
+ else:
598
+ current_text = ""
599
+
600
+ if state.reasoning_ended:
601
+ if not state.tool_call_text_started:
602
+ state.tool_call_text_started = True
603
+ state.previous_text = ""
604
+ state.previous_token_ids = []
605
+ delta_text = current_text
606
+ delta_token_ids = current_token_ids
607
+
608
+ delta_message = self.extract_tool_calls_streaming(
609
+ previous_text=state.previous_text,
610
+ current_text=current_text,
611
+ delta_text=delta_text,
612
+ previous_token_ids=state.previous_token_ids,
613
+ current_token_ids=current_token_ids,
614
+ delta_token_ids=delta_token_ids,
615
+ request=request, # type: ignore[arg-type]
616
+ )
617
+
618
+ elif self._reasoning_parser:
619
+ delta_message = self.extract_reasoning_streaming(
620
+ previous_text=state.previous_text,
621
+ current_text=current_text,
622
+ delta_text=delta_text,
623
+ previous_token_ids=state.previous_token_ids,
624
+ current_token_ids=current_token_ids,
625
+ delta_token_ids=delta_token_ids,
626
+ )
627
+
628
+ elif self._tool_parser:
629
+ delta_message = self.extract_tool_calls_streaming(
630
+ previous_text=state.previous_text,
631
+ current_text=current_text,
632
+ delta_text=delta_text,
633
+ previous_token_ids=state.previous_token_ids,
634
+ current_token_ids=current_token_ids,
635
+ delta_token_ids=delta_token_ids,
636
+ request=request, # type: ignore[arg-type]
637
+ )
638
+
639
+ else:
640
+ delta_message = DeltaMessage(content=delta_text)
641
+
642
+ state.previous_text = current_text
643
+ state.previous_token_ids = current_token_ids
644
+ return delta_message
645
+
646
647
class _WrappedParser(DelegatingParser):
648
"""