@tduncan / Nvidia-Optimization-Stack-VLLM / commits / ac2ae8798c

[Frontend] Support count_reasoning_tokens in the Streaming Parser Engine (#45802)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Chauncey committed Aug 15, 2026 at 21:51 UTC ac2ae8798c7ef59c1924d5c0447a4279fa645770
16 files changed +669 -73
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+65
@@ -28,9 +28,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
28 from vllm.entrypoints.openai.chat_completion.serving import (
29 OpenAIServingChat,
30 _get_mm_token_counts,
31 + _make_completion_tokens_details,
32 _make_prompt_tokens_details,
33 )
34 from vllm.entrypoints.openai.engine.protocol import (
35 + DeltaMessage,
36 ErrorResponse,
37 RequestResponseMetadata,
38 )
@@ -626,6 +628,7 @@ def _build_minimal_metrics_serving_chat(
628 serving.response_role = "assistant"
629 serving.parser_cls = None
630 serving.enable_auto_tools = False
631 + serving._include_reasoning_tokens_details = False
632 serving.enable_prompt_tokens_details = False
633 serving.enable_log_outputs = False
634 serving.enable_log_deltas = False
@@ -666,6 +669,13 @@ async def _single_request_output(
669 yield request_output
670
671
672 +async def _stream_request_outputs(
673 + *request_outputs: RequestOutput,
674 +) -> AsyncIterator[RequestOutput]:
675 + for request_output in request_outputs:
676 + yield request_output
677 +
678 +
679 async def _collect_metrics_stream_chunks(
680 serving: OpenAIServingChat,
681 request: ChatCompletionRequest,
@@ -724,6 +734,7 @@ async def test_chat_per_request_metrics_follow_server_flag():
734 request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
735 )
736 assert disabled_response.metrics is None
737 + assert disabled_response.usage.completion_tokens_details is None
738
739 enabled_serving = _build_minimal_metrics_serving_chat(
740 enable_per_request_metrics=True
@@ -781,6 +792,55 @@ async def test_chat_streaming_metrics_ride_on_usage_chunk():
792 assert usage_chunks[-1]["metrics"]["time_to_first_token_ms"] == pytest.approx(500.0)
793
794
795 +@pytest.mark.asyncio
796 +async def test_streaming_reasoning_usage_counts_across_deltas():
797 + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
798 + serving._include_reasoning_tokens_details = True
799 + serving.model_config = None
800 +
801 + parser = MagicMock()
802 + parser.parse_delta.side_effect = [
803 + DeltaMessage(reasoning="reasoning"),
804 + DeltaMessage(content="answer"),
805 + ]
806 + parser.count_reasoning_tokens.side_effect = lambda token_ids: sum(
807 + token_id == 20 for token_id in token_ids
808 + )
809 + serving.parser_cls = MagicMock(return_value=parser)
810 +
811 + first = _make_metrics_request_output(metrics=None, token_ids=(10, 20))
812 + first.outputs[0].text = "<think>reasoning"
813 + first.outputs[0].finish_reason = None
814 + second = _make_metrics_request_output(metrics=None, token_ids=(11, 30))
815 + second.outputs[0].text = "</think>answer"
816 +
817 + request = ChatCompletionRequest(
818 + model="test-model",
819 + messages=[{"role": "user", "content": "Test prompt"}],
820 + max_tokens=10,
821 + stream=True,
822 + stream_options={"include_usage": True},
823 + )
824 + chunks: list[dict[str, Any]] = []
825 + async for line in serving.chat_completion_stream_generator(
826 + request,
827 + _stream_request_outputs(first, second),
828 + "chatcmpl-test-id",
829 + "test-model",
830 + conversation=[{"role": "user", "content": "Test"}],
831 + tokenizer=MagicMock(),
832 + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
833 + ):
834 + payload = line.removeprefix("data: ").strip()
835 + if payload != "[DONE]":
836 + chunks.append(json.loads(payload))
837 +
838 + usage_chunks = [chunk for chunk in chunks if chunk.get("usage")]
839 + assert usage_chunks[-1]["usage"]["completion_tokens_details"] == {
840 + "reasoning_tokens": 1
841 + }
842 +
843 +
844 @dataclass
845 class MockEngine:
846 model_config: MockModelConfig = field(default_factory=MockModelConfig)
@@ -810,6 +870,7 @@ async def _async_serving_chat_init():
870 def test_async_serving_chat_init():
871 serving_completion = asyncio.run(_async_serving_chat_init())
872 assert serving_completion.chat_template == CHAT_TEMPLATE
873 + assert serving_completion._include_reasoning_tokens_details is False
874
875
876 def test_mm_prompt_tokens_details():
@@ -850,6 +911,10 @@ def test_mm_prompt_tokens_details():
911 assert details.multimodal_tokens == {"image": 600, "video": 1200}
912
913
914 +def test_completion_tokens_details():
915 + assert _make_completion_tokens_details(7).reasoning_tokens == 7
916 +
917 +
918 @pytest.mark.asyncio
919 async def test_serving_chat_returns_correct_model_name():
920 mock_engine = MagicMock(spec=AsyncLLM)
tests/entrypoints/openai/responses/test_serving_responses.py
+4
@@ -360,6 +360,10 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
360 def get_vocab(self):
361 return self._vocab
362
363 + def decode(self, token_ids):
364 + id_to_token = {v: k for k, v in self._vocab.items()}
365 + return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
366 +
367 # Force non-harmony, SimpleContext path
368 monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
369
tests/parser/engine/test_engine.py
+51
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Tests for the streaming parser engine core pipeline."""
4
5 +from dataclasses import replace
6 from unittest.mock import MagicMock
7
8 import pytest
@@ -71,6 +72,16 @@ def _think_config() -> ParserEngineConfig:
72 )
73
74
75 +def _token_think_config() -> ParserEngineConfig:
76 + return replace(
77 + _think_config(),
78 + token_id_terminals={
79 + "THINK_START": "<think>",
80 + "THINK_END": "</think>",
81 + },
82 + )
83 +
84 +
85 class TestNonStreaming:
86 def test_plain_text(self):
87 engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
@@ -441,6 +452,46 @@ class TestTokenIdFiltering:
452 assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
453
454
455 +class TestReasoningTokenCounts:
456 + def test_counts_reasoning_and_excludes_boundaries_and_content(self):
457 + engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
458 +
459 + events = engine.feed(
460 + "<think>tok1tok2</think>tok3",
461 + [_START_ID, 1, 2, _END_ID, 3],
462 + )
463 + events.extend(engine.finish())
464 +
465 + reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
466 + assert "".join(e.value for e in reasoning) == "tok1tok2"
467 + assert sum(e.token_count for e in reasoning) == 2
468 + assert engine.reasoning_token_count == 2
469 +
470 + def test_counts_tokens_after_final_deferred_start_terminal(self):
471 + engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
472 +
473 + assert engine.feed("", [_START_ID, 1, 2]) == []
474 + events = engine.feed("<think>tok1tok2", [])
475 +
476 + reasoning = [e for e in events if e.type == EventType.REASONING_CHUNK]
477 + assert "".join(e.value for e in reasoning) == "tok1tok2"
478 + assert sum(e.token_count for e in reasoning) == 2
479 + assert engine.reasoning_token_count == 2
480 +
481 + def test_deferred_reasoning_tokens_stay_before_next_end_terminal(self):
482 + engine = StreamingParserEngine(_token_think_config(), _make_think_tokenizer())
483 +
484 + assert engine.feed("", [_START_ID, 1, 2]) == []
485 + events = engine.feed("<think>tok1tok2</think>tok3", [_END_ID, 3])
486 +
487 + assert (
488 + sum(e.token_count for e in events if e.type == EventType.REASONING_CHUNK)
489 + == 2
490 + )
491 + assert sum(e.token_count for e in events if e.type == EventType.TEXT_CHUNK) == 1
492 + assert engine.reasoning_token_count == 2
493 +
494 +
495 def _func_prefix_config() -> ParserEngineConfig:
496 """Config mixing token-ID terminals (TOOL_START/END) with
497 text-only terminals (FUNC_PREFIX) and fallback transitions."""
tests/parser/engine/test_parser_engine.py
+124
@@ -33,6 +33,7 @@ from vllm.parser.engine.parser_engine_config import (
33 ParserState,
34 Transition,
35 )
36 +from vllm.parser.parser_manager import ParserManager
37
38 # ── Shared test configs ──────────────────────────────────────────────
39
@@ -892,6 +893,129 @@ class _CombinedDelegating(DelegatingParser):
893 tool_parser_cls = _CombinedToolAdapter
894
895
896 +def test_parser_manager_uses_shared_engine_directly(monkeypatch):
897 + monkeypatch.setattr(
898 + ParserManager,
899 + "get_reasoning_parser",
900 + classmethod(lambda cls, name: _CombinedReasoningAdapter),
901 + )
902 + monkeypatch.setattr(
903 + ParserManager,
904 + "get_tool_parser",
905 + classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
906 + )
907 +
908 + parser_cls = ParserManager.get_parser(
909 + tool_parser_name="combined",
910 + reasoning_parser_name="combined",
911 + enable_auto_tools=True,
912 + )
913 +
914 + assert parser_cls is not None
915 + assert parser_cls is _CombinedTestEngine
916 + parser = parser_cls(make_mock_tokenizer(_VOCAB))
917 + request = _make_delegating_request()
918 + reasoning, content, _ = parser.parse(
919 + "ab</think>c",
920 + request,
921 + model_output_token_ids=[ord("a"), ord("b"), 201, ord("c")],
922 + )
923 + assert reasoning == "ab"
924 + assert content == "c"
925 + assert parser.count_reasoning_tokens([]) == 2
926 +
927 +
928 +def test_parser_manager_preserves_reasoning_only_adapter(monkeypatch):
929 + monkeypatch.setattr(
930 + ParserManager,
931 + "get_reasoning_parser",
932 + classmethod(lambda cls, name: _CombinedReasoningAdapter),
933 + )
934 + monkeypatch.setattr(
935 + ParserManager,
936 + "get_tool_parser",
937 + classmethod(lambda cls, name, enabled, model: None),
938 + )
939 +
940 + parser_cls = ParserManager.get_parser(reasoning_parser_name="combined")
941 +
942 + assert parser_cls is not None
943 + parser = parser_cls(make_mock_tokenizer(_VOCAB))
944 + assert parser.reasoning_parser is not None
945 + assert parser.tool_parser is None
946 + reasoning, content, _ = parser.parse(
947 + 'ab</think><tool_call>{"name":"h","arguments":{}}</tool_call>',
948 + _make_delegating_request(),
949 + model_output_token_ids=[ord("a"), ord("b"), 201],
950 + )
951 + assert reasoning == "ab"
952 + assert content == '<tool_call>{"name":"h","arguments":{}}</tool_call>'
953 + assert parser.count_reasoning_tokens([ord("a"), ord("b"), 201]) == 2
954 +
955 +
956 +def test_parser_manager_preserves_tool_only_adapter(monkeypatch):
957 + monkeypatch.setattr(
958 + ParserManager,
959 + "get_reasoning_parser",
960 + classmethod(lambda cls, name: None),
961 + )
962 + monkeypatch.setattr(
963 + ParserManager,
964 + "get_tool_parser",
965 + classmethod(lambda cls, name, enabled, model: _CombinedToolAdapter),
966 + )
967 +
968 + parser_cls = ParserManager.get_parser(
969 + tool_parser_name="combined", enable_auto_tools=True
970 + )
971 +
972 + assert parser_cls is not None
973 + parser = parser_cls(make_mock_tokenizer(_VOCAB))
974 + assert parser.reasoning_parser is None
975 + assert parser.tool_parser is not None
976 + reasoning, content, tool_calls = parser.parse(
977 + "<think>ab</think>c",
978 + _make_delegating_request(),
979 + model_output_token_ids=[200, ord("a"), ord("b"), 201, ord("c")],
980 + )
981 + assert reasoning is None
982 + assert content == "<think>ab</think>c"
983 + assert tool_calls == []
984 +
985 +
986 +def test_parser_manager_rejects_non_engine_adapter_metadata():
987 + class TraditionalParser:
988 + pass
989 +
990 + class InvalidEngineAdapter:
991 + _parser_engine_cls = TraditionalParser
992 +
993 + assert ParserManager._get_parser_engine_cls(TraditionalParser) is None
994 + assert ParserManager._get_parser_engine_cls(InvalidEngineAdapter) is None
995 + assert (
996 + ParserManager._get_parser_engine_cls(_CombinedReasoningAdapter)
997 + is _CombinedTestEngine
998 + )
999 +
1000 +
1001 +def test_reasoning_adapter_counts_after_final_non_streaming_parse():
1002 + parser = _CombinedReasoningAdapter(make_mock_tokenizer(_VOCAB))
1003 + request = _make_delegating_request()
1004 + token_ids = [ord("a"), ord("b"), 201, ord("c")]
1005 +
1006 + parser.extract_reasoning_streaming(
1007 + "",
1008 + "ab</think>c",
1009 + "ab</think>c",
1010 + [],
1011 + token_ids,
1012 + token_ids,
1013 + )
1014 + parser.extract_reasoning("ab</think>c", request)
1015 +
1016 + assert parser.count_reasoning_tokens(token_ids) == 2
1017 +
1018 +
1019 def _make_delegating_request():
1020 req = MagicMock(spec=ChatCompletionRequest)
1021 req.tools = []
tests/parser/engine/test_token_id_scanner.py
+20 -1
@@ -163,9 +163,28 @@ class TestHoldbackTextRecovery:
163 assert len(result) == 0
164
165 flushed = scanner.flush_pending()
166 - assert len(flushed) == 1
166 + assert len(flushed) == 2
167 assert isinstance(flushed[0], PreLexedTerminal)
168 assert flushed[0].terminal == "TOOL_START"
169 + assert isinstance(flushed[1], TextChunk)
170 + assert flushed[1].text == ""
171 + assert flushed[1].token_count == 2
172 +
173 + def test_deferred_terminal_preserves_trailing_token_count(self, tokenizer):
174 + tokenizer.decode.side_effect = lambda ids: {
175 + CHANNEL_START_ID: CHANNEL_START,
176 + 201: "alpha",
177 + 202: "beta",
178 + }[ids[0]]
179 + scanner = TokenIDScanner({CHANNEL_START_ID: "THINK_START"}, tokenizer)
180 +
181 + assert scanner.scan("", [CHANNEL_START_ID, 201, 202]) == []
182 +
183 + result = scanner.scan(f"{CHANNEL_START}alphabeta", [])
184 + assert isinstance(result[0], PreLexedTerminal)
185 + assert isinstance(result[1], TextChunk)
186 + assert result[1].text == "alphabeta"
187 + assert result[1].token_count == 2
188
189 def test_holdback_before_start_tag(self, scanner):
190 result = scanner.scan(
vllm/entrypoints/openai/chat_completion/serving.py
+50
@@ -36,6 +36,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
36 ChatMessage,
37 )
38 from vllm.entrypoints.openai.engine.protocol import (
39 + CompletionTokenUsageInfo,
40 DeltaMessage,
41 ErrorResponse,
42 FunctionCall,
@@ -105,6 +106,12 @@ def _make_prompt_tokens_details(
106 )
107
108
109 +def _make_completion_tokens_details(
110 + reasoning_tokens: int,
111 +) -> CompletionTokenUsageInfo:
112 + return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens)
113 +
114 +
115 class OpenAIServingChat(GenerateBaseServing):
116 def __init__(
117 self,
@@ -146,6 +153,7 @@ class OpenAIServingChat(GenerateBaseServing):
153 self.enable_log_deltas = enable_log_deltas
154
155 self.enable_auto_tools: bool = enable_auto_tools
156 + self._include_reasoning_tokens_details = bool(reasoning_parser)
157 self.parser_cls = ParserManager.get_parser(
158 tool_parser_name=tool_parser,
159 reasoning_parser_name=reasoning_parser,
@@ -439,6 +447,9 @@ class OpenAIServingChat(GenerateBaseServing):
447 # Send response for each token for each request.n (index)
448 num_choices = 1 if request.n is None else request.n
449 previous_num_tokens = [0] * num_choices
450 + # TODO: Remove once all reasoning parsers use the Parser Engine.
451 + generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)]
452 + previous_reasoning_tokens = [0] * num_choices
453 finish_reason_sent = [False] * num_choices
454 num_prompt_tokens = 0
455 num_cached_tokens = None
@@ -537,6 +548,11 @@ class OpenAIServingChat(GenerateBaseServing):
548 prompt_tokens=num_prompt_tokens,
549 completion_tokens=0,
550 total_tokens=num_prompt_tokens,
551 + completion_tokens_details=(
552 + _make_completion_tokens_details(0)
553 + if self._include_reasoning_tokens_details
554 + else None
555 + ),
556 )
557
558 data = chunk.model_dump_json(exclude_unset=True)
@@ -573,6 +589,11 @@ class OpenAIServingChat(GenerateBaseServing):
589 prompt_tokens=num_prompt_tokens,
590 completion_tokens=0,
591 total_tokens=num_prompt_tokens,
592 + completion_tokens_details=(
593 + _make_completion_tokens_details(0)
594 + if self._include_reasoning_tokens_details
595 + else None
596 + ),
597 )
598
599 data = chunk.model_dump_json(exclude_unset=True)
@@ -631,6 +652,11 @@ class OpenAIServingChat(GenerateBaseServing):
652
653 # set the previous values for the next iteration
654 previous_num_tokens[i] += len(output.token_ids)
655 + if parser is not None:
656 + generated_token_ids[i].extend(output.token_ids)
657 + previous_reasoning_tokens[i] = parser.count_reasoning_tokens(
658 + tuple(generated_token_ids[i])
659 + )
660
661 # if the message delta is None (e.g. because it was a
662 # "control token" for tool calls or the parser otherwise
@@ -756,6 +782,13 @@ class OpenAIServingChat(GenerateBaseServing):
782 prompt_tokens=num_prompt_tokens,
783 completion_tokens=completion_tokens,
784 total_tokens=num_prompt_tokens + completion_tokens,
785 + completion_tokens_details=(
786 + _make_completion_tokens_details(
787 + previous_reasoning_tokens[i]
788 + )
789 + if self._include_reasoning_tokens_details
790 + else None
791 + ),
792 )
793
794 data = chunk.model_dump_json(exclude_unset=True)
@@ -769,6 +802,11 @@ class OpenAIServingChat(GenerateBaseServing):
802 prompt_tokens=num_prompt_tokens,
803 completion_tokens=completion_tokens,
804 total_tokens=num_prompt_tokens + completion_tokens,
805 + completion_tokens_details=_make_completion_tokens_details(
806 + sum(previous_reasoning_tokens)
807 + )
808 + if self._include_reasoning_tokens_details
809 + else None,
810 )
811 final_usage.prompt_tokens_details = _make_prompt_tokens_details(
812 self.enable_prompt_tokens_details,
@@ -813,6 +851,11 @@ class OpenAIServingChat(GenerateBaseServing):
851 prompt_tokens=num_prompt_tokens,
852 completion_tokens=num_completion_tokens,
853 total_tokens=num_prompt_tokens + num_completion_tokens,
854 + completion_tokens_details=_make_completion_tokens_details(
855 + sum(previous_reasoning_tokens)
856 + )
857 + if self._include_reasoning_tokens_details
858 + else None,
859 )
860
861 # Log complete streaming response if output logging is enabled
@@ -871,6 +914,7 @@ class OpenAIServingChat(GenerateBaseServing):
914 )
915
916 choices: list[ChatCompletionResponseChoice] = []
917 + total_reasoning_tokens = 0
918
919 role = self.get_chat_request_role(request)
920 tool_parser_cls = (
@@ -908,6 +952,7 @@ class OpenAIServingChat(GenerateBaseServing):
952 suppress_metadata = not request.include_reasoning and parser is not None
953 if not request.include_reasoning:
954 reasoning = None
955 + total_reasoning_tokens += parser.count_reasoning_tokens(token_ids)
956 if suppress_metadata:
957 logprobs = None
958 else:
@@ -1058,6 +1103,11 @@ class OpenAIServingChat(GenerateBaseServing):
1103 prompt_tokens=num_prompt_tokens,
1104 completion_tokens=num_generated_tokens,
1105 total_tokens=num_prompt_tokens + num_generated_tokens,
1106 + completion_tokens_details=_make_completion_tokens_details(
1107 + total_reasoning_tokens
1108 + )
1109 + if self._include_reasoning_tokens_details
1110 + else None,
1111 )
1112 usage.prompt_tokens_details = _make_prompt_tokens_details(
1113 self.enable_prompt_tokens_details,
vllm/entrypoints/openai/engine/protocol.py
+5
@@ -117,11 +117,16 @@ class PromptTokenUsageInfo(OpenAIBaseModel):
117 request has no multimodal input."""
118
119
120 +class CompletionTokenUsageInfo(OpenAIBaseModel):
121 + reasoning_tokens: int = 0
122 +
123 +
124 class UsageInfo(OpenAIBaseModel):
125 prompt_tokens: int = 0
126 total_tokens: int = 0
127 completion_tokens: int | None = 0
128 prompt_tokens_details: PromptTokenUsageInfo | None = None
129 + completion_tokens_details: CompletionTokenUsageInfo | None = None
130
131
132 class PerRequestTimingMetrics(OpenAIBaseModel):
vllm/entrypoints/openai/responses/serving.py
+6 -8
@@ -487,6 +487,7 @@ class OpenAIServingResponses(GenerateBaseServing):
487 response_parser=response_parser,
488 )
489
490 + reasoning_parser_kwargs = None
491 if (
492 context.response_parser is not None
493 and context.response_parser.reasoning_parser is not None
@@ -518,9 +519,7 @@ class OpenAIServingResponses(GenerateBaseServing):
519 priority=self._get_priority(request, raw_request),
520 trace_headers=trace_headers,
521 session_id=session_id,
521 - reasoning_parser_kwargs=reasoning_parser_kwargs
522 - if self.parser and self.parser.reasoning_parser_cls is not None
523 - else None,
522 + reasoning_parser_kwargs=reasoning_parser_kwargs,
523 )
524 generators.append(generator)
525
@@ -873,6 +872,8 @@ class OpenAIServingResponses(GenerateBaseServing):
872 if final_output.finish_reason == "length":
873 status = "incomplete"
874
875 + # TODO: Build final response items from the accumulated streaming
876 + # parser results instead of reparsing the complete output.
877 output = self._make_response_output_items(
878 request,
879 final_output,
@@ -901,13 +902,10 @@ class OpenAIServingResponses(GenerateBaseServing):
902 num_reasoning_tokens == 0
903 and isinstance(context, (SimpleContext, ParsableContext))
904 and context.response_parser is not None
904 - and context.response_parser.reasoning_parser is not None
905 ):
906 accumulated = getattr(context, "_accumulated_token_ids", []) or []
907 - num_reasoning_tokens = (
908 - context.response_parser.reasoning_parser.count_reasoning_tokens(
909 - accumulated
910 - )
907 + num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
908 + accumulated
909 )
910
911 usage = ResponseUsage(
vllm/parser/abstract_parser.py
+10
@@ -367,6 +367,10 @@ class Parser:
367 tool call extraction via internal stream state.
368 """
369
370 + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
371 + """Return the number of reasoning tokens in generated token IDs."""
372 + return 0
373 +
374
375 class DelegatingParser(Parser):
376 """
@@ -939,6 +943,12 @@ class DelegatingParser(Parser):
943
944 return delta_message
945
946 + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
947 + """Count reasoning tokens through the configured reasoning parser."""
948 + if self._reasoning_parser is None:
949 + return 0
950 + return self._reasoning_parser.count_reasoning_tokens(token_ids)
951 +
952 def _flush_engine_parsers(
953 self, delta_message: DeltaMessage | None
954 ) -> DeltaMessage | None:
vllm/parser/engine/adapters.py
+19 -1
@@ -47,6 +47,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
47 def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None:
48 super().__init__(tokenizer, *args, **kwargs)
49 self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg]
50 + self._parser_engine_kwargs = kwargs
51 + self._counting_parser_engine: ParserEngine | None = None
52 + # TODO: Remove once Responses finalization reuses accumulated streaming
53 + # parser results instead of reparsing the complete output.
54 + self._streaming_count_valid = False
55
56 @contextmanager
57 def _skip_tool_parsing(self) -> Iterator[None]:
@@ -71,6 +76,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
76 model_output: str,
77 request: ChatCompletionRequest | ResponsesRequest,
78 ) -> tuple[str | None, str | None]:
79 + self._streaming_count_valid = False
80 with self._skip_tool_parsing():
81 return self._parser_engine.extract_reasoning(model_output, request)
82
@@ -83,6 +89,7 @@ class ParserEngineReasoningAdapter(ReasoningParser):
89 current_token_ids: Sequence[int],
90 delta_token_ids: Sequence[int],
91 ) -> DeltaMessage | None:
92 + self._streaming_count_valid = True
93 with self._skip_tool_parsing():
94 return self._parser_engine.extract_reasoning_streaming(
95 previous_text,
@@ -122,7 +129,18 @@ class ParserEngineReasoningAdapter(ReasoningParser):
129 return self._parser_engine.get_streaming_fallback_content(text, request)
130
131 def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
125 - return self._parser_engine.count_reasoning_tokens(token_ids)
132 + if self._streaming_count_valid:
133 + return self._parser_engine.count_reasoning_tokens(token_ids)
134 + if not token_ids:
135 + return 0
136 + if self._counting_parser_engine is None:
137 + self._counting_parser_engine = self._parser_engine_cls(
138 + self.model_tokenizer, **self._parser_engine_kwargs
139 + ) # type: ignore[call-arg]
140 + self._counting_parser_engine._single_pass_parse(
141 + self.model_tokenizer.decode(token_ids), token_ids
142 + )
143 + return self._counting_parser_engine.count_reasoning_tokens(token_ids)
144
145
146 class ParserEngineToolAdapter(ToolParser):
vllm/parser/engine/events.py
+1
@@ -24,3 +24,4 @@ class SemanticEvent:
24 type: EventType
25 value: str = ""
26 tool_index: int = -1
27 + token_count: int = 0
vllm/parser/engine/incremental_lexer.py
+81 -8
@@ -24,6 +24,7 @@ class TerminalDef:
24 class LexToken:
25 terminal: str
26 value: str
27 + token_count: int = 0
28
29
30 class LexerShape:
@@ -104,6 +105,7 @@ class IncrementalLexer:
105 self.terminals = shape.terminals
106 self.content_terminal = content_terminal
107 self.buffer = ""
108 + self._token_counts: list[int] = []
109
110 self._literal_strings = shape.literal_strings
111 self._max_literal_len = shape.max_literal_len
@@ -114,15 +116,23 @@ class IncrementalLexer:
116
117 def reset(self) -> None:
118 self.buffer = ""
119 + self._token_counts.clear()
120
118 - def feed(self, text: str) -> list[LexToken]:
121 + def feed(
122 + self,
123 + text: str,
124 + token_texts: tuple[str, ...] = (),
125 + token_count: int = 0,
126 + ) -> list[LexToken]:
127 + char_token_counts = self._char_token_counts(text, token_texts, token_count)
128 if not self.buffer and self._has_only_literals and self._literal_first_chars:
129 for ch in text:
130 if ch in self._literal_first_chars:
131 break
132 else:
124 - return [LexToken(self.content_terminal, text)]
133 + return [LexToken(self.content_terminal, text, sum(char_token_counts))]
134 self.buffer += text
135 + self._token_counts.extend(char_token_counts)
136 return self._drain()
137
138 def flush(self) -> list[LexToken]:
@@ -130,10 +140,46 @@ class IncrementalLexer:
140 if self.buffer:
141 tokens.extend(self._drain(final=True))
142 if self.buffer:
133 - tokens.append(LexToken(self.content_terminal, self.buffer))
143 + tokens.append(
144 + LexToken(self.content_terminal, self.buffer, sum(self._token_counts))
145 + )
146 self.buffer = ""
147 + self._token_counts.clear()
148 return tokens
149
150 + @staticmethod
151 + def _char_token_counts(
152 + text: str,
153 + token_texts: tuple[str, ...],
154 + token_count: int,
155 + ) -> list[int]:
156 + counts = [0] * len(text)
157 + if not text:
158 + return counts
159 + if token_texts:
160 + pos = 0
161 + assigned = 0
162 + for token_text in token_texts:
163 + if not token_text:
164 + continue
165 + found = text.find(token_text, pos)
166 + if found < 0:
167 + continue
168 + counts[found] += 1
169 + assigned += 1
170 + pos = found + len(token_text)
171 + missing = token_count - assigned
172 + if missing > 0:
173 + counts[0] += missing
174 + elif token_count:
175 + counts[0] = token_count
176 + return counts
177 +
178 + def _pop_token_count(self, length: int) -> int:
179 + token_count = sum(self._token_counts[:length])
180 + del self._token_counts[:length]
181 + return token_count
182 +
183 def _drain(self, *, final: bool = False) -> list[LexToken]:
184 tokens: list[LexToken] = []
185 first_chars = self._literal_first_chars
@@ -150,8 +196,11 @@ class IncrementalLexer:
196 has_potential = True
197 break
198 if not has_potential:
153 - tokens.append(LexToken(content_terminal, self.buffer))
199 + tokens.append(
200 + LexToken(content_terminal, self.buffer, sum(self._token_counts))
201 + )
202 self.buffer = ""
203 + self._token_counts.clear()
204 break
205
206 best_match: tuple[str, str, int] | None = None
@@ -175,7 +224,13 @@ class IncrementalLexer:
224 longer_match = True
225 break
226 if not longer_match:
178 - tokens.append(LexToken(best_match[0], best_match[1]))
227 + tokens.append(
228 + LexToken(
229 + best_match[0],
230 + best_match[1],
231 + self._pop_token_count(best_match[2]),
232 + )
233 + )
234 self.buffer = self.buffer[best_match[2] :]
235 continue
236 break
@@ -183,15 +238,33 @@ class IncrementalLexer:
238 break
239
240 if best_match is not None:
186 - tokens.append(LexToken(best_match[0], best_match[1]))
241 + tokens.append(
242 + LexToken(
243 + best_match[0],
244 + best_match[1],
245 + self._pop_token_count(best_match[2]),
246 + )
247 + )
248 self.buffer = self.buffer[best_match[2] :]
249 else:
250 content_end = self._find_content_boundary()
251 if content_end > 0:
191 - tokens.append(LexToken(content_terminal, self.buffer[:content_end]))
252 + tokens.append(
253 + LexToken(
254 + content_terminal,
255 + self.buffer[:content_end],
256 + self._pop_token_count(content_end),
257 + )
258 + )
259 self.buffer = self.buffer[content_end:]
260 else:
194 - tokens.append(LexToken(content_terminal, self.buffer[0]))
261 + tokens.append(
262 + LexToken(
263 + content_terminal,
264 + self.buffer[0],
265 + self._pop_token_count(1),
266 + )
267 + )
268 self.buffer = self.buffer[1:]
269
270 return tokens
vllm/parser/engine/parser_engine.py
+2 -17
@@ -625,23 +625,8 @@ class ParserEngine(Parser):
625 return None
626
627 def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
628 - start_id = self._reasoning_start_token_id
629 - end_id = self._reasoning_end_token_id
630 - if start_id is None or end_id is None:
631 - return 0
632 - count = 0
633 - depth = 0
634 - for token_id in token_ids:
635 - if token_id == start_id:
636 - depth += 1
637 - continue
638 - if token_id == end_id:
639 - if depth > 0:
640 - depth -= 1
641 - continue
642 - if depth > 0:
643 - count += 1
644 - return count
628 + """Return reasoning tokens observed by the parser engine so far."""
629 + return self._engine.reasoning_token_count
630
631 # ── Single-pass parse helper ────────────────────────────────────────
632
vllm/parser/engine/streaming_parser_engine.py
+80 -17
@@ -167,6 +167,17 @@ class StreamingParserEngine:
167 self.skip_tool_parsing = False
168 self.reset(initial_state=initial_state)
169
170 + @property
171 + def reasoning_token_count(self) -> int:
172 + return self._reasoning_token_count
173 +
174 + def _record_reasoning_tokens(self, events: Sequence[SemanticEvent]) -> None:
175 + self._reasoning_token_count += sum(
176 + event.token_count
177 + for event in events
178 + if event.type == EventType.REASONING_CHUNK
179 + )
180 +
181 def _reset_args_state(self) -> None:
182 self._args_buffer: str = ""
183 self._args_safe_end: int = 0
@@ -186,6 +197,7 @@ class StreamingParserEngine:
197 )
198 self.tool_index = -1
199 self._ever_had_token_ids = False
200 + self._reasoning_token_count = 0
201 # DO NOT reset skip_tool_parsing here — callers set it before
202 # calling methods that trigger reset() (e.g. extract_reasoning),
203 # and clearing it silently breaks non-streaming tool-call-as-
@@ -193,6 +205,7 @@ class StreamingParserEngine:
205 self._scanner.reset()
206 self._lexer.reset()
207 self._message_header_buffer = ""
208 + self._message_header_token_count = 0
209 self._in_skipped_tool_span = False
210 self._reset_args_state()
211
@@ -218,18 +231,30 @@ class StreamingParserEngine:
231 has_special = True
232 break
233 if not has_special:
221 - return self._emit_for_state(delta_text)
234 + events = self._emit_for_state(
235 + delta_text, token_count=len(delta_token_ids)
236 + )
237 + self._record_reasoning_tokens(events)
238 + return events
239
240 scanner_items = self._scanner.scan(delta_text, delta_token_ids)
241
242 if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk):
226 - lex_tokens = self._lexer.feed(scanner_items[0].text)
243 + item = scanner_items[0]
244 + lex_tokens = self._lexer.feed(item.text, item.token_texts, item.token_count)
245 if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL:
228 - text = lex_tokens[0].value
229 - return self._emit_for_state(text)
230 - return self._process_lex_tokens(lex_tokens)
246 + events = self._emit_for_state(
247 + lex_tokens[0].value,
248 + token_count=lex_tokens[0].token_count,
249 + )
250 + else:
251 + events = self._process_lex_tokens(lex_tokens)
252 + self._record_reasoning_tokens(events)
253 + return events
254
232 - return self._process_scanner_items(scanner_items)
255 + events = self._process_scanner_items(scanner_items)
256 + self._record_reasoning_tokens(events)
257 + return events
258
259 def _process_scanner_items(
260 self, items: Sequence[LexerInput]
@@ -240,7 +265,18 @@ class StreamingParserEngine:
265 events.extend(self._process_lex_tokens(self._lexer.flush()))
266 events.extend(self._on_terminal(item.terminal, item.text))
267 elif isinstance(item, TextChunk):
243 - events.extend(self._process_lex_tokens(self._lexer.feed(item.text)))
268 + if not item.text and item.token_count:
269 + events.extend(
270 + self._emit_for_state("", token_count=item.token_count)
271 + )
272 + else:
273 + events.extend(
274 + self._process_lex_tokens(
275 + self._lexer.feed(
276 + item.text, item.token_texts, item.token_count
277 + )
278 + )
279 + )
280 return events
281
282 def finish(self) -> list[SemanticEvent]:
@@ -285,11 +321,14 @@ class StreamingParserEngine:
321 EventType.TEXT_CHUNK,
322 value=self._message_header_buffer,
323 tool_index=self.tool_index,
324 + token_count=self._message_header_token_count,
325 )
326 )
327 self._message_header_buffer = ""
328 + self._message_header_token_count = 0
329 self.state = ParserState.CONTENT
330
331 + self._record_reasoning_tokens(events)
332 return events
333
334 def parse_complete(self, text: str) -> list[SemanticEvent]:
@@ -303,9 +342,11 @@ class StreamingParserEngine:
342 strict = self._token_id_terminal_names if self._ever_had_token_ids else None
343 for tok in tokens:
344 if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict):
306 - events.extend(self._on_content(tok.value))
345 + events.extend(self._on_content(tok.value, tok.token_count))
346 else:
308 - events.extend(self._on_terminal(tok.terminal, tok.value))
347 + events.extend(
348 + self._on_terminal(tok.terminal, tok.value, tok.token_count)
349 + )
350 return events
351
352 _TOOL_STATES = frozenset(
@@ -317,7 +358,9 @@ class StreamingParserEngine:
358 }
359 )
360
320 - def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]:
361 + def _on_terminal(
362 + self, terminal: str, value: str, token_count: int = 0
363 + ) -> list[SemanticEvent]:
364 key = (self.state, terminal)
365 transition = self.config.transitions.get(key)
366
@@ -327,7 +370,7 @@ class StreamingParserEngine:
370 # The projected skip state may not define the wrapper closer.
371 if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
372 self._in_skipped_tool_span = False
330 - return self._emit_for_state(value)
373 + return self._emit_for_state(value, token_count)
374
375 if self.skip_tool_parsing and terminal in self._tool_terminals:
376 # Inkling reuses one terminal for tool, text, and reasoning exits.
@@ -345,6 +388,7 @@ class StreamingParserEngine:
388 leaving_message_header = self.state == ParserState.MESSAGE_HEADER
389 if leaving_message_header:
390 self._message_header_buffer = ""
391 + self._message_header_token_count = 0
392 # A tool terminal that implicitly ends reasoning must report
393 # that even from the header state, or the reasoning pass never
394 # hands the block to the tool pass.
@@ -381,13 +425,14 @@ class StreamingParserEngine:
425 return []
426
427 if transition.skip_in_token_id_mode and self._ever_had_token_ids:
384 - return self._emit_for_state(value)
428 + return self._emit_for_state(value, token_count)
429
386 - return self._apply_transition(transition, value)
430 + return self._apply_transition(transition, value, token_count)
431
388 - def _emit_for_state(self, text: str) -> list[SemanticEvent]:
432 + def _emit_for_state(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
433 if self.state == ParserState.MESSAGE_HEADER:
434 self._message_header_buffer += text
435 + self._message_header_token_count += token_count
436 return []
437 if self.state == ParserState.TOOL_ARGS:
438 if self.config.tool_args_json:
@@ -397,26 +442,36 @@ class StreamingParserEngine:
442 EventType.ARG_VALUE_CHUNK,
443 value=text,
444 tool_index=self.tool_index,
445 + token_count=token_count,
446 )
447 ]
448 content_type = self.config.content_events.get(self.state)
449 if content_type is not None:
404 - return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)]
450 + return [
451 + SemanticEvent(
452 + content_type,
453 + value=text,
454 + tool_index=self.tool_index,
455 + token_count=token_count,
456 + )
457 + ]
458 return []
459
407 - def _on_content(self, text: str) -> list[SemanticEvent]:
460 + def _on_content(self, text: str, token_count: int = 0) -> list[SemanticEvent]:
461 if not text:
462 return []
410 - return self._emit_for_state(text)
463 + return self._emit_for_state(text, token_count)
464
465 def _apply_transition(
466 self,
467 transition: Transition,
468 value: str,
469 + token_count: int = 0,
470 ) -> list[SemanticEvent]:
471 events: list[SemanticEvent] = []
472 previous_state = self.state
473 message_header = ""
474 + message_header_token_count = 0
475
476 if (
477 self.state == ParserState.TOOL_ARGS
@@ -434,7 +489,9 @@ class StreamingParserEngine:
489
490 if previous_state == ParserState.MESSAGE_HEADER:
491 message_header = self._message_header_buffer
492 + message_header_token_count = self._message_header_token_count
493 self._message_header_buffer = ""
494 + self._message_header_token_count = 0
495
496 self.state = transition.next_state
497
@@ -454,6 +511,12 @@ class StreamingParserEngine:
511 event_type,
512 value=event_value,
513 tool_index=self.tool_index,
514 + token_count=(
515 + message_header_token_count
516 + if previous_state == ParserState.MESSAGE_HEADER
517 + and event_type == EventType.TEXT_CHUNK
518 + else token_count
519 + ),
520 )
521 )
522
vllm/parser/engine/token_id_scanner.py
+121 -17
@@ -14,6 +14,8 @@ DROP_TERMINAL = "__DROP__"
14 @dataclass(slots=True)
15 class TextChunk:
16 text: str
17 + token_texts: tuple[str, ...] = ()
18 + token_count: int = 0
19
20
21 @dataclass(slots=True)
@@ -49,18 +51,30 @@ class TokenIDScanner:
51 self.tokenizer = tokenizer
52 self._token_text_cache: dict[int, str] = {}
53 self._deferred_terminals: list[PreLexedTerminal] = []
54 + self._deferred_prefix_token_counts: list[int] = []
55 + self._deferred_trailing_token_count = 0
56 self._deferred_post_text: str = ""
57
58 def reset(self) -> None:
59 """Clear mutable state for reuse. Preserves the token text cache."""
60 self._deferred_terminals.clear()
61 + self._deferred_prefix_token_counts.clear()
62 + self._deferred_trailing_token_count = 0
63 self._deferred_post_text = ""
64
65 def _decode_token(self, token_id: int) -> str:
66 if token_id not in self._token_text_cache:
61 - self._token_text_cache[token_id] = self.tokenizer.decode([token_id])
67 + token_text = self.tokenizer.decode([token_id])
68 + self._token_text_cache[token_id] = (
69 + token_text if isinstance(token_text, str) else ""
70 + )
71 return self._token_text_cache[token_id]
72
73 + def _decode_tokens(self, token_ids: Sequence[int]) -> tuple[str, ...]:
74 + if self.tokenizer is None:
75 + return ()
76 + return tuple(self._decode_token(tid) for tid in token_ids)
77 +
78 _EMPTY: tuple[LexerInput, ...] = ()
79
80 def scan(
@@ -70,13 +84,23 @@ class TokenIDScanner:
84 ) -> Sequence[LexerInput]:
85 prefix_items: list[LexerInput] = []
86 effective_text = delta_text
87 + deferred_trailing_count = 0
88
89 if self._deferred_terminals:
75 - prefix_items, effective_text = self._resolve_deferred(delta_text)
90 + prefix_items, effective_text, deferred_trailing_count = (
91 + self._resolve_deferred(delta_text, len(delta_token_ids))
92 + )
93
94 if not self.token_id_to_terminal:
95 if effective_text:
79 - prefix_items.append(TextChunk(effective_text))
96 + token_texts = self._decode_tokens(delta_token_ids)
97 + prefix_items.append(
98 + TextChunk(
99 + effective_text,
100 + token_texts,
101 + deferred_trailing_count + len(token_texts),
102 + )
103 + )
104 return prefix_items
105
106 has_special = False
@@ -89,14 +113,23 @@ class TokenIDScanner:
113 if not has_special:
114 if effective_text:
115 if not prefix_items:
92 - return [TextChunk(effective_text)]
93 - prefix_items.append(TextChunk(effective_text))
116 + token_texts = self._decode_tokens(delta_token_ids)
117 + return [TextChunk(effective_text, token_texts, len(token_texts))]
118 + token_texts = self._decode_tokens(delta_token_ids)
119 + prefix_items.append(
120 + TextChunk(
121 + effective_text,
122 + token_texts,
123 + deferred_trailing_count + len(token_texts),
124 + )
125 + )
126 return prefix_items or self._EMPTY
127
96 - token_texts = [self._decode_token(tid) for tid in delta_token_ids]
128 + decoded_token_texts = [self._decode_token(tid) for tid in delta_token_ids]
129
130 results: list[LexerInput] = []
131 text_accum: list[str] = []
132 + token_text_accum: list[str] = []
133
134 for idx, tid in enumerate(delta_token_ids):
135 terminal = self.token_id_to_terminal.get(tid)
@@ -104,16 +137,31 @@ class TokenIDScanner:
137 if text_accum:
138 joined = "".join(text_accum)
139 if joined:
107 - results.append(TextChunk(joined))
140 + results.append(
141 + TextChunk(
142 + joined,
143 + tuple(token_text_accum),
144 + len(token_text_accum),
145 + )
146 + )
147 text_accum.clear()
109 - results.append(PreLexedTerminal(terminal, tid, token_texts[idx]))
148 + token_text_accum.clear()
149 + results.append(
150 + PreLexedTerminal(terminal, tid, decoded_token_texts[idx])
151 + )
152 else:
111 - text_accum.append(token_texts[idx])
153 + text_accum.append(decoded_token_texts[idx])
154 + token_text_accum.append(decoded_token_texts[idx])
155
156 if text_accum:
157 joined = "".join(text_accum)
158 if joined:
116 - results.append(TextChunk(joined))
159 + results.append(
160 + TextChunk(joined, tuple(token_text_accum), len(token_text_accum))
161 + )
162 +
163 + if deferred_trailing_count:
164 + results.insert(0, TextChunk("", token_count=deferred_trailing_count))
165
166 if effective_text:
167 results = self._recover_holdback_text(effective_text, results)
@@ -124,9 +172,15 @@ class TokenIDScanner:
172 # transition before the preceding text has arrived. The
173 # deferred terminals will be resolved against the actual
174 # delta_text in a subsequent scan() or flushed by finish().
175 + prefix_token_count = 0
176 for r in results:
177 + if isinstance(r, TextChunk):
178 + prefix_token_count += r.token_count
179 if isinstance(r, PreLexedTerminal):
180 self._deferred_terminals.append(r)
181 + self._deferred_prefix_token_counts.append(prefix_token_count)
182 + prefix_token_count = 0
183 + self._deferred_trailing_token_count += prefix_token_count
184 results = []
185
186 return prefix_items + results
@@ -136,16 +190,30 @@ class TokenIDScanner:
190 return []
191 results: list[LexerInput] = []
192 if self._deferred_post_text:
139 - results.append(TextChunk(self._deferred_post_text))
193 + prefix_count = (
194 + self._deferred_prefix_token_counts[0]
195 + if self._deferred_prefix_token_counts
196 + else 0
197 + )
198 + results.append(
199 + TextChunk(self._deferred_post_text, token_count=prefix_count)
200 + )
201 self._deferred_post_text = ""
202 results.extend(self._deferred_terminals)
203 + if self._deferred_trailing_token_count:
204 + results.append(
205 + TextChunk("", token_count=self._deferred_trailing_token_count)
206 + )
207 self._deferred_terminals.clear()
208 + self._deferred_prefix_token_counts.clear()
209 + self._deferred_trailing_token_count = 0
210 return results
211
212 def _resolve_deferred(
213 self,
214 delta_text: str,
148 - ) -> tuple[list[LexerInput], str]:
215 + current_token_count: int = 0,
216 + ) -> tuple[list[LexerInput], str, int]:
217 """Resolve deferred terminals against new delta_text.
218
219 When a previous ``scan()`` deferred a terminal (its text hadn't
@@ -160,7 +228,11 @@ class TokenIDScanner:
228 should be scanned with the current delta's token IDs.
229 """
230 deferred = self._deferred_terminals
231 + prefix_token_counts = self._deferred_prefix_token_counts
232 + trailing_token_count = self._deferred_trailing_token_count
233 self._deferred_terminals = []
234 + self._deferred_prefix_token_counts = []
235 + self._deferred_trailing_token_count = 0
236
237 results: list[LexerInput] = []
238 remaining = delta_text
@@ -171,10 +243,13 @@ class TokenIDScanner:
243
244 # Duplicate-text deferred terminals resolve left-to-right via
245 # find(); correct when each terminal text appears once in sequence.
174 - for terminal in deferred:
246 + for idx, terminal in enumerate(deferred):
247 + prefix_token_count = prefix_token_counts[idx]
248 pos = remaining.find(terminal.text)
249 if pos > 0:
177 - results.append(TextChunk(remaining[:pos]))
250 + results.append(
251 + TextChunk(remaining[:pos], token_count=prefix_token_count)
252 + )
253 results.append(terminal)
254 remaining = remaining[pos + len(terminal.text) :]
255 elif pos == 0:
@@ -185,10 +260,16 @@ class TokenIDScanner:
260 # only the terminal provides a reliable split point.
261 if remaining:
262 self._deferred_post_text += remaining
263 + prefix_token_count += current_token_count
264 remaining = ""
265 self._deferred_terminals.append(terminal)
266 + self._deferred_prefix_token_counts.append(prefix_token_count)
267 +
268 + if self._deferred_terminals:
269 + self._deferred_trailing_token_count = trailing_token_count
270 + trailing_token_count = 0
271
191 - return results, remaining
272 + return results, remaining, trailing_token_count
273
274 def _recover_holdback_text(
275 self,
@@ -257,6 +338,16 @@ class TokenIDScanner:
338 if not anchors:
339 return [TextChunk(delta_text)]
340
341 + token_groups: list[list[str]] = [[] for _ in range(len(anchors) + 1)]
342 + count_groups = [0] * (len(anchors) + 1)
343 + group_idx = 0
344 + for item in results:
345 + if isinstance(item, PreLexedTerminal):
346 + group_idx += 1
347 + else:
348 + token_groups[group_idx].extend(item.token_texts)
349 + count_groups[group_idx] += item.token_count
350 +
351 # Resolve positions right-to-left: each anchor gets the
352 # rightmost occurrence that is still before the next anchor.
353 positions: list[int] = [-1] * len(anchors)
@@ -274,7 +365,13 @@ class TokenIDScanner:
365 pos = positions[i]
366 if pos >= consumed:
367 if pos > consumed:
277 - new_results.append(TextChunk(delta_text[consumed:pos]))
368 + new_results.append(
369 + TextChunk(
370 + delta_text[consumed:pos],
371 + tuple(token_groups[i]),
372 + count_groups[i],
373 + )
374 + )
375 new_results.append(anchor)
376 consumed = pos + len(anchor.text)
377 else:
@@ -291,6 +388,13 @@ class TokenIDScanner:
388 self._deferred_post_text += delta_text[consumed:]
389 consumed = len(delta_text)
390 self._deferred_terminals.append(anchor)
391 + self._deferred_prefix_token_counts.append(count_groups[i])
392 if consumed < len(delta_text):
295 - new_results.append(TextChunk(delta_text[consumed:]))
393 + new_results.append(
394 + TextChunk(
395 + delta_text[consumed:],
396 + tuple(token_groups[-1]),
397 + count_groups[-1],
398 + )
399 + )
400 return new_results
vllm/parser/parser_manager.py
+30 -4
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
9
10 if TYPE_CHECKING:
11 from vllm.parser.abstract_parser import Parser
12 + from vllm.parser.engine.parser_engine import ParserEngine
13 from vllm.reasoning import ReasoningParser
14 from vllm.tool_parsers import ToolParser
15
@@ -17,10 +18,30 @@ logger = init_logger(__name__)
18
19 class ParserManager:
20 """
20 - Provides a unified Parser by composing individual reasoning and tool
21 - parsers from their respective registries.
21 + Provides a unified Parser from the reasoning and tool parser registries.
22 +
23 + Parser engine adapters backed by the same engine are collapsed back into
24 + that engine. Other parser pairs are composed through ``DelegatingParser``.
25 """
26
27 + @staticmethod
28 + def _get_parser_engine_cls(
29 + parser_cls: type[object] | None,
30 + ) -> type[ParserEngine] | None:
31 + if parser_cls is None:
32 + return None
33 + parser_engine_cls = getattr(parser_cls, "_parser_engine_cls", None)
34 + if parser_engine_cls is None:
35 + return None
36 +
37 + from vllm.parser.engine.parser_engine import ParserEngine
38 +
39 + if not isinstance(parser_engine_cls, type) or not issubclass(
40 + parser_engine_cls, ParserEngine
41 + ):
42 + return None
43 + return parser_engine_cls
44 +
45 @classmethod
46 def get_tool_parser(
47 cls,
@@ -84,8 +105,8 @@ class ParserManager:
105 """
106 Get a Parser that handles both reasoning and tool parsing.
107
87 - Composes individual reasoning and tool parsers into a single
88 - DelegatingParser subclass.
108 + Reuses a shared parser engine when possible, otherwise composes the
109 + individual parsers into a ``DelegatingParser`` subclass.
110
111 Args:
112 tool_parser_name: The name of the tool parser.
@@ -116,6 +137,11 @@ class ParserManager:
137 HarmonyParser.tool_parser_cls = tool_parser_cls
138 return HarmonyParser
139
140 + reasoning_engine_cls = cls._get_parser_engine_cls(reasoning_parser_cls)
141 + tool_engine_cls = cls._get_parser_engine_cls(tool_parser_cls)
142 + if reasoning_engine_cls is not None and reasoning_engine_cls is tool_engine_cls:
143 + return reasoning_engine_cls
144 +
145 if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3":
146 from vllm.parser.kimi_k3 import KimiK3Parser
147