[Bugfix][Frontend] Count Responses reasoning tokens per tool round (#58927)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 27, 2026 at 20:34 UTC
3184226984b17fe0dba960ace5a2d7d57df44ab6
4 files changed
+94
-56
tests/entrypoints/openai/responses/test_parsable_context_unit.py
+31
@@ -84,6 +84,18 @@ class _ReasoningOnlyParser(DelegatingParser):
84
return None
85
86
87
+class _CountingParser(_NoOpParser):
88
+ """Parser that records the ids it counts; token id 7 is reasoning."""
89
+
90
+ def __init__(self, tokenizer, *args, **kwargs):
91
+ super().__init__(tokenizer)
92
+ self.counted_ids: list[list[int]] = []
93
+
94
+ def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
95
+ self.counted_ids.append(list(token_ids))
96
+ return sum(token_id == 7 for token_id in token_ids)
97
+
98
+
99
class _StubToolParser:
100
"""Minimal tool parser stub that always returns a hardcoded tool call."""
101
@@ -339,6 +351,25 @@ def test_multi_turn_accumulation():
351
assert texts == ["First turn", "Second turn"]
352
353
354
+def test_reasoning_tokens_counted_per_round():
355
+ """Each round is counted on its own ids, not the concatenated rounds."""
356
+ ctx = _make_context(_CountingParser)
357
+
358
+ ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
359
+ ctx.append_output(_make_request_output(token_ids=[7, 2]))
360
+
361
+ assert ctx.response_parser.counted_ids == [[7, 7, 1], [7, 2]]
362
+ assert ctx.num_reasoning_tokens == 3
363
+
364
+
365
+def test_reasoning_tokens_zero_without_parser():
366
+ ctx = _make_context(None)
367
+
368
+ ctx.append_output(_make_request_output(token_ids=[7, 7, 1]))
369
+
370
+ assert ctx.num_reasoning_tokens == 0
371
+
372
+
373
def test_num_init_messages_offset():
374
"""Initial messages are preserved and offset works correctly."""
375
init_messages = [MagicMock(type="message")]
tests/entrypoints/openai/responses/test_serving_responses.py
+54
-45
@@ -960,24 +960,19 @@ class TestValidateGeneratorInput:
960
assert isinstance(result, ErrorResponse)
961
962
963
-@pytest.mark.asyncio
964
-async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
965
- """Ensure reasoning_tokens usage is derived from thinking token spans."""
966
-
967
- class FakeTokenizer:
968
- def __init__(self):
969
- self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
963
+class _Qwen3FakeTokenizer:
964
+ def __init__(self):
965
+ self._vocab = {"<think>": 1, "</think>": 2, "reason": 3, "final": 4}
966
971
- def get_vocab(self):
972
- return self._vocab
967
+ def get_vocab(self):
968
+ return self._vocab
969
974
- def decode(self, token_ids):
975
- id_to_token = {v: k for k, v in self._vocab.items()}
976
- return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
970
+ def decode(self, token_ids):
971
+ id_to_token = {v: k for k, v in self._vocab.items()}
972
+ return "".join(id_to_token.get(token_id, "x") for token_id in token_ids)
973
978
- # Force non-harmony, SimpleContext path
979
- monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
974
975
+def _make_qwen3_serving() -> tuple[OpenAIServingResponses, _Qwen3FakeTokenizer]:
976
engine_client = MagicMock()
977
model_config = MagicMock()
978
model_config.hf_config.model_type = "test"
@@ -987,68 +982,82 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
982
engine_client.input_processor = MagicMock()
983
engine_client.renderer = MagicMock()
984
990
- tokenizer = FakeTokenizer()
985
+ tokenizer = _Qwen3FakeTokenizer()
986
engine_client.renderer.get_tokenizer.return_value = tokenizer
987
993
- models = MagicMock()
994
-
988
serving = OpenAIServingResponses(
989
engine_client=engine_client,
997
- models=models,
990
+ models=MagicMock(),
991
online_renderer=MagicMock(),
992
request_logger=None,
993
chat_template=None,
994
chat_template_content_format="auto",
995
reasoning_parser="qwen3",
996
)
997
+ return serving, tokenizer
998
1005
- request = ResponsesRequest(input="hi", tools=[], stream=False)
1006
- response_parser = serving._make_response_parser(
1007
- request,
1008
- tokenizer,
1009
- serving._effective_chat_template_kwargs(request),
1010
- )
999
1012
- # Build a SimpleContext with thinking tokens in the output.
1013
- context = SimpleContext(response_parser=response_parser)
1014
- token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
1015
- completion = CompletionOutput(
1016
- index=0,
1017
- text="<think>reason</think>final",
1018
- token_ids=token_ids,
1019
- cumulative_logprob=0.0,
1020
- logprobs=None,
1021
- finish_reason="stop",
1022
- stop_reason=None,
1023
- )
1024
- req_output = RequestOutput(
1000
+def _make_text_request_output(text: str, token_ids: list[int]) -> RequestOutput:
1001
+ return RequestOutput(
1002
request_id="req",
1003
prompt="hi",
1004
prompt_token_ids=[7, 8],
1005
prompt_logprobs=None,
1029
- outputs=[completion],
1006
+ outputs=[
1007
+ CompletionOutput(
1008
+ index=0,
1009
+ text=text,
1010
+ token_ids=token_ids,
1011
+ cumulative_logprob=0.0,
1012
+ logprobs=None,
1013
+ finish_reason="stop",
1014
+ stop_reason=None,
1015
+ )
1016
+ ],
1017
finished=True,
1018
num_cached_tokens=0,
1019
num_cache_creation_tokens=3,
1020
)
1034
- context.append_output(req_output)
1021
1022
+
1023
+async def _run_full_generator(serving, request, context, tokenizer):
1024
async def dummy_result_generator():
1025
yield None
1026
1039
- sampling_params = SamplingParams(max_tokens=16)
1040
- metadata = RequestResponseMetadata(request_id="req")
1041
-
1042
- response = await serving.responses_full_generator(
1027
+ return await serving.responses_full_generator(
1028
request=request,
1044
- sampling_params=sampling_params,
1029
+ sampling_params=SamplingParams(max_tokens=16),
1030
result_generator=dummy_result_generator(),
1031
context=context,
1032
model_name="test-model",
1033
tokenizer=tokenizer,
1049
- request_metadata=metadata,
1034
+ request_metadata=RequestResponseMetadata(request_id="req"),
1035
+ )
1036
+
1037
+
1038
+@pytest.mark.asyncio
1039
+async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
1040
+ """Ensure reasoning_tokens usage is derived from thinking token spans."""
1041
+ # Force non-harmony, SimpleContext path
1042
+ monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
1043
+
1044
+ serving, tokenizer = _make_qwen3_serving()
1045
+ request = ResponsesRequest(input="hi", tools=[], stream=False)
1046
+ response_parser = serving._make_response_parser(
1047
+ request,
1048
+ tokenizer,
1049
+ serving._effective_chat_template_kwargs(request),
1050
+ )
1051
+
1052
+ # Build a SimpleContext with thinking tokens in the output.
1053
+ context = SimpleContext(response_parser=response_parser)
1054
+ # <think> 10 </think> 20 -> reasoning token count = 1
1055
+ context.append_output(
1056
+ _make_text_request_output("<think>reason</think>final", [1, 10, 2, 20])
1057
)
1058
1059
+ response = await _run_full_generator(serving, request, context, tokenizer)
1060
+
1061
assert response.usage.output_tokens_details.reasoning_tokens == 1
1062
assert response.usage.input_tokens_details.cache_write_tokens == 3
1063
vllm/entrypoints/openai/responses/context.py
+4
@@ -359,6 +359,10 @@ class ParsableContext(ConversationContext):
359
enable_auto_tools=self.enable_auto_tools,
360
model_output_token_ids=completion.token_ids,
361
)
362
+ # Each round is a separate generation, so count it on its own ids.
363
+ self.num_reasoning_tokens += self.response_parser.count_reasoning_tokens(
364
+ completion.token_ids or []
365
+ )
366
if not self.request.include_reasoning:
367
reasoning = None
368
self.response_messages.extend(
vllm/entrypoints/openai/responses/serving.py
+5
-11
@@ -832,18 +832,12 @@ class OpenAIServingResponses(GenerateBaseServing):
832
num_generated_tokens = context.num_output_tokens
833
num_cached_tokens = context.num_cached_tokens
834
num_reasoning_tokens = context.num_reasoning_tokens
835
- # For text-based reasoning parsers (e.g., <think>...</think>),
836
- # HarmonyContext already counts reasoning tokens via channels.
837
- # For Simple/Parsable contexts, derive reasoning_tokens from
838
- # accumulated output token IDs using the parser if not already set.
839
- if (
840
- num_reasoning_tokens == 0
841
- and isinstance(context, (SimpleContext, ParsableContext))
842
- and context.response_parser is not None
843
- ):
844
- accumulated = getattr(context, "_accumulated_token_ids", []) or []
835
+ # HarmonyContext and ParsableContext count reasoning tokens as each
836
+ # round is appended. SimpleContext is single-round but accumulates
837
+ # streaming deltas, so count its full output once here.
838
+ if isinstance(context, SimpleContext) and context.response_parser is not None:
839
num_reasoning_tokens = context.response_parser.count_reasoning_tokens(
846
- accumulated
840
+ context._accumulated_token_ids
841
)
842
843
usage = ResponseUsage(