[Refactor] Responses API parser state into conversation context (#46030)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Chauncey committed
Jun 23, 2026 at 13:42 UTC
3ce582376280a8d2fae8345789b2abaf6f7f07b8
3 files changed
+112
-62
tests/entrypoints/openai/responses/test_serving_responses.py
+27
-20
@@ -386,8 +386,15 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
386
reasoning_parser="qwen3",
387
)
388
389
+ request = ResponsesRequest(input="hi", tools=[], stream=False)
390
+ response_parser = serving._make_response_parser(
391
+ request,
392
+ tokenizer,
393
+ serving._effective_chat_template_kwargs(request),
394
+ )
395
+
396
# Build a SimpleContext with thinking tokens in the output.
390
- context = SimpleContext()
397
+ context = SimpleContext(response_parser=response_parser)
398
token_ids = [1, 10, 2, 20] # <think> 10 </think> 20 -> reasoning token count = 1
399
completion = CompletionOutput(
400
index=0,
@@ -412,7 +419,6 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
419
async def dummy_result_generator():
420
yield None
421
415
- request = ResponsesRequest(input="hi", tools=[], stream=False)
422
sampling_params = SamplingParams(max_tokens=16)
423
metadata = RequestResponseMetadata(request_id="req")
424
@@ -636,9 +642,9 @@ class TestHarmonyPreambleStreaming:
642
assert "response.output_text.done" not in type_names
643
644
639
-def _make_simple_context_with_output(text, token_ids):
645
+def _make_simple_context_with_output(text, token_ids, response_parser=None):
646
"""Create a SimpleContext with a RequestOutput containing the given text."""
641
- ctx = SimpleContext()
647
+ ctx = SimpleContext(response_parser=response_parser)
648
completion = CompletionOutput(
649
index=0,
650
text=text,
@@ -719,6 +725,7 @@ def _mock_parser_with_reasoning(serving, delta_sequence: list[DeltaMessage]):
725
mock_parser_instance.parse_delta = mock_parse_delta
726
mock_parser_instance.is_reasoning_end = MagicMock(return_value=False)
727
serving.parser = MagicMock(return_value=mock_parser_instance)
728
+ return mock_parser_instance
729
730
731
class TestStreamingReasoningToContentTransition:
@@ -745,12 +752,12 @@ class TestStreamingReasoningToContentTransition:
752
DeltaMessage(reasoning=" end", content="hello"), # mixed delta
753
DeltaMessage(content=" world"),
754
]
748
- _mock_parser_with_reasoning(serving, delta_sequence)
755
+ response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
756
# Create contexts for each streaming chunk
757
contexts = [
751
- _make_simple_context_with_output("chunk1", [10]),
752
- _make_simple_context_with_output("chunk2", [20]),
753
- _make_simple_context_with_output("chunk3", [30]),
758
+ _make_simple_context_with_output("chunk1", [10], response_parser),
759
+ _make_simple_context_with_output("chunk2", [20], response_parser),
760
+ _make_simple_context_with_output("chunk3", [30], response_parser),
761
]
762
763
async def result_generator():
@@ -767,7 +774,7 @@ class TestStreamingReasoningToContentTransition:
774
request=request,
775
sampling_params=sampling_params,
776
result_generator=result_generator(),
770
- context=SimpleContext(),
777
+ context=SimpleContext(response_parser=response_parser),
778
model_name="test-model",
779
tokenizer=MagicMock(),
780
request_metadata=metadata,
@@ -813,11 +820,11 @@ class TestStreamingReasoningToContentTransition:
820
DeltaMessage(reasoning="thinking"),
821
DeltaMessage(content="answer"),
822
]
816
- _mock_parser_with_reasoning(serving, delta_sequence)
823
+ response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
824
825
contexts = [
819
- _make_simple_context_with_output("chunk1", [10]),
820
- _make_simple_context_with_output("chunk2", [20]),
826
+ _make_simple_context_with_output("chunk1", [10], response_parser),
827
+ _make_simple_context_with_output("chunk2", [20], response_parser),
828
]
829
830
async def result_generator():
@@ -834,7 +841,7 @@ class TestStreamingReasoningToContentTransition:
841
request=request,
842
sampling_params=sampling_params,
843
result_generator=result_generator(),
837
- context=SimpleContext(),
844
+ context=SimpleContext(response_parser=response_parser),
845
model_name="test-model",
846
tokenizer=MagicMock(),
847
request_metadata=metadata,
@@ -875,11 +882,11 @@ class TestStreamingReasoningToContentTransition:
882
DeltaMessage(reasoning="step 1"),
883
DeltaMessage(reasoning=" step 2"),
884
]
878
- _mock_parser_with_reasoning(serving, delta_sequence)
885
+ response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
886
887
contexts = [
881
- _make_simple_context_with_output("chunk1", [10]),
882
- _make_simple_context_with_output("chunk2", [20]),
888
+ _make_simple_context_with_output("chunk1", [10], response_parser),
889
+ _make_simple_context_with_output("chunk2", [20], response_parser),
890
]
891
892
async def result_generator():
@@ -896,7 +903,7 @@ class TestStreamingReasoningToContentTransition:
903
request=request,
904
sampling_params=sampling_params,
905
result_generator=result_generator(),
899
- context=SimpleContext(),
906
+ context=SimpleContext(response_parser=response_parser),
907
model_name="test-model",
908
tokenizer=MagicMock(),
909
request_metadata=metadata,
@@ -936,10 +943,10 @@ class TestAutoToolStreaming:
943
@staticmethod
944
async def _collect_events(delta_sequence: list[DeltaMessage]):
945
serving = _make_serving_instance_with_reasoning()
939
- _mock_parser_with_reasoning(serving, delta_sequence)
946
+ response_parser = _mock_parser_with_reasoning(serving, delta_sequence)
947
948
contexts = [
942
- _make_simple_context_with_output("chunk", [i])
949
+ _make_simple_context_with_output("chunk", [i], response_parser)
950
for i in range(len(delta_sequence))
951
]
952
@@ -974,7 +981,7 @@ class TestAutoToolStreaming:
981
request=request,
982
sampling_params=sampling_params,
983
result_generator=result_generator(),
977
- context=SimpleContext(),
984
+ context=SimpleContext(response_parser=response_parser),
985
model_name="test-model",
986
tokenizer=MagicMock(),
987
request_metadata=metadata,
vllm/entrypoints/openai/responses/context.py
+29
-7
@@ -107,6 +107,8 @@ class TurnMetrics:
107
108
109
class ConversationContext(ABC):
110
+ response_parser: Parser | None = None
111
+
112
@abstractmethod
113
def append_output(self, output: RequestOutput) -> None:
114
pass
@@ -167,8 +169,25 @@ def _create_json_parse_error_messages(
169
class SimpleContext(ConversationContext):
170
"""This is a context that cannot handle MCP tool calls"""
171
170
- def __init__(self):
172
+ def __init__(
173
+ self,
174
+ *,
175
+ response_parser: Parser | None = None,
176
+ parser_cls: type[Parser] | None = None,
177
+ tokenizer: TokenizerLike | None = None,
178
+ request: ResponsesRequest | None = None,
179
+ chat_template_kwargs: dict[str, Any] | None = None,
180
+ ):
181
self.last_output = None
182
+ self.response_parser = response_parser or (
183
+ parser_cls(
184
+ tokenizer,
185
+ request.tools,
186
+ chat_template_kwargs=chat_template_kwargs,
187
+ )
188
+ if parser_cls is not None and tokenizer is not None and request is not None
189
+ else None
190
+ )
191
192
# Accumulated final output for streaming mode
193
self._accumulated_text: str = ""
@@ -181,7 +200,7 @@ class SimpleContext(ConversationContext):
200
# todo num_reasoning_tokens is not implemented yet.
201
self.num_reasoning_tokens = 0
202
# not implemented yet for SimpleContext
184
- self.all_turn_metrics = []
203
+ self.all_turn_metrics: list[TurnMetrics] = []
204
205
self.input_messages: list[ResponseRawMessageAndToken] = []
206
self.kv_transfer_params: dict[str, Any] | None = None
@@ -280,6 +299,7 @@ class ParsableContext(ConversationContext):
299
available_tools: list[str] | None,
300
chat_template: str | None,
301
chat_template_content_format: ChatTemplateContentFormatOption,
302
+ response_parser: Parser | None = None,
303
enable_auto_tools: bool = False,
304
tool_call_id_type: str = "random",
305
):
@@ -296,13 +316,13 @@ class ParsableContext(ConversationContext):
316
self.enable_auto_tools = enable_auto_tools
317
self.tool_call_id_type = tool_call_id_type
318
299
- self.parser_instance: Parser | None = None
300
- if parser_cls is not None:
319
+ self.response_parser = response_parser
320
+ if self.response_parser is None and parser_cls is not None:
321
chat_template_kwargs = request.build_chat_params(
322
default_template=chat_template,
323
default_template_content_format=chat_template_content_format,
324
).chat_template_kwargs
305
- self.parser_instance = parser_cls(
325
+ self.response_parser = parser_cls(
326
tokenizer,
327
tools=request.tools,
328
chat_template_kwargs=chat_template_kwargs,
@@ -334,8 +354,8 @@ class ParsableContext(ConversationContext):
354
completion = output.outputs[0]
355
self.finish_reason = completion.finish_reason
356
337
- if self.parser_instance is not None:
338
- reasoning, content, tool_calls = self.parser_instance.parse(
357
+ if self.response_parser is not None:
358
+ reasoning, content, tool_calls = self.response_parser.parse(
359
completion.text,
360
self.request,
361
enable_auto_tools=self.enable_auto_tools,
@@ -591,8 +611,10 @@ class HarmonyContext(ConversationContext):
611
messages: list,
612
available_tools: list[str],
613
function_tool_names: frozenset[str] | None = None,
614
+ response_parser: Parser | None = None,
615
):
616
self._messages = messages
617
+ self.response_parser = response_parser
618
self.finish_reason: str | None = None
619
self.available_tools = available_tools
620
self.function_tool_names = function_tool_names
vllm/entrypoints/openai/responses/serving.py
+56
-35
@@ -260,6 +260,20 @@ class OpenAIServingResponses(OpenAIServing):
260
.chat_template_kwargs
261
)
262
263
+ def _make_response_parser(
264
+ self,
265
+ request: ResponsesRequest,
266
+ tokenizer: TokenizerLike,
267
+ chat_template_kwargs: dict[str, Any],
268
+ ) -> Parser | None:
269
+ if self.parser is None:
270
+ return None
271
+ return self.parser(
272
+ tokenizer,
273
+ request.tools,
274
+ chat_template_kwargs=chat_template_kwargs,
275
+ )
276
+
277
def _validate_generator_input(
278
self,
279
engine_input: EngineInput,
@@ -443,16 +457,27 @@ class OpenAIServingResponses(OpenAIServing):
457
else await self._get_trace_headers(raw_request.headers)
458
)
459
460
+ chat_template_kwargs = self._effective_chat_template_kwargs(request)
461
+ response_parser = self._make_response_parser(
462
+ request, tokenizer, chat_template_kwargs
463
+ )
464
+
465
context: ConversationContext
466
function_tool_names = extract_function_tool_names(request.tools)
467
if self.use_harmony:
468
if request.stream:
469
context = StreamingHarmonyContext(
451
- messages, available_tools, function_tool_names
470
+ messages,
471
+ available_tools,
472
+ function_tool_names,
473
+ response_parser=response_parser,
474
)
475
else:
476
context = HarmonyContext(
455
- messages, available_tools, function_tool_names
477
+ messages,
478
+ available_tools,
479
+ function_tool_names,
480
+ response_parser=response_parser,
481
)
482
else:
483
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
@@ -463,6 +488,7 @@ class OpenAIServingResponses(OpenAIServing):
488
tokenizer=tokenizer,
489
parser_cls=self.parser,
490
request=request,
491
+ response_parser=response_parser,
492
available_tools=available_tools,
493
chat_template=self.chat_template,
494
chat_template_content_format=self.chat_template_content_format,
@@ -470,17 +496,17 @@ class OpenAIServingResponses(OpenAIServing):
496
tool_call_id_type=self.tool_call_id_type,
497
)
498
else:
473
- context = SimpleContext()
499
+ context = SimpleContext(
500
+ response_parser=response_parser,
501
+ )
502
475
- if self.parser and self.parser.reasoning_parser_cls is not None:
476
- chat_template_kwargs = self._effective_chat_template_kwargs(request)
503
+ if (
504
+ context.response_parser is not None
505
+ and context.response_parser.reasoning_parser is not None
506
+ ):
507
reasoning_parser_kwargs = {
508
"chat_template_kwargs": chat_template_kwargs,
509
}
480
- reasoning_parser = self.parser.reasoning_parser_cls(
481
- tokenizer,
482
- chat_template_kwargs=chat_template_kwargs,
483
- )
510
if (
511
isinstance(
512
struct_out := sampling_params.structured_outputs,
@@ -490,8 +516,10 @@ class OpenAIServingResponses(OpenAIServing):
516
):
517
sampling_params.structured_outputs = replace(
518
struct_out,
493
- structural_tag=reasoning_parser.prepare_structured_tag(
494
- struct_out.structural_tag, self.tool_server
519
+ structural_tag=(
520
+ context.response_parser.reasoning_parser.prepare_structured_tag(
521
+ struct_out.structural_tag, self.tool_server
522
+ )
523
),
524
)
525
generator = self._generate_with_builtin_tools(
@@ -833,7 +861,12 @@ class OpenAIServingResponses(OpenAIServing):
861
if final_output.finish_reason == "length":
862
status = "incomplete"
863
836
- output = self._make_response_output_items(request, final_output, tokenizer)
864
+ output = self._make_response_output_items(
865
+ request,
866
+ final_output,
867
+ tokenizer,
868
+ parser=context.response_parser,
869
+ )
870
871
if request.enable_response_messages:
872
input_messages = context.input_messages
@@ -854,16 +887,16 @@ class OpenAIServingResponses(OpenAIServing):
887
# accumulated output token IDs using the parser if not already set.
888
if (
889
num_reasoning_tokens == 0
857
- and self.parser is not None
858
- and self.parser.reasoning_parser_cls is not None
890
and isinstance(context, (SimpleContext, ParsableContext))
891
+ and context.response_parser is not None
892
+ and context.response_parser.reasoning_parser is not None
893
):
861
- reasoning_parser = self.parser.reasoning_parser_cls(
862
- tokenizer,
863
- chat_template_kwargs=self._effective_chat_template_kwargs(request),
864
- )
894
accumulated = getattr(context, "_accumulated_token_ids", []) or []
866
- num_reasoning_tokens = reasoning_parser.count_reasoning_tokens(accumulated)
895
+ num_reasoning_tokens = (
896
+ context.response_parser.reasoning_parser.count_reasoning_tokens(
897
+ accumulated
898
+ )
899
+ )
900
901
usage = ResponseUsage(
902
input_tokens=num_prompt_tokens,
@@ -1005,6 +1038,7 @@ class OpenAIServingResponses(OpenAIServing):
1038
request: ResponsesRequest,
1039
final_output: CompletionOutput,
1040
tokenizer: TokenizerLike,
1041
+ parser: Parser | None = None,
1042
) -> list[ResponseOutputItem]:
1043
# Log complete response if output logging is enabled
1044
if self.enable_log_outputs and self.request_logger:
@@ -1028,11 +1062,7 @@ class OpenAIServingResponses(OpenAIServing):
1062
)
1063
1064
# Use parser to extract reasoning, content, and tool calls
1031
- if self.parser:
1032
- chat_template_kwargs = self._effective_chat_template_kwargs(request)
1033
- parser = self.parser(
1034
- tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs
1035
- )
1065
+ if parser:
1066
reasoning, content, tool_calls = parser.parse(
1067
final_output.text,
1068
request,
@@ -1350,15 +1380,6 @@ class OpenAIServingResponses(OpenAIServing):
1380
],
1381
) -> AsyncGenerator[StreamingResponsesResponse, None]:
1382
processor = SimpleStreamingEventProcessor()
1353
- parser = (
1354
- self.parser(
1355
- tokenizer,
1356
- request.tools,
1357
- chat_template_kwargs=self._effective_chat_template_kwargs(request),
1358
- )
1359
- if self.parser
1360
- else None
1361
- )
1383
1384
def _get_logprobs(
1385
output: CompletionOutput,
@@ -1382,8 +1403,8 @@ class OpenAIServingResponses(OpenAIServing):
1403
delta_text = output.text
1404
delta_token_ids = as_list(output.token_ids)
1405
1385
- if parser:
1386
- delta_message = parser.parse_delta(
1406
+ if ctx.response_parser:
1407
+ delta_message = ctx.response_parser.parse_delta(
1408
delta_text=delta_text,
1409
delta_token_ids=delta_token_ids,
1410
request=request,