[Bugfix][Frontend] Respect max_output_tokens in the Harmony tool-call loop (#58551)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Du Bin <8174807+dubin555@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Denis Ermakov committed
Sep 25, 2026 at 23:00 UTC
7871963fcc8cd3484addb2f7abddcd0f2478028d
3 files changed
+57
-2
tests/entrypoints/openai/responses/test_serving_responses.py
+48
@@ -748,6 +748,7 @@ class TestInitializeToolSessions:
748
class ToolCallingHarmonyContext(HarmonyContext):
749
def __init__(self):
750
self._messages = []
751
+ self.request = None
752
self._needs_tool_call = True
753
754
def append_output(self, output) -> None:
@@ -804,6 +805,53 @@ class TestInitializeToolSessions:
805
assert followup_engine_input.get("cache_salt") == "request-salt"
806
assert followup_engine_input["prompt_token_ids"] == [3, 4]
807
808
+ @pytest.mark.asyncio
809
+ async def test_harmony_tool_followup_respects_max_output_tokens(
810
+ self, serving_responses_instance
811
+ ):
812
+ class ToolCallingHarmonyContext(HarmonyContext):
813
+ def __init__(self, request):
814
+ self._messages = []
815
+ self.request = request
816
+ self._needs_tool_call = True
817
+
818
+ def append_output(self, output) -> None:
819
+ pass
820
+
821
+ def need_builtin_tool_call(self) -> bool:
822
+ return self._needs_tool_call
823
+
824
+ async def call_tool(self):
825
+ self._needs_tool_call = False
826
+ return []
827
+
828
+ def append_tool_output(self, output) -> None:
829
+ pass
830
+
831
+ max_tokens_per_turn = []
832
+
833
+ async def generate_output(engine_input, sampling_params, *args, **kwargs):
834
+ max_tokens_per_turn.append(sampling_params.max_tokens)
835
+ yield MagicMock()
836
+
837
+ serving_responses_instance.engine_client.generate.side_effect = generate_output
838
+ serving_responses_instance.online_renderer.render_responses_harmony_messages = (
839
+ MagicMock(return_value=tokens_input([1, 2, 3]))
840
+ )
841
+ context = ToolCallingHarmonyContext(
842
+ ResponsesRequest(input="test", max_output_tokens=10)
843
+ )
844
+
845
+ async for _ in serving_responses_instance._generate_with_builtin_tools(
846
+ request_id="req",
847
+ engine_input=tokens_input([1]),
848
+ sampling_params=SamplingParams(max_tokens=10),
849
+ context=context,
850
+ ):
851
+ pass
852
+
853
+ assert max_tokens_per_turn == [10, 10]
854
+
855
@pytest.mark.asyncio
856
async def test_initialize_tool_sessions(
857
self, serving_responses_instance, mock_context, mock_exit_stack
vllm/entrypoints/openai/responses/context.py
+2
@@ -615,12 +615,14 @@ class HarmonyContext(ConversationContext):
615
available_tools: list[str],
616
function_tool_names: frozenset[str],
617
response_parser: Parser | None = None,
618
+ request: ResponsesRequest | None = None,
619
):
620
from vllm.parser.harmony import HarmonyParser, Segment
621
622
assert isinstance(response_parser, HarmonyParser)
623
624
self._messages = messages
625
+ self.request = request
626
self.response_parser: HarmonyParser = response_parser
627
self.finish_reason: str | None = None
628
self.available_tools = available_tools
vllm/entrypoints/openai/responses/serving.py
+7
-2
@@ -443,6 +443,7 @@ class OpenAIServingResponses(GenerateBaseServing):
443
available_tools,
444
function_tool_names,
445
response_parser=response_parser,
446
+ request=request,
447
)
448
else:
449
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
@@ -678,8 +679,12 @@ class OpenAIServingResponses(GenerateBaseServing):
679
tok_params=tok_params,
680
)
681
681
- sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
682
- engine_input
682
+ sampling_params.max_tokens = get_max_tokens(
683
+ max_model_len,
684
+ context.request.max_output_tokens if context.request else None,
685
+ self._extract_prompt_len(engine_input),
686
+ self.default_sampling_params,
687
+ self.override_max_tokens,
688
)
689
elif isinstance(context, ParsableContext):
690
(engine_input,) = await self._render_next_turn(