[Bugfix][Frontend] Apply Harmony adjust_request in batched chat completions (#58958)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Sep 28, 2026 at 00:12 UTC
6f65fde4194bcbe97dafd45897e944b47b245f3a
3 files changed
+84
-17
tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py
+63
@@ -3,10 +3,20 @@
3
4
import json
5
from collections.abc import AsyncGenerator
6
+from unittest.mock import MagicMock
7
8
import httpx
9
import pytest
10
11
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
12
+ BASE_MODEL_PATHS,
13
+ MockHFConfig,
14
+ MockModelConfig,
15
+ _build_renderer,
16
+)
17
+from tests.entrypoints.openai.chat_completion.test_serving_chat import (
18
+ MODEL_NAME as MOCK_MODEL_NAME,
19
+)
20
from tests.utils import RemoteOpenAIServer
21
from vllm.entrypoints.generate.base.protocol import RequestResponseMetadata
22
from vllm.entrypoints.openai.chat_completion.batch_serving import (
@@ -15,7 +25,10 @@ from vllm.entrypoints.openai.chat_completion.batch_serving import (
25
from vllm.entrypoints.openai.chat_completion.protocol import (
26
BatchChatCompletionRequest,
27
)
28
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
29
from vllm.outputs import CompletionOutput, RequestOutput
30
+from vllm.renderers.online_renderer import OnlineRenderer
31
+from vllm.v1.engine.async_llm import AsyncLLM
32
33
# any model with a chat template defined in tokenizer_config should work here
34
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
@@ -322,3 +335,53 @@ async def test_batched_echo_prepends_matching_assistant_prefix() -> None:
335
)
336
337
assert response.choices[0].message.content == "PREFIX ASSISTANT ANSWER"
338
+
339
+
340
+@pytest.mark.asyncio
341
+@pytest.mark.skip_global_cleanup
342
+async def test_batched_harmony_response_format_uses_structural_tag() -> None:
343
+ """Harmony must constrain only the final channel of every conversation."""
344
+ mock_engine = MagicMock(spec=AsyncLLM)
345
+ mock_engine.model_config = MockModelConfig()
346
+ mock_engine.errored = False
347
+ mock_engine.input_processor = MagicMock()
348
+ mock_engine.model_config.hf_config = MockHFConfig(model_type="gpt_oss")
349
+ mock_engine.renderer = _build_renderer(mock_engine.model_config)
350
+ mock_engine.generate.side_effect = lambda *args, **kwargs: _generator(0, "ok")
351
+
352
+ models = OpenAIServingModels(mock_engine, BASE_MODEL_PATHS)
353
+ online_renderer = OnlineRenderer(
354
+ model_config=mock_engine.model_config,
355
+ renderer=mock_engine.renderer,
356
+ request_logger=None,
357
+ chat_template=None,
358
+ chat_template_content_format="auto",
359
+ reasoning_parser="openai_gptoss",
360
+ )
361
+ serving = OpenAIServingChatBatch(
362
+ mock_engine,
363
+ models,
364
+ response_role="assistant",
365
+ online_renderer=online_renderer,
366
+ chat_template=None,
367
+ chat_template_content_format="auto",
368
+ request_logger=None,
369
+ )
370
+
371
+ request = BatchChatCompletionRequest(
372
+ model=MOCK_MODEL_NAME,
373
+ messages=[
374
+ [{"role": "user", "content": "France?"}],
375
+ [{"role": "user", "content": "Japan?"}],
376
+ ],
377
+ response_format={
378
+ "type": "json_schema",
379
+ "json_schema": {"name": "city", "schema": {"type": "object"}},
380
+ },
381
+ )
382
+ await serving.create_batch_chat_completion(request)
383
+
384
+ calls = mock_engine.generate.call_args_list
385
+ assert len(calls) == 2
386
+ for call in calls:
387
+ assert call.args[1].structured_outputs.structural_tag is not None
vllm/entrypoints/openai/chat_completion/batch_serving.py
+1
@@ -91,6 +91,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
91
for messages in request.messages:
92
single_request = request.to_chat_completion_request(messages)
93
if renderer.use_harmony:
94
+ renderer.adjust_harmony_request(single_request)
95
conversation, engine_prompts = renderer._make_request_with_harmony(
96
single_request, should_include_tools=tool_dicts is not None
97
)
vllm/renderers/online_renderer.py
+20
-17
@@ -259,15 +259,7 @@ class OnlineRenderer:
259
)
260
else:
261
# For GPT-OSS.
262
- if self.parser is not None:
263
- # HarmonyParser doesn't need chat_template_kwargs
264
- # TODO: Unify adjust_request() call with non-harmony branch
265
- self.parser(
266
- self.renderer.get_tokenizer(),
267
- request.tools,
268
- model_config=self.model_config,
269
- ).adjust_request(request=request)
270
-
262
+ self.adjust_harmony_request(request)
263
should_include_tools = tool_dicts is not None
264
conversation, engine_inputs = self._make_request_with_harmony(
265
request, should_include_tools
@@ -355,14 +347,7 @@ class OnlineRenderer:
347
previous_response_outputs: list[ResponseOutputItem] | None,
348
tool_server: "ToolServer | None",
349
) -> ResponsesRenderResult | ErrorResponse:
358
- if self.parser is not None:
359
- # HarmonyParser doesn't need chat_template_kwargs
360
- # TODO: Unify adjust_request() call with non-harmony branch
361
- self.parser(
362
- self.renderer.get_tokenizer(),
363
- request.tools,
364
- model_config=self.model_config,
365
- ).adjust_request(request=request)
350
+ self.adjust_harmony_request(request)
351
352
if previous_messages is not None and any(
353
not isinstance(message, OpenAIMessage) for message in previous_messages
@@ -488,6 +473,24 @@ class OnlineRenderer:
473
engine_input=engine_inputs[0],
474
)
475
476
+ def adjust_harmony_request(
477
+ self, request: ChatCompletionRequest | ResponsesRequest
478
+ ) -> None:
479
+ """Apply the Harmony parser's ``adjust_request`` to ``request`` in place.
480
+
481
+ Every Harmony render path must call this before sampling params are
482
+ built from ``request``.
483
+ """
484
+ if self.parser is None:
485
+ return
486
+ # HarmonyParser doesn't need chat_template_kwargs
487
+ # TODO: Unify adjust_request() call with non-harmony branch
488
+ self.parser(
489
+ self.renderer.get_tokenizer(),
490
+ request.tools,
491
+ model_config=self.model_config,
492
+ ).adjust_request(request=request)
493
+
494
def _make_request_with_harmony(
495
self,
496
request: ChatCompletionRequest,