[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Sep 26, 2026 at 10:00 UTC
dfab5043336cfae4739d3d0ae54047463e26f17d
6 files changed
+83
-13
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+38
-5
@@ -1383,6 +1383,11 @@ Q35_TEMPLATE = (
1383
"{%- endfor %}"
1384
)
1385
1386
+PERMISSIVE_TEMPLATE = (
1387
+ "{%- for message in messages %}{{- message.role }}: {{ message.content }}\n"
1388
+ "{%- endfor %}"
1389
+)
1390
+
1391
1392
class TestDetectMergeInlineSystem:
1393
"""Verify _detect_merge_inline_system auto-detection.
@@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem:
1407
def test_no_restriction_no_merge(self):
1408
"""Template without restriction accepts mid-conversation system."""
1409
assert (
1405
- AnthropicServingMessages._detect_merge_inline_system(
1406
- "{%- for message in messages %}"
1407
- "{{- message.role }}: {{ message.content }}\n"
1408
- "{%- endfor %}"
1409
- )
1410
+ AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE)
1411
is False
1412
)
1413
@@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem:
1415
"""No chat_template → conservative default: merge."""
1416
assert AnthropicServingMessages._detect_merge_inline_system(None) is True
1417
1418
+ @pytest.mark.parametrize(
1419
+ ("resolved", "expected"),
1420
+ [(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)],
1421
+ )
1422
+ def test_probes_resolved_template_without_cli_override(
1423
+ self, monkeypatch, resolved, expected
1424
+ ):
1425
+ """Without --chat-template, probe the tokenizer's template (#58727).
1426
+
1427
+ Merging defeats prefix caching, so it is warned about at startup.
1428
+ """
1429
+ import vllm.entrypoints.anthropic.serving as serving_mod
1430
+ from vllm.renderers.hf import HfRenderer
1431
+
1432
+ monkeypatch.setattr(
1433
+ serving_mod, "resolve_chat_template", lambda *a, **kw: resolved
1434
+ )
1435
+ mock_logger = MagicMock()
1436
+ monkeypatch.setattr(serving_mod, "logger", mock_logger)
1437
+ renderer = MagicMock(spec=HfRenderer)
1438
+ renderer.tokenizer = MagicMock()
1439
+ online_renderer = SimpleNamespace(
1440
+ renderer=renderer,
1441
+ chat_template=None,
1442
+ model_config=None,
1443
+ )
1444
+ assert (
1445
+ AnthropicServingMessages._should_merge_inline_system(online_renderer)
1446
+ is expected
1447
+ )
1448
+ assert mock_logger.warning.called is expected
1449
+
1450
1451
# ======================================================================
1452
# Full (non-streaming) response conversion: messages_full_converter
tests/entrypoints/openai/chat_completion/test_chat_error.py
+2
@@ -45,6 +45,8 @@ class MockModelConfig:
45
trust_remote_code = False
46
tokenizer_mode = "auto"
47
max_model_len = 100
48
+ revision = None
49
+ code_revision = None
50
tokenizer_revision = None
51
multimodal_config = MultiModalConfig()
52
hf_config = MockHFConfig()
tests/entrypoints/openai/completion/test_completion_error.py
+2
@@ -56,6 +56,8 @@ class MockModelConfig:
56
trust_remote_code = False
57
tokenizer_mode = "auto"
58
max_model_len = 100
59
+ revision = None
60
+ code_revision = None
61
tokenizer_revision = None
62
multimodal_config = MultiModalConfig()
63
hf_config = MockHFConfig()
tests/entrypoints/scale_out/render/test_render.py
+3
-3
@@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client):
651
async def test_messages_render_merges_inline_system(client):
652
"""Inline system messages merge into the leading system block.
653
654
- Without a --chat-template arg the /v1/messages server path detects
655
- merge_inline_system=True, so render must produce the same tokens as
656
- the manually pre-merged request.
654
+ The model's chat template rejects non-leading system messages, so the
655
+ /v1/messages server path detects merge_inline_system=True and render must
656
+ produce the same tokens as the manually pre-merged request.
657
"""
658
inline = await client.post(
659
"/v1/messages/render",
vllm/entrypoints/anthropic/serving.py
+37
-2
@@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
51
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
52
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
53
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
54
+from vllm.logger import init_logger
55
+from vllm.renderers.hf import HfRenderer, resolve_chat_template
56
from vllm.renderers.online_renderer import OnlineRenderer
57
56
-logger = logging.getLogger(__name__)
58
+logger = init_logger(__name__)
59
60
61
def _build_anthropic_usage(
@@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat):
146
"length": "max_tokens",
147
"tool_calls": "tool_use",
148
}
147
- self._merge_inline_system = self._detect_merge_inline_system(chat_template)
149
+ self._merge_inline_system = self._should_merge_inline_system(online_renderer)
150
# Resolved lazily from the renderer when "auto".
151
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
152
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
@@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat):
195
components = self._extract_prompt_components(engine_input)
196
return components.token_ids, components.text
197
198
+ @classmethod
199
+ def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool:
200
+ """Probe the chat templates the renderer applies, not the CLI override.
201
+
202
+ Both the tool and non-tool templates are checked since they may differ.
203
+ """
204
+ renderer = online_renderer.renderer
205
+ tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, [])
206
+ if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None:
207
+ merge = cls._detect_merge_inline_system(online_renderer.chat_template)
208
+ else:
209
+ merge = any(
210
+ cls._detect_merge_inline_system(
211
+ resolve_chat_template(
212
+ renderer.tokenizer,
213
+ online_renderer.chat_template,
214
+ tools,
215
+ model_config=online_renderer.model_config,
216
+ )
217
+ )
218
+ for tools in tool_variants
219
+ )
220
+ if merge:
221
+ logger.warning(
222
+ "The chat template requires system-first ordering, so inline "
223
+ "system messages in /v1/messages requests (e.g. Claude Code's "
224
+ "per-turn reminders) are merged into the leading system prompt. "
225
+ "Each new one changes the prompt prefix, so the rest of the "
226
+ "conversation misses the prefix cache. Pass a --chat-template "
227
+ "that accepts non-leading system messages to avoid this."
228
+ )
229
+ return merge
230
+
231
@staticmethod
232
def _detect_merge_inline_system(chat_template: str | None) -> bool:
233
"""Auto-detect whether the chat template requires system-first ordering.
vllm/entrypoints/scale_out/render/serving.py
+1
-3
@@ -57,9 +57,7 @@ class ServingRender(BaseServing):
57
self.tool_server = tool_server
58
59
self._merge_inline_system = (
60
- AnthropicServingMessages._detect_merge_inline_system(
61
- online_renderer.chat_template
62
- )
60
+ AnthropicServingMessages._should_merge_inline_system(online_renderer)
61
)
62
63
self._placeholder_metadata_parser: MultiModalDataParser | None = None