fix(anthropic): auto-detect template support for mid-conversation system messages (#46025)
Signed-off-by: felix0080 <felix0080@users.noreply.github.com> Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: felix0080 <felix0080@users.noreply.github.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
MrFan committed
Jun 19, 2026 at 04:19 UTC
4ce2d0145312809ef6122ccb7be8ae7cafa462a9
2 files changed
+120
-6
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+47
@@ -1096,3 +1096,50 @@ class TestMessageStartIncludesTypeAndRole:
1096
message = events[0][1]["message"]
1097
assert message["type"] == "message"
1098
assert message["role"] == "assistant"
1099
+
1100
+
1101
+# ======================================================================
1102
+# Auto-detection of system-first template requirement
1103
+# ======================================================================
1104
+
1105
+
1106
+Q35_TEMPLATE = (
1107
+ "{%- for message in messages %}"
1108
+ "{%- if message.role == 'system' %}"
1109
+ "{%- if not loop.first %}"
1110
+ "{{- raise_exception('System message must be at the beginning.') }}"
1111
+ "{%- endif %}"
1112
+ "{%- endif %}"
1113
+ "{%- endfor %}"
1114
+)
1115
+
1116
+
1117
+class TestDetectMergeInlineSystem:
1118
+ """Verify _detect_merge_inline_system auto-detection.
1119
+
1120
+ Tests three scenarios:
1121
+ 1. Template with system-first guard (e.g. Qwen) → merge needed
1122
+ 2. Template without restrictions → no merge, cache-friendly
1123
+ 3. No template provided → safe default: merge
1124
+ """
1125
+
1126
+ def test_qwen_template_requires_merge(self):
1127
+ """Template with loop.first guard rejects mid-conversation system."""
1128
+ assert (
1129
+ AnthropicServingMessages._detect_merge_inline_system(Q35_TEMPLATE) is True
1130
+ )
1131
+
1132
+ def test_no_restriction_no_merge(self):
1133
+ """Template without restriction accepts mid-conversation system."""
1134
+ assert (
1135
+ AnthropicServingMessages._detect_merge_inline_system(
1136
+ "{%- for message in messages %}"
1137
+ "{{- message.role }}: {{ message.content }}\n"
1138
+ "{%- endfor %}"
1139
+ )
1140
+ is False
1141
+ )
1142
+
1143
+ def test_no_template_defaults_merge(self):
1144
+ """No chat_template → conservative default: merge."""
1145
+ assert AnthropicServingMessages._detect_merge_inline_system(None) is True
vllm/entrypoints/anthropic/serving.py
+73
-6
@@ -12,6 +12,7 @@ import uuid
12
from collections.abc import AsyncGenerator
13
from typing import TYPE_CHECKING, Any
14
15
+import jinja2
16
from fastapi import Request
17
18
from vllm.engine.protocol import EngineClient
@@ -99,6 +100,36 @@ class AnthropicServingMessages(OpenAIServingChat):
100
"length": "max_tokens",
101
"tool_calls": "tool_use",
102
}
103
+ self._merge_inline_system = self._detect_merge_inline_system(chat_template)
104
+
105
+ @staticmethod
106
+ def _detect_merge_inline_system(chat_template: str | None) -> bool:
107
+ """Auto-detect whether the chat template requires system-first ordering.
108
+
109
+ Renders a [system, user, system, user] conversation against the
110
+ template; if it raises (e.g. Qwen's ``loop.first`` guard), the
111
+ model needs inline system messages merged into the leading block.
112
+ """
113
+ if not chat_template:
114
+ return True
115
+ try:
116
+ env = jinja2.sandbox.ImmutableSandboxedEnvironment(
117
+ trim_blocks=True,
118
+ lstrip_blocks=True,
119
+ extensions=[jinja2.ext.loopcontrols],
120
+ )
121
+ env.from_string(chat_template).render(
122
+ messages=[
123
+ {"role": "system", "content": "t"},
124
+ {"role": "user", "content": "t"},
125
+ {"role": "system", "content": "t"},
126
+ {"role": "user", "content": "t"},
127
+ ],
128
+ add_generation_prompt=False,
129
+ )
130
+ return False
131
+ except jinja2.TemplateError:
132
+ return True
133
134
@staticmethod
135
def _convert_image_source_to_url(source: dict[str, Any]) -> str:
@@ -123,13 +154,24 @@ class AnthropicServingMessages(OpenAIServingChat):
154
155
@classmethod
156
def _convert_anthropic_to_openai_request(
126
- cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest
157
+ cls,
158
+ anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
159
+ *,
160
+ merge_inline_system: bool = False,
161
) -> ChatCompletionRequest:
162
"""Convert Anthropic message format to OpenAI format"""
163
openai_messages: list[dict[str, Any]] = []
164
131
- cls._convert_system_message(anthropic_request, openai_messages)
132
- cls._convert_messages(anthropic_request.messages, openai_messages)
165
+ cls._convert_system_message(
166
+ anthropic_request,
167
+ openai_messages,
168
+ merge_inline_system=merge_inline_system,
169
+ )
170
+ cls._convert_messages(
171
+ anthropic_request.messages,
172
+ openai_messages,
173
+ merge_inline_system=merge_inline_system,
174
+ )
175
req = cls._build_base_request(anthropic_request, openai_messages)
176
cls._handle_streaming_options(req, anthropic_request)
177
cls._handle_output_config(req, anthropic_request)
@@ -142,6 +184,8 @@ class AnthropicServingMessages(OpenAIServingChat):
184
cls,
185
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
186
openai_messages: list[dict[str, Any]],
187
+ *,
188
+ merge_inline_system: bool = False,
189
) -> None:
190
"""Convert Anthropic system message to OpenAI format"""
191
system_parts: list[str] = []
@@ -159,6 +203,17 @@ class AnthropicServingMessages(OpenAIServingChat):
203
continue
204
system_parts.append(block.text)
205
206
+ # When the template requires system-first ordering, extract inline
207
+ # system messages from the messages array and merge them into the
208
+ # top-level block so the template doesn't reject them.
209
+ if merge_inline_system:
210
+ for msg in anthropic_request.messages:
211
+ if msg.role != "system":
212
+ continue
213
+ text = cls._extract_system_text(msg)
214
+ if text:
215
+ system_parts.append(text)
216
+
217
if system_parts:
218
openai_messages.append({"role": "system", "content": "".join(system_parts)})
219
@@ -180,7 +235,11 @@ class AnthropicServingMessages(OpenAIServingChat):
235
236
@classmethod
237
def _convert_messages(
183
- cls, messages: list, openai_messages: list[dict[str, Any]]
238
+ cls,
239
+ messages: list,
240
+ openai_messages: list[dict[str, Any]],
241
+ *,
242
+ merge_inline_system: bool = False,
243
) -> None:
244
"""Convert Anthropic messages to OpenAI format"""
245
for msg in messages:
@@ -190,6 +249,8 @@ class AnthropicServingMessages(OpenAIServingChat):
249
# doesn't strip billing headers and may produce messages with
250
# no "content" key.
251
if msg.role == "system":
252
+ if merge_inline_system:
253
+ continue # already merged into top-level by _convert_system_message
254
text = cls._extract_system_text(msg)
255
if text:
256
openai_messages.append({"role": "system", "content": text})
@@ -497,7 +558,10 @@ class AnthropicServingMessages(OpenAIServingChat):
558
"""
559
if logger.isEnabledFor(logging.DEBUG):
560
logger.debug("Received messages request %s", request.model_dump_json())
500
- chat_req = self._convert_anthropic_to_openai_request(request)
561
+ chat_req = self._convert_anthropic_to_openai_request(
562
+ request,
563
+ merge_inline_system=self._merge_inline_system,
564
+ )
565
if logger.isEnabledFor(logging.DEBUG):
566
logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json())
567
generator = await self.create_chat_completion(chat_req, raw_request)
@@ -905,7 +969,10 @@ class AnthropicServingMessages(OpenAIServingChat):
969
raw_request: Request | None = None,
970
) -> AnthropicCountTokensResponse | ErrorResponse:
971
"""Implements Anthropic's messages.count_tokens endpoint."""
908
- chat_req = self._convert_anthropic_to_openai_request(request)
972
+ chat_req = self._convert_anthropic_to_openai_request(
973
+ request,
974
+ merge_inline_system=self._merge_inline_system,
975
+ )
976
result = await self.render_chat_request(chat_req)
977
if isinstance(result, ErrorResponse):
978
return result