A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 04:45 UTC
[Anthropic] Support system role messages inside messages array (#44283)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Aleksandar Yanakiev <alexander.yanakiev@discretestack.com> Co-authored-by: Ang Kah Min, Kelvin <syraxius@hotmail.com>
Chauncey committed
Jun 3, 2026 at 02:13 UTC
ed9a7526b6deea866c04edcaacbbecadb591b7ef
3 files changed
+173
-17
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+140
@@ -635,3 +635,143 @@ class TestThinkingBlockConversion:
635
# Redacted thinking is ignored, normal thinking still becomes reasoning.
636
assert asst.get("reasoning") == "Thinking..."
637
assert asst.get("content") == "Hi!"
638
+
639
+
640
+class TestInlineSystemMessageInMessagesArray:
641
+ """Verify that ``role: system`` messages embedded inside the ``messages``
642
+ array are accepted and merged with the top-level ``system`` prompt.
643
+
644
+ This handles clients that place system messages inside the messages array
645
+ instead of the Anthropic-standard top-level ``system`` field.
646
+ """
647
+
648
+ def test_inline_system_merged_with_top_level_system(self):
649
+ """Full integration: inline system + top-level system + user message."""
650
+ request = _make_request(
651
+ [
652
+ {
653
+ "role": "user",
654
+ "content": [
655
+ {
656
+ "type": "text",
657
+ "text": "<system-reminder>\n.....\n</system-reminder>\n\n",
658
+ },
659
+ {
660
+ "type": "text",
661
+ "text": "help?",
662
+ "cache_control": {"type": "ephemeral"},
663
+ },
664
+ ],
665
+ },
666
+ {
667
+ "role": "system",
668
+ "content": ".....",
669
+ },
670
+ ],
671
+ system=[
672
+ {
673
+ "type": "text",
674
+ "text": "x-anthropic-billing-header: "
675
+ "cc_version=2.1.160.bca; cc_entrypoint=cli; cch=d1d48;",
676
+ },
677
+ {
678
+ "type": "text",
679
+ "text": "You are Claude Code, Anthropic's official CLI for Claude.",
680
+ "cache_control": {"type": "ephemeral"},
681
+ },
682
+ {
683
+ "type": "text",
684
+ "text": "....",
685
+ "cache_control": {"type": "ephemeral"},
686
+ },
687
+ ],
688
+ tools=[],
689
+ )
690
+
691
+ result = _convert(request)
692
+
693
+ # First message should be the merged system prompt.
694
+ assert result.messages[0]["role"] == "system"
695
+ # Billing header stripped, inline system appended.
696
+ assert (
697
+ result.messages[0]["content"]
698
+ == "You are Claude Code, Anthropic's official CLI for Claude."
699
+ "...."
700
+ "....."
701
+ )
702
+
703
+ # Second message should be the user message, content preserved.
704
+ assert result.messages[1]["role"] == "user"
705
+ user_content = result.messages[1]["content"]
706
+ assert len(user_content) == 2
707
+ assert user_content[0] == {
708
+ "type": "text",
709
+ "text": "<system-reminder>\n.....\n</system-reminder>\n\n",
710
+ }
711
+ assert user_content[1] == {
712
+ "type": "text",
713
+ "text": "help?",
714
+ }
715
+
716
+ def test_inline_system_string_only(self):
717
+ """Only an inline system string, no top-level system."""
718
+ request = _make_request(
719
+ [
720
+ {"role": "user", "content": "Hello"},
721
+ {"role": "system", "content": "Be concise."},
722
+ ]
723
+ )
724
+ result = _convert(request)
725
+
726
+ assert result.messages[0]["role"] == "system"
727
+ assert result.messages[0]["content"] == "Be concise."
728
+ assert result.messages[1]["role"] == "user"
729
+
730
+ def test_inline_system_list_content(self):
731
+ """Inline system with list content blocks."""
732
+ request = _make_request(
733
+ [
734
+ {"role": "user", "content": "Hi"},
735
+ {
736
+ "role": "system",
737
+ "content": [
738
+ {"type": "text", "text": "Part one. "},
739
+ {"type": "text", "text": "Part two."},
740
+ ],
741
+ },
742
+ ]
743
+ )
744
+ result = _convert(request)
745
+
746
+ assert result.messages[0]["role"] == "system"
747
+ assert result.messages[0]["content"] == "Part one. Part two."
748
+
749
+ def test_multiple_inline_system_messages(self):
750
+ """Multiple inline system messages should all be merged."""
751
+ request = _make_request(
752
+ [
753
+ {"role": "system", "content": "First system."},
754
+ {"role": "user", "content": "Hello"},
755
+ {"role": "system", "content": "Second system."},
756
+ ]
757
+ )
758
+ result = _convert(request)
759
+
760
+ assert result.messages[0]["role"] == "system"
761
+ assert result.messages[0]["content"] == "First system.Second system."
762
+ assert result.messages[1]["role"] == "user"
763
+
764
+ def test_inline_system_with_top_level_string(self):
765
+ """Top-level system is a string, inline system is also present."""
766
+ request = _make_request(
767
+ [
768
+ {"role": "user", "content": "Hello"},
769
+ {"role": "system", "content": "Inline hint."},
770
+ ],
771
+ system="Top-level prompt.",
772
+ )
773
+ result = _convert(request)
774
+
775
+ assert result.messages[0]["role"] == "system"
776
+ assert result.messages[0]["content"] == "Top-level prompt.Inline hint."
777
+ assert result.messages[1]["role"] == "user"
vllm/entrypoints/anthropic/protocol.py
+1
-1
@@ -65,7 +65,7 @@ class AnthropicContentBlock(BaseModel):
65
class AnthropicMessage(BaseModel):
66
"""Message structure"""
67
68
- role: Literal["user", "assistant"]
68
+ role: Literal["user", "assistant", "system"]
69
content: str | list[AnthropicContentBlock]
70
71
vllm/entrypoints/anthropic/serving.py
+32
-16
@@ -143,23 +143,36 @@ class AnthropicServingMessages(OpenAIServingChat):
143
openai_messages: list[dict[str, Any]],
144
) -> None:
145
"""Convert Anthropic system message to OpenAI format"""
146
- if not anthropic_request.system:
147
- return
146
+ system_parts: list[str] = []
147
149
- if isinstance(anthropic_request.system, str):
150
- openai_messages.append(
151
- {"role": "system", "content": anthropic_request.system}
152
- )
153
- else:
154
- system_prompt = ""
155
- for block in anthropic_request.system:
156
- if block.type == "text" and block.text:
157
- # Strip Claude Code's attribution header which contains
158
- # a per-request hash that defeats prefix caching.
159
- if block.text.startswith("x-anthropic-billing-header"):
160
- continue
161
- system_prompt += block.text
162
- openai_messages.append({"role": "system", "content": system_prompt})
148
+ # Top-level system field
149
+ if anthropic_request.system:
150
+ if isinstance(anthropic_request.system, str):
151
+ system_parts.append(anthropic_request.system)
152
+ else:
153
+ for block in anthropic_request.system:
154
+ if block.type == "text" and block.text:
155
+ # Strip Claude Code's attribution header which contains
156
+ # a per-request hash that defeats prefix caching.
157
+ if block.text.startswith("x-anthropic-billing-header"):
158
+ continue
159
+ system_parts.append(block.text)
160
+
161
+ # System messages embedded inside the messages array
162
+ for msg in anthropic_request.messages:
163
+ if msg.role != "system":
164
+ continue
165
+ if isinstance(msg.content, str):
166
+ system_parts.append(msg.content)
167
+ else:
168
+ for block in msg.content:
169
+ if block.type == "text" and block.text:
170
+ if block.text.startswith("x-anthropic-billing-header"):
171
+ continue
172
+ system_parts.append(block.text)
173
+
174
+ if system_parts:
175
+ openai_messages.append({"role": "system", "content": "".join(system_parts)})
176
177
@classmethod
178
def _convert_messages(
@@ -167,6 +180,9 @@ class AnthropicServingMessages(OpenAIServingChat):
180
) -> None:
181
"""Convert Anthropic messages to OpenAI format"""
182
for msg in messages:
183
+ if msg.role == "system":
184
+ continue
185
+
186
openai_msg: dict[str, Any] = {"role": msg.role} # type: ignore
187
188
if isinstance(msg.content, str):