A high-throughput and memory-efficient inference and serving engine for LLMs
fix(anthropic): preserve inline system message position for prefix caching (#44602)
Signed-off-by: felix0080 <felix0080@users.noreply.github.com> Co-authored-by: felix0080 <felix0080@users.noreply.github.com>
MrFan committed
Jun 18, 2026 at 15:58 UTC
1e9f04da14a6fe349c828928c0f94cf4fcce5363
2 files changed
+103
-28
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+79
-15
@@ -655,10 +655,11 @@ class TestThinkingBlockConversion:
655
656
class TestInlineSystemMessageInMessagesArray:
657
"""Verify that ``role: system`` messages embedded inside the ``messages``
658
- array are accepted and merged with the top-level ``system`` prompt.
658
+ array are preserved in their original position.
659
660
- This handles clients that place system messages inside the messages array
661
- instead of the Anthropic-standard top-level ``system`` field.
660
+ Unlike the previous approach that merged all system messages into a single
661
+ leading system message (breaking prefix caching), this preserves the
662
+ conversation structure so KV-cache hits remain intact.
663
"""
664
665
def test_inline_system_merged_with_top_level_system(self):
@@ -706,17 +707,15 @@ class TestInlineSystemMessageInMessagesArray:
707
708
result = _convert(request)
709
709
- # First message should be the merged system prompt.
710
+ # First message: top-level system prompt (billing header stripped).
711
assert result.messages[0]["role"] == "system"
711
- # Billing header stripped, inline system appended.
712
assert (
713
result.messages[0]["content"]
714
== "You are Claude Code, Anthropic's official CLI for Claude."
715
"...."
716
- "....."
716
)
717
719
- # Second message should be the user message, content preserved.
718
+ # Second message: user message, content preserved at original position.
719
assert result.messages[1]["role"] == "user"
720
user_content = result.messages[1]["content"]
721
assert len(user_content) == 2
@@ -729,6 +728,11 @@ class TestInlineSystemMessageInMessagesArray:
728
"text": "help?",
729
}
730
731
+ # Third message: inline system stays in original position
732
+ # (after user, not merged into leading system).
733
+ assert result.messages[2]["role"] == "system"
734
+ assert result.messages[2]["content"] == "....."
735
+
736
def test_inline_system_string_only(self):
737
"""Only an inline system string, no top-level system."""
738
request = _make_request(
@@ -739,9 +743,11 @@ class TestInlineSystemMessageInMessagesArray:
743
)
744
result = _convert(request)
745
742
- assert result.messages[0]["role"] == "system"
743
- assert result.messages[0]["content"] == "Be concise."
744
- assert result.messages[1]["role"] == "user"
746
+ # Inline system stays in its original position.
747
+ assert result.messages[0]["role"] == "user"
748
+ assert result.messages[0]["content"] == "Hello"
749
+ assert result.messages[1]["role"] == "system"
750
+ assert result.messages[1]["content"] == "Be concise."
751
752
def test_inline_system_list_content(self):
753
"""Inline system with list content blocks."""
@@ -759,11 +765,15 @@ class TestInlineSystemMessageInMessagesArray:
765
)
766
result = _convert(request)
767
762
- assert result.messages[0]["role"] == "system"
763
- assert result.messages[0]["content"] == "Part one. Part two."
768
+ # Inline system stays in its original position;
769
+ # text blocks are concatenated (same as top-level system).
770
+ assert result.messages[0]["role"] == "user"
771
+ assert result.messages[0]["content"] == "Hi"
772
+ assert result.messages[1]["role"] == "system"
773
+ assert result.messages[1]["content"] == "Part one. Part two."
774
775
def test_multiple_inline_system_messages(self):
766
- """Multiple inline system messages should all be merged."""
776
+ """Multiple inline system messages each stay in their position."""
777
request = _make_request(
778
[
779
{"role": "system", "content": "First system."},
@@ -773,9 +783,13 @@ class TestInlineSystemMessageInMessagesArray:
783
)
784
result = _convert(request)
785
786
+ # Each system message stays in its original position.
787
assert result.messages[0]["role"] == "system"
777
- assert result.messages[0]["content"] == "First system.Second system."
788
+ assert result.messages[0]["content"] == "First system."
789
assert result.messages[1]["role"] == "user"
790
+ assert result.messages[1]["content"] == "Hello"
791
+ assert result.messages[2]["role"] == "system"
792
+ assert result.messages[2]["content"] == "Second system."
793
794
def test_inline_system_with_top_level_string(self):
795
"""Top-level system is a string, inline system is also present."""
@@ -788,9 +802,59 @@ class TestInlineSystemMessageInMessagesArray:
802
)
803
result = _convert(request)
804
805
+ # Top-level system goes first; inline system stays in position.
806
assert result.messages[0]["role"] == "system"
792
- assert result.messages[0]["content"] == "Top-level prompt.Inline hint."
807
+ assert result.messages[0]["content"] == "Top-level prompt."
808
assert result.messages[1]["role"] == "user"
809
+ assert result.messages[1]["content"] == "Hello"
810
+ assert result.messages[2]["role"] == "system"
811
+ assert result.messages[2]["content"] == "Inline hint."
812
+
813
+ def test_inline_system_billing_header_stripped(self):
814
+ """Inline system that is only a billing header is omitted."""
815
+ request = _make_request(
816
+ [
817
+ {"role": "user", "content": "Hello"},
818
+ {
819
+ "role": "system",
820
+ "content": "x-anthropic-billing-header: cc_version=2.1.160",
821
+ },
822
+ {"role": "assistant", "content": "Hi there"},
823
+ ]
824
+ )
825
+ result = _convert(request)
826
+
827
+ # Billing-header-only system message should be dropped entirely.
828
+ assert len(result.messages) == 2
829
+ assert result.messages[0]["role"] == "user"
830
+ assert result.messages[1]["role"] == "assistant"
831
+
832
+ def test_inline_system_billing_header_mixed_with_content(self):
833
+ """Inline system with billing header block + real content."""
834
+ request = _make_request(
835
+ [
836
+ {"role": "user", "content": "Hello"},
837
+ {
838
+ "role": "system",
839
+ "content": [
840
+ {
841
+ "type": "text",
842
+ "text": "x-anthropic-billing-header: "
843
+ "cc_version=2.1.160.bca; cch=d1d48;",
844
+ },
845
+ {"type": "text", "text": "Real system content."},
846
+ ],
847
+ },
848
+ ]
849
+ )
850
+ result = _convert(request)
851
+
852
+ # Billing header stripped, real content preserved in position.
853
+ assert len(result.messages) == 2
854
+ assert result.messages[0]["role"] == "user"
855
+ assert result.messages[0]["content"] == "Hello"
856
+ assert result.messages[1]["role"] == "system"
857
+ assert result.messages[1]["content"] == "Real system content."
858
859
860
# ======================================================================
vllm/entrypoints/anthropic/serving.py
+24
-13
@@ -159,29 +159,40 @@ class AnthropicServingMessages(OpenAIServingChat):
159
continue
160
system_parts.append(block.text)
161
162
- # System messages embedded inside the messages array
163
- for msg in anthropic_request.messages:
164
- if msg.role != "system":
165
- continue
166
- if isinstance(msg.content, str):
167
- system_parts.append(msg.content)
168
- else:
169
- for block in msg.content:
170
- if block.type == "text" and block.text:
171
- if block.text.startswith("x-anthropic-billing-header"):
172
- continue
173
- system_parts.append(block.text)
174
-
162
if system_parts:
163
openai_messages.append({"role": "system", "content": "".join(system_parts)})
164
165
+ @classmethod
166
+ def _extract_system_text(cls, msg) -> str | None:
167
+ """Extract text from a system message, stripping billing headers."""
168
+ if isinstance(msg.content, str):
169
+ text = msg.content
170
+ if text.startswith("x-anthropic-billing-header"):
171
+ return None
172
+ return text
173
+ parts: list[str] = []
174
+ for block in msg.content:
175
+ if block.type == "text" and block.text:
176
+ if block.text.startswith("x-anthropic-billing-header"):
177
+ continue
178
+ parts.append(block.text)
179
+ return "".join(parts) if parts else None
180
+
181
@classmethod
182
def _convert_messages(
183
cls, messages: list, openai_messages: list[dict[str, Any]]
184
) -> None:
185
"""Convert Anthropic messages to OpenAI format"""
186
for msg in messages:
187
+ # Handle system messages in-place: extract text, strip billing
188
+ # headers, and only emit if there is real content. This avoids
189
+ # going through _convert_block / _convert_message_content which
190
+ # doesn't strip billing headers and may produce messages with
191
+ # no "content" key.
192
if msg.role == "system":
193
+ text = cls._extract_system_text(msg)
194
+ if text:
195
+ openai_messages.append({"role": "system", "content": text})
196
continue
197
198
openai_msg: dict[str, Any] = {"role": msg.role} # type: ignore