A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[Frontend] Exclude anthropic billing header to avoid prefix cache miss (#36829)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Nick Hill committed
Mar 11, 2026 at 18:20 UTC
262b76a09fafe15cff7642f3eee433fb903cf1d8
3 files changed
+56
docs/serving/integrations/claude_code.md
+3
@@ -60,6 +60,9 @@ The environment variables:
60
!!! tip
61
You can add these environment variables to your shell profile (e.g., `.bashrc`, `.zshrc`), Claude Code configuration file (`~/.claude/settings.json`), or create a wrapper script for convenience.
62
63
+!!! warning
64
+ Claude Code recently started injecting a per-request hash in the system prompt, which can defeat [prefix caching](../../design/prefix_caching.md) because the prompt changes on every request, causing greatly reduced performance. This is addressed automatically in vLLM versions > 0.17.1 but for older versions `"CLAUDE_CODE_ATTRIBUTION_HEADER": "0"` should be added to the `"env"` section of `~/.claude/settings.json` (see this [blog post](https://unsloth.ai/docs/basics/claude-code#fixing-90-slower-inference-in-claude-code) from Unsloth).
65
+
66
## Testing the Setup
67
68
Once Claude Code launches, try a simple prompt to verify the connection:
tests/entrypoints/openai/test_anthropic_messages_conversion.py
+49
@@ -324,3 +324,52 @@ class TestToolResultContent:
324
if m["role"] == "user" and isinstance(m.get("content"), list)
325
]
326
assert len(user_follow_ups) == 0
327
+
328
+
329
+# ======================================================================
330
+# Attribution header stripping
331
+# ======================================================================
332
+
333
+
334
+class TestAttributionHeaderStripping:
335
+ def test_billing_header_stripped_from_system(self):
336
+ """Claude Code's x-anthropic-billing-header block should be
337
+ stripped to preserve prefix caching."""
338
+ request = _make_request(
339
+ [{"role": "user", "content": "Hello"}],
340
+ system=[
341
+ {"type": "text", "text": "You are a helpful assistant."},
342
+ {
343
+ "type": "text",
344
+ "text": "x-anthropic-billing-header: "
345
+ "cc_version=2.1.37.abc; cc_entrypoint=cli;",
346
+ },
347
+ ],
348
+ )
349
+ result = _convert(request)
350
+ system_msg = result.messages[0]
351
+ assert system_msg["role"] == "system"
352
+ assert system_msg["content"] == "You are a helpful assistant."
353
+
354
+ def test_system_without_billing_header_unchanged(self):
355
+ """Normal system blocks should pass through unchanged."""
356
+ request = _make_request(
357
+ [{"role": "user", "content": "Hello"}],
358
+ system=[
359
+ {"type": "text", "text": "You are a helpful assistant."},
360
+ {"type": "text", "text": " Be concise."},
361
+ ],
362
+ )
363
+ result = _convert(request)
364
+ system_msg = result.messages[0]
365
+ assert system_msg["content"] == "You are a helpful assistant. Be concise."
366
+
367
+ def test_system_string_unchanged(self):
368
+ """String system prompts should pass through unchanged."""
369
+ request = _make_request(
370
+ [{"role": "user", "content": "Hello"}],
371
+ system="You are a helpful assistant.",
372
+ )
373
+ result = _convert(request)
374
+ system_msg = result.messages[0]
375
+ assert system_msg["content"] == "You are a helpful assistant."
vllm/entrypoints/anthropic/serving.py
+4
@@ -143,6 +143,10 @@ class AnthropicServingMessages(OpenAIServingChat):
143
system_prompt = ""
144
for block in anthropic_request.system:
145
if block.type == "text" and block.text:
146
+ # Strip Claude Code's attribution header which contains
147
+ # a per-request hash that defeats prefix caching.
148
+ if block.text.startswith("x-anthropic-billing-header"):
149
+ continue
150
system_prompt += block.text
151
openai_messages.append({"role": "system", "content": system_prompt})
152