[Frontend] Report cache usage in Anthropic /v1/messages API (#40912)
Signed-off-by: mistral0105 <zhangshuoming17@mails.ucas.ac.cn> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
shuoming zhang committed
Jun 21, 2026 at 05:12 UTC
891cc4b9c58fa0ab7e4b29ee1df90724647229fd
2 files changed
+295
-10
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+239
-1
@@ -8,6 +8,8 @@ AnthropicServingMessages._convert_anthropic_to_openai_request().
8
Also covers extended-thinking edge cases such as ``redacted_thinking``
9
blocks echoed back by Anthropic clients, and streaming conversion in
10
``message_stream_converter``.
11
+
12
+Also covers cache usage computation in ``_build_anthropic_usage``.
13
"""
14
15
import json
@@ -18,7 +20,11 @@ import pytest
20
from vllm.entrypoints.anthropic.protocol import (
21
AnthropicMessagesRequest,
22
)
21
-from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
23
+from vllm.entrypoints.anthropic.serving import (
24
+ AnthropicServingMessages,
25
+ _build_anthropic_usage,
26
+ _get_cached_tokens,
27
+)
28
from vllm.entrypoints.openai.chat_completion.protocol import (
29
ChatCompletionResponseStreamChoice,
30
ChatCompletionStreamResponse,
@@ -27,6 +33,7 @@ from vllm.entrypoints.openai.engine.protocol import (
33
DeltaFunctionCall,
34
DeltaMessage,
35
DeltaToolCall,
36
+ PromptTokenUsageInfo,
37
UsageInfo,
38
)
39
@@ -653,6 +660,108 @@ class TestThinkingBlockConversion:
660
assert asst.get("content") == "Hi!"
661
662
663
+# ======================================================================
664
+# Cache usage computation
665
+# ======================================================================
666
+
667
+
668
+class TestGetCachedTokens:
669
+ """Tests for _get_cached_tokens helper."""
670
+
671
+ def test_none_usage(self):
672
+ assert _get_cached_tokens(None) is None
673
+
674
+ def test_no_prompt_tokens_details(self):
675
+ usage = UsageInfo(prompt_tokens=100, completion_tokens=10)
676
+ assert _get_cached_tokens(usage) is None
677
+
678
+ def test_cached_tokens_present(self):
679
+ usage = UsageInfo(
680
+ prompt_tokens=100,
681
+ completion_tokens=10,
682
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
683
+ )
684
+ assert _get_cached_tokens(usage) == 80
685
+
686
+ def test_cached_tokens_zero(self):
687
+ """Zero cached tokens should return 0, not None."""
688
+ usage = UsageInfo(
689
+ prompt_tokens=100,
690
+ completion_tokens=10,
691
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
692
+ )
693
+ assert _get_cached_tokens(usage) == 0
694
+
695
+ def test_cached_tokens_none_in_details(self):
696
+ usage = UsageInfo(
697
+ prompt_tokens=100,
698
+ completion_tokens=10,
699
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=None),
700
+ )
701
+ assert _get_cached_tokens(usage) is None
702
+
703
+
704
+class TestBuildAnthropicUsage:
705
+ """Tests for _build_anthropic_usage helper.
706
+
707
+ Anthropic defines: total_input = input_tokens + cache_read + cache_creation
708
+ vLLM's prompt_tokens is the total.
709
+ """
710
+
711
+ def test_no_cache_info(self):
712
+ """When cache info is unavailable, return raw prompt_tokens."""
713
+ result = _build_anthropic_usage(100, 10, None)
714
+ assert result.input_tokens == 100
715
+ assert result.output_tokens == 10
716
+ assert result.cache_read_input_tokens is None
717
+ assert result.cache_creation_input_tokens is None
718
+
719
+ def test_cache_hit(self):
720
+ """When cache is hit, input_tokens excludes cached tokens."""
721
+ usage = UsageInfo(
722
+ prompt_tokens=100,
723
+ completion_tokens=10,
724
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
725
+ )
726
+ result = _build_anthropic_usage(100, 10, usage)
727
+ assert result.input_tokens == 20 # 100 - 80
728
+ assert result.output_tokens == 10
729
+ assert result.cache_read_input_tokens == 80
730
+ assert result.cache_creation_input_tokens == 0
731
+
732
+ def test_zero_cached_tokens(self):
733
+ """Zero cached tokens should still set cache_creation to 0."""
734
+ usage = UsageInfo(
735
+ prompt_tokens=100,
736
+ completion_tokens=10,
737
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
738
+ )
739
+ result = _build_anthropic_usage(100, 10, usage)
740
+ assert result.input_tokens == 100 # 100 - 0
741
+ assert result.cache_read_input_tokens == 0
742
+ assert result.cache_creation_input_tokens == 0
743
+
744
+ def test_all_tokens_cached(self):
745
+ """When all tokens are cached, input_tokens should be 0."""
746
+ usage = UsageInfo(
747
+ prompt_tokens=100,
748
+ completion_tokens=10,
749
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=100),
750
+ )
751
+ result = _build_anthropic_usage(100, 10, usage)
752
+ assert result.input_tokens == 0
753
+ assert result.cache_read_input_tokens == 100
754
+ assert result.cache_creation_input_tokens == 0
755
+
756
+ def test_no_prompt_tokens_details(self):
757
+ """UsageInfo without prompt_tokens_details returns no cache info."""
758
+ usage = UsageInfo(prompt_tokens=100, completion_tokens=10)
759
+ result = _build_anthropic_usage(100, 10, usage)
760
+ assert result.input_tokens == 100
761
+ assert result.cache_read_input_tokens is None
762
+ assert result.cache_creation_input_tokens is None
763
+
764
+
765
class TestInlineSystemMessageInMessagesArray:
766
"""Verify that ``role: system`` messages embedded inside the ``messages``
767
array are preserved in their original position.
@@ -1098,6 +1207,135 @@ class TestMessageStartIncludesTypeAndRole:
1207
assert message["role"] == "assistant"
1208
1209
1210
+class TestStreamingCacheUsageSemantics:
1211
+ """Locks in the documented streaming behavior of cache usage fields.
1212
+
1213
+ vLLM's OpenAI chat completion streaming only attaches
1214
+ ``prompt_tokens_details`` to the terminal usage chunk. The Anthropic layer
1215
+ mirrors that contract: cache fields are omitted on ``message_start`` (key
1216
+ absence signals "unknown") and populated on ``message_delta`` (the final
1217
+ cumulative count). This is intentionally consistent with vLLM's OpenAI
1218
+ behavior, even though Anthropic's upstream API populates cache fields on
1219
+ ``message_start``; closing that gap requires plumbing cache info into the
1220
+ first chunk at the OpenAI layer, which is out of scope here.
1221
+ """
1222
+
1223
+ @pytest.mark.asyncio
1224
+ async def test_streaming_cache_fields_absent_then_populated(self):
1225
+ """First chunk lacks prompt_tokens_details (vLLM contract);
1226
+ message_start omits cache fields. The final chunk carries
1227
+ prompt_tokens_details, so message_delta carries resolved values."""
1228
+
1229
+ async def sse_input():
1230
+ yield _make_stream_chunk(
1231
+ delta=DeltaMessage(role="assistant", content="hi"),
1232
+ usage=UsageInfo(prompt_tokens=100, total_tokens=100),
1233
+ )
1234
+ yield _make_stream_chunk(finish_reason="stop")
1235
+ yield _make_stream_chunk(
1236
+ choices=[],
1237
+ usage=UsageInfo(
1238
+ prompt_tokens=100,
1239
+ completion_tokens=5,
1240
+ total_tokens=105,
1241
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80),
1242
+ ),
1243
+ )
1244
+ yield "data: [DONE]"
1245
+
1246
+ converter = _make_stream_converter()
1247
+ output = []
1248
+ async for event in converter.message_stream_converter(sse_input()):
1249
+ output.append(event)
1250
+ events = _parse_sse_events(output)
1251
+
1252
+ # message_start: cache fields unknown → omitted from JSON entirely.
1253
+ start_usage = events[0][1]["message"]["usage"]
1254
+ assert events[0][0] == "message_start"
1255
+ assert start_usage["input_tokens"] == 100
1256
+ assert "cache_read_input_tokens" not in start_usage
1257
+ assert "cache_creation_input_tokens" not in start_usage
1258
+
1259
+ # message_delta: authoritative usage with cache fields populated.
1260
+ delta_usage = next(
1261
+ data["usage"] for ev, data in events if ev == "message_delta"
1262
+ )
1263
+ assert delta_usage["input_tokens"] == 20 # 100 - 80
1264
+ assert delta_usage["cache_read_input_tokens"] == 80
1265
+ assert delta_usage["cache_creation_input_tokens"] == 0
1266
+
1267
+ @pytest.mark.asyncio
1268
+ async def test_streaming_no_cache_hit(self):
1269
+ """When the final chunk reports cached_tokens=0, message_delta carries
1270
+ cache fields = 0 (cache miss); message_start still omits them."""
1271
+
1272
+ async def sse_input():
1273
+ yield _make_stream_chunk(
1274
+ delta=DeltaMessage(role="assistant"),
1275
+ usage=UsageInfo(prompt_tokens=50, total_tokens=50),
1276
+ )
1277
+ yield _make_stream_chunk(finish_reason="stop")
1278
+ yield _make_stream_chunk(
1279
+ choices=[],
1280
+ usage=UsageInfo(
1281
+ prompt_tokens=50,
1282
+ completion_tokens=5,
1283
+ total_tokens=55,
1284
+ prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0),
1285
+ ),
1286
+ )
1287
+ yield "data: [DONE]"
1288
+
1289
+ converter = _make_stream_converter()
1290
+ output = []
1291
+ async for event in converter.message_stream_converter(sse_input()):
1292
+ output.append(event)
1293
+ events = _parse_sse_events(output)
1294
+
1295
+ start_usage = events[0][1]["message"]["usage"]
1296
+ delta_usage = next(
1297
+ data["usage"] for ev, data in events if ev == "message_delta"
1298
+ )
1299
+ assert start_usage["input_tokens"] == 50
1300
+ assert "cache_read_input_tokens" not in start_usage
1301
+ assert "cache_creation_input_tokens" not in start_usage
1302
+ assert delta_usage["input_tokens"] == 50 # 50 - 0
1303
+ assert delta_usage["cache_read_input_tokens"] == 0
1304
+ assert delta_usage["cache_creation_input_tokens"] == 0
1305
+
1306
+ @pytest.mark.asyncio
1307
+ async def test_streaming_no_prompt_tokens_details_at_all(self):
1308
+ """If --enable-prompt-tokens-details is off, no chunk carries cache
1309
+ info; both message_start and message_delta omit cache fields."""
1310
+
1311
+ async def sse_input():
1312
+ yield _make_stream_chunk(
1313
+ delta=DeltaMessage(role="assistant"),
1314
+ usage=UsageInfo(prompt_tokens=30, total_tokens=30),
1315
+ )
1316
+ yield _make_stream_chunk(finish_reason="stop")
1317
+ yield _make_stream_chunk(
1318
+ choices=[],
1319
+ usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32),
1320
+ )
1321
+ yield "data: [DONE]"
1322
+
1323
+ converter = _make_stream_converter()
1324
+ output = []
1325
+ async for event in converter.message_stream_converter(sse_input()):
1326
+ output.append(event)
1327
+ events = _parse_sse_events(output)
1328
+
1329
+ start_usage = events[0][1]["message"]["usage"]
1330
+ delta_usage = next(
1331
+ data["usage"] for ev, data in events if ev == "message_delta"
1332
+ )
1333
+ assert "cache_read_input_tokens" not in start_usage
1334
+ assert "cache_creation_input_tokens" not in start_usage
1335
+ assert "cache_read_input_tokens" not in delta_usage
1336
+ assert "cache_creation_input_tokens" not in delta_usage
1337
+
1338
+
1339
# ======================================================================
1340
# Auto-detection of system-first template requirement
1341
# ======================================================================
vllm/entrypoints/anthropic/serving.py
+56
-9
@@ -43,6 +43,7 @@ from vllm.entrypoints.openai.engine.protocol import (
43
JsonSchemaResponseFormat,
44
ResponseFormat,
45
StreamOptions,
46
+ UsageInfo,
47
)
48
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
49
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
@@ -54,6 +55,49 @@ if TYPE_CHECKING:
55
logger = logging.getLogger(__name__)
56
57
58
+def _get_cached_tokens(usage: UsageInfo | None) -> int | None:
59
+ """Extract cached token count from OpenAI UsageInfo."""
60
+ if usage is None or usage.prompt_tokens_details is None:
61
+ return None
62
+ return usage.prompt_tokens_details.cached_tokens
63
+
64
+
65
+def _build_anthropic_usage(
66
+ prompt_tokens: int,
67
+ completion_tokens: int | None,
68
+ usage: UsageInfo | None,
69
+) -> AnthropicUsage:
70
+ """Build an AnthropicUsage from OpenAI-style token counts.
71
+
72
+ Anthropic defines ``total_input == input_tokens + cache_read +
73
+ cache_creation``. vLLM's ``prompt_tokens`` is the total, so
74
+ ``input_tokens = prompt_tokens - cached_tokens``.
75
+
76
+ OpenAI usage only exposes ``cached_tokens`` (hits); there is no
77
+ cache-creation analog, so ``cache_creation_input_tokens`` is ``0``
78
+ when cache info is present. When cache info is absent (e.g.
79
+ ``--enable-prompt-tokens-details`` off, or a streaming chunk that
80
+ hasn't carried it yet), cache fields are left **unset** so
81
+ ``exclude_unset=True`` serialization omits them entirely.
82
+
83
+ ``completion_tokens`` follows ``UsageInfo`` and may be ``None`` on
84
+ intermediate stream chunks; we coerce to ``0`` for the wire format.
85
+ """
86
+ output_tokens = completion_tokens or 0
87
+ cached = _get_cached_tokens(usage)
88
+ if cached is not None:
89
+ return AnthropicUsage(
90
+ input_tokens=prompt_tokens - cached,
91
+ output_tokens=output_tokens,
92
+ cache_read_input_tokens=cached,
93
+ cache_creation_input_tokens=0,
94
+ )
95
+ return AnthropicUsage(
96
+ input_tokens=prompt_tokens,
97
+ output_tokens=output_tokens,
98
+ )
99
+
100
+
101
def wrap_data_with_event(data: str, event: str):
102
return f"event: {event}\ndata: {data}\n\n"
103
@@ -582,9 +626,10 @@ class AnthropicServingMessages(OpenAIServingChat):
626
id=generator.id,
627
content=[],
628
model=generator.model,
585
- usage=AnthropicUsage(
586
- input_tokens=generator.usage.prompt_tokens,
587
- output_tokens=generator.usage.completion_tokens,
629
+ usage=_build_anthropic_usage(
630
+ generator.usage.prompt_tokens,
631
+ generator.usage.completion_tokens,
632
+ generator.usage,
633
),
634
kv_transfer_params=generator.kv_transfer_params,
635
)
@@ -765,11 +810,12 @@ class AnthropicServingMessages(OpenAIServingChat):
810
model=origin_chunk.model,
811
stop_reason=None,
812
stop_sequence=None,
768
- usage=AnthropicUsage(
769
- input_tokens=origin_chunk.usage.prompt_tokens
813
+ usage=_build_anthropic_usage(
814
+ origin_chunk.usage.prompt_tokens
815
if origin_chunk.usage
816
else 0,
772
- output_tokens=0,
817
+ 0,
818
+ origin_chunk.usage,
819
),
820
),
821
)
@@ -788,13 +834,14 @@ class AnthropicServingMessages(OpenAIServingChat):
834
chunk = AnthropicStreamEvent(
835
type="message_delta",
836
delta=AnthropicDelta(stop_reason=stop_reason),
791
- usage=AnthropicUsage(
792
- input_tokens=origin_chunk.usage.prompt_tokens
837
+ usage=_build_anthropic_usage(
838
+ origin_chunk.usage.prompt_tokens
839
if origin_chunk.usage
840
else 0,
795
- output_tokens=origin_chunk.usage.completion_tokens
841
+ origin_chunk.usage.completion_tokens
842
if origin_chunk.usage
843
else 0,
844
+ origin_chunk.usage,
845
),
846
)
847
data = chunk.model_dump_json(exclude_unset=True)