A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 02:15 UTC
[Frontend] Fixes anthropic /v1/messages streaming not containing input_tokens on first chunk (#29971)
Signed-off-by: bbartels <benjamin@bartels.dev>
Benjamin Bartels committed
Dec 4, 2025 at 05:50 UTC
fca3f4665838605e268a8408bc7ca359f5d5c14b
2 files changed
+20
-1
tests/entrypoints/openai/test_messages.py
+11
@@ -69,9 +69,20 @@ async def test_anthropic_streaming(client: anthropic.AsyncAnthropic):
69
stream=True,
70
)
71
72
+ first_chunk = None
73
+ chunk_count = 0
74
async for chunk in resp:
75
+ chunk_count += 1
76
+ if first_chunk is None and chunk.type == "message_start":
77
+ first_chunk = chunk
78
print(chunk.model_dump_json())
79
80
+ assert chunk_count > 0
81
+ assert first_chunk is not None, "message_start chunk was never observed"
82
+ assert first_chunk.usage is not None, "first chunk should include usage stats"
83
+ assert first_chunk.usage["output_tokens"] == 0
84
+ assert first_chunk.usage["input_tokens"] > 5
85
+
86
87
@pytest.mark.asyncio
88
async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic):
vllm/entrypoints/anthropic/serving_messages.py
+9
-1
@@ -183,7 +183,9 @@ class AnthropicServingMessages(OpenAIServingChat):
183
184
if anthropic_request.stream:
185
req.stream = anthropic_request.stream
186
- req.stream_options = StreamOptions.validate({"include_usage": True})
186
+ req.stream_options = StreamOptions.validate(
187
+ {"include_usage": True, "continuous_usage_stats": True}
188
+ )
189
190
if anthropic_request.tool_choice is None:
191
req.tool_choice = None
@@ -323,6 +325,12 @@ class AnthropicServingMessages(OpenAIServingChat):
325
content=[],
326
model=origin_chunk.model,
327
),
328
+ usage=AnthropicUsage(
329
+ input_tokens=origin_chunk.usage.prompt_tokens
330
+ if origin_chunk.usage
331
+ else 0,
332
+ output_tokens=0,
333
+ ),
334
)
335
first_item = False
336
data = chunk.model_dump_json(exclude_unset=True)