A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 13:45 UTC
[Frontend] Fixes anthropic streaming message_start usage nesting (#30266)
Signed-off-by: bbartels <benjamin@bartels.dev>
Benjamin Bartels committed
Dec 12, 2025 at 16:28 UTC
f3237f3f6b1ce3ea3b1881a059811c2695ffe650
2 files changed
+12
-9
tests/entrypoints/openai/test_messages.py
+6
-3
@@ -79,9 +79,12 @@ async def test_anthropic_streaming(client: anthropic.AsyncAnthropic):
79
80
assert chunk_count > 0
81
assert first_chunk is not None, "message_start chunk was never observed"
82
- assert first_chunk.usage is not None, "first chunk should include usage stats"
83
- assert first_chunk.usage["output_tokens"] == 0
84
- assert first_chunk.usage["input_tokens"] > 5
82
+ assert first_chunk.message is not None, "first chunk should include message"
83
+ assert first_chunk.message.usage is not None, (
84
+ "first chunk should include usage stats"
85
+ )
86
+ assert first_chunk.message.usage.output_tokens == 0
87
+ assert first_chunk.message.usage.input_tokens > 5
88
89
90
@pytest.mark.asyncio
vllm/entrypoints/anthropic/serving_messages.py
+6
-6
@@ -324,12 +324,12 @@ class AnthropicServingMessages(OpenAIServingChat):
324
id=origin_chunk.id,
325
content=[],
326
model=origin_chunk.model,
327
- ),
328
- usage=AnthropicUsage(
329
- input_tokens=origin_chunk.usage.prompt_tokens
330
- if origin_chunk.usage
331
- else 0,
332
- output_tokens=0,
327
+ usage=AnthropicUsage(
328
+ input_tokens=origin_chunk.usage.prompt_tokens
329
+ if origin_chunk.usage
330
+ else 0,
331
+ output_tokens=0,
332
+ ),
333
),
334
)
335
first_item = False