A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Fix Anthropic tool_use content handling dropping args (#45287)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
Jun 11, 2026 at 16:01 UTC
235b63c0046d2fbf4ab1bf810a1eb729f1f3fc27
2 files changed
+252
-5
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+222
-1
@@ -6,13 +6,29 @@ Tests the image source handling and tool_result content parsing in
6
AnthropicServingMessages._convert_anthropic_to_openai_request().
7
8
Also covers extended-thinking edge cases such as ``redacted_thinking``
9
-blocks echoed back by Anthropic clients.
9
+blocks echoed back by Anthropic clients, and streaming conversion in
10
+``message_stream_converter``.
11
"""
12
13
+import json
14
+from unittest.mock import MagicMock
15
+
16
+import pytest
17
+
18
from vllm.entrypoints.anthropic.protocol import (
19
AnthropicMessagesRequest,
20
)
21
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
22
+from vllm.entrypoints.openai.chat_completion.protocol import (
23
+ ChatCompletionResponseStreamChoice,
24
+ ChatCompletionStreamResponse,
25
+)
26
+from vllm.entrypoints.openai.engine.protocol import (
27
+ DeltaFunctionCall,
28
+ DeltaMessage,
29
+ DeltaToolCall,
30
+ UsageInfo,
31
+)
32
33
_convert = AnthropicServingMessages._convert_anthropic_to_openai_request
34
_img_url = AnthropicServingMessages._convert_image_source_to_url
@@ -775,3 +791,208 @@ class TestInlineSystemMessageInMessagesArray:
791
assert result.messages[0]["role"] == "system"
792
assert result.messages[0]["content"] == "Top-level prompt.Inline hint."
793
assert result.messages[1]["role"] == "user"
794
+
795
+
796
+# ======================================================================
797
+# Streaming conversion: message_stream_converter
798
+# ======================================================================
799
+
800
+
801
+def _make_stream_converter():
802
+ obj = MagicMock(spec=AnthropicServingMessages)
803
+ obj.stop_reason_map = {
804
+ "stop": "end_turn",
805
+ "length": "max_tokens",
806
+ "tool_calls": "tool_use",
807
+ }
808
+ obj.message_stream_converter = (
809
+ AnthropicServingMessages.message_stream_converter.__get__(obj)
810
+ )
811
+ return obj
812
+
813
+
814
+def _parse_sse_events(raw_events: list[str]) -> list[tuple[str, dict]]:
815
+ results = []
816
+ for raw in raw_events:
817
+ headers = dict(
818
+ line.split(": ", 1) for line in raw.strip().split("\n") if ": " in line
819
+ )
820
+ if "event" in headers and "data" in headers:
821
+ results.append((headers["event"], json.loads(headers["data"])))
822
+ return results
823
+
824
+
825
+def _make_stream_chunk(
826
+ *,
827
+ delta: DeltaMessage | None = None,
828
+ finish_reason: str | None = None,
829
+ choices: list[ChatCompletionResponseStreamChoice] | None = None,
830
+ usage: UsageInfo | None = None,
831
+) -> str:
832
+ if choices is None:
833
+ choices = [
834
+ ChatCompletionResponseStreamChoice(
835
+ index=0,
836
+ delta=delta or DeltaMessage(),
837
+ finish_reason=finish_reason,
838
+ )
839
+ ]
840
+ chunk = ChatCompletionStreamResponse(
841
+ id="chatcmpl-test",
842
+ created=0,
843
+ model="test-model",
844
+ choices=choices,
845
+ usage=usage,
846
+ )
847
+ return f"data: {chunk.model_dump_json()}"
848
+
849
+
850
+def _tc(*, args, id=None, name=None):
851
+ return DeltaToolCall(
852
+ index=0,
853
+ id=id,
854
+ function=DeltaFunctionCall(name=name, arguments=args),
855
+ )
856
+
857
+
858
+class TestMessageStreamConverterToolUseContentBuffering:
859
+ """Regression test for tool_use arguments being silently dropped.
860
+
861
+ With speculative decoding or multi-token prediction, a single delta
862
+ can carry both the final tool_call argument fragment and trailing
863
+ content.
864
+ """
865
+
866
+ @pytest.mark.asyncio
867
+ async def test_tool_use_args_not_dropped_when_content_in_same_chunk(
868
+ self,
869
+ ):
870
+ async def sse_input():
871
+ yield _make_stream_chunk(
872
+ delta=DeltaMessage(role="assistant"),
873
+ usage=UsageInfo(prompt_tokens=10, total_tokens=10),
874
+ )
875
+ yield _make_stream_chunk(
876
+ delta=DeltaMessage(
877
+ tool_calls=[
878
+ _tc(id="call_abc123", name="read_file", args=""),
879
+ ]
880
+ )
881
+ )
882
+ yield _make_stream_chunk(
883
+ delta=DeltaMessage(
884
+ tool_calls=[
885
+ _tc(args='{"path":"/tmp/f"'),
886
+ ]
887
+ )
888
+ )
889
+ # BUG TRIGGER: final tool_call args and trailing content in
890
+ # one delta, as happens with spec decoding / multi-token
891
+ # prediction where multiple tokens land in a single chunk.
892
+ yield _make_stream_chunk(
893
+ delta=DeltaMessage(
894
+ content="\nOkay",
895
+ tool_calls=[_tc(args="}")],
896
+ )
897
+ )
898
+ yield _make_stream_chunk(finish_reason="tool_calls")
899
+ yield _make_stream_chunk(
900
+ choices=[],
901
+ usage=UsageInfo(
902
+ prompt_tokens=10,
903
+ total_tokens=30,
904
+ completion_tokens=20,
905
+ ),
906
+ )
907
+ yield "data: [DONE]"
908
+
909
+ converter = _make_stream_converter()
910
+ output = []
911
+ async for event in converter.message_stream_converter(sse_input()):
912
+ output.append(event)
913
+
914
+ events = _parse_sse_events(output)
915
+
916
+ assert events[0][0] == "message_start"
917
+
918
+ arg_fragments = [
919
+ data["delta"]["partial_json"]
920
+ for _, data in events
921
+ if data.get("delta", {}).get("type") == "input_json_delta"
922
+ ]
923
+ full_args = "".join(arg_fragments)
924
+ assert full_args == '{"path":"/tmp/f"}'
925
+
926
+ text_deltas = [
927
+ data["delta"]["text"]
928
+ for _, data in events
929
+ if data.get("delta", {}).get("type") == "text_delta"
930
+ ]
931
+ assert text_deltas == ["\nOkay"]
932
+
933
+ block_starts = [
934
+ (data["content_block"]["type"], data.get("index"))
935
+ for ev_type, data in events
936
+ if ev_type == "content_block_start"
937
+ ]
938
+ assert block_starts[0] == ("tool_use", 0)
939
+ assert block_starts[1] == ("text", 1)
940
+
941
+ msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
942
+ assert msg_deltas[0]["delta"]["stop_reason"] == "tool_use"
943
+
944
+ assert events[-1][0] == "message_stop"
945
+
946
+ @pytest.mark.asyncio
947
+ async def test_buffered_content_flushed_on_done_without_usage_chunk(self):
948
+ """Content buffered during tool_use must be emitted even if the
949
+ stream jumps straight from finish_reason to [DONE], skipping the
950
+ empty-choices usage chunk."""
951
+
952
+ async def sse_input():
953
+ yield _make_stream_chunk(
954
+ delta=DeltaMessage(role="assistant"),
955
+ usage=UsageInfo(prompt_tokens=10, total_tokens=10),
956
+ )
957
+ yield _make_stream_chunk(
958
+ delta=DeltaMessage(
959
+ tool_calls=[
960
+ _tc(id="call_xyz", name="get_weather", args=""),
961
+ ]
962
+ )
963
+ )
964
+ yield _make_stream_chunk(
965
+ delta=DeltaMessage(
966
+ tool_calls=[_tc(args='{"city":"NYC"}')],
967
+ )
968
+ )
969
+ yield _make_stream_chunk(
970
+ delta=DeltaMessage(content="\nDone"),
971
+ finish_reason="tool_calls",
972
+ )
973
+ # No empty-choices usage chunk — go straight to [DONE].
974
+ yield "data: [DONE]"
975
+
976
+ converter = _make_stream_converter()
977
+ output = []
978
+ async for event in converter.message_stream_converter(sse_input()):
979
+ output.append(event)
980
+
981
+ events = _parse_sse_events(output)
982
+
983
+ text_deltas = [
984
+ data["delta"]["text"]
985
+ for _, data in events
986
+ if data.get("delta", {}).get("type") == "text_delta"
987
+ ]
988
+ assert text_deltas == ["\nDone"]
989
+
990
+ block_starts = [
991
+ data["content_block"]["type"]
992
+ for ev_type, data in events
993
+ if ev_type == "content_block_start"
994
+ ]
995
+ assert "tool_use" in block_starts
996
+ assert "text" in block_starts
997
+
998
+ assert events[-1][0] == "message_stop"
vllm/entrypoints/anthropic/serving.py
+30
-4
@@ -564,6 +564,7 @@ class AnthropicServingMessages(OpenAIServingChat):
564
self.block_signature: str | None = None
565
self.signature_emitted: bool = False
566
self.tool_use_id: str | None = None
567
+ self.pending_content: list[str] = []
568
569
def reset(self) -> None:
570
self.block_type = None
@@ -571,6 +572,7 @@ class AnthropicServingMessages(OpenAIServingChat):
572
self.block_signature = None
573
self.signature_emitted = False
574
self.tool_use_id = None
575
+ self.pending_content.clear()
576
577
def start(self, block: AnthropicContentBlock) -> None:
578
self.block_type = block.type
@@ -635,10 +637,30 @@ class AnthropicServingMessages(OpenAIServingChat):
637
state.start(block)
638
return event
639
640
+ def stop_and_flush() -> list[str]:
641
+ buffered = list(state.pending_content)
642
+ state.pending_content.clear()
643
+ events = stop_active_block()
644
+ if not buffered:
645
+ return events
646
+ text = "".join(buffered)
647
+ events.append(start_block(AnthropicContentBlock(type="text", text="")))
648
+ pc_chunk = AnthropicStreamEvent(
649
+ index=state.block_index,
650
+ type="content_block_delta",
651
+ delta=AnthropicDelta(type="text_delta", text=text),
652
+ )
653
+ pc_data = pc_chunk.model_dump_json(exclude_unset=True)
654
+ events.append(wrap_data_with_event(pc_data, "content_block_delta"))
655
+ events.extend(stop_active_block())
656
+ return events
657
+
658
async for item in generator:
659
if item.startswith("data:"):
660
data_str = item[5:].strip().rstrip("\n")
661
if data_str == "[DONE]":
662
+ for event in stop_and_flush():
663
+ yield event
664
stop_message = AnthropicStreamEvent(
665
type="message_stop",
666
)
@@ -675,7 +697,7 @@ class AnthropicServingMessages(OpenAIServingChat):
697
698
# last chunk including usage info
699
if len(origin_chunk.choices) == 0:
678
- for event in stop_active_block():
700
+ for event in stop_and_flush():
701
yield event
702
stop_reason = self.stop_reason_map.get(
703
finish_reason or "stop"
@@ -707,7 +729,7 @@ class AnthropicServingMessages(OpenAIServingChat):
729
pass
730
else:
731
if state.block_type != "thinking":
710
- for event in stop_active_block():
732
+ for event in stop_and_flush():
733
yield event
734
start_event = start_block(
735
AnthropicContentBlock(
@@ -733,9 +755,13 @@ class AnthropicServingMessages(OpenAIServingChat):
755
if origin_chunk.choices[0].delta.content is not None:
756
if origin_chunk.choices[0].delta.content == "":
757
pass
758
+ elif state.block_type == "tool_use":
759
+ state.pending_content.append(
760
+ origin_chunk.choices[0].delta.content
761
+ )
762
else:
763
if state.block_type != "text":
738
- for event in stop_active_block():
764
+ for event in stop_and_flush():
765
yield event
766
start_event = start_block(
767
AnthropicContentBlock(type="text", text="")
@@ -773,7 +799,7 @@ class AnthropicServingMessages(OpenAIServingChat):
799
state.tool_use_id != tool_call.id
800
and tool_name is not None
801
):
776
- for event in stop_active_block():
802
+ for event in stop_and_flush():
803
yield event
804
start_event = start_block(
805
AnthropicContentBlock(