@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 25b3242d8b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 04:45 UTC

Fix Responses API streaming for multiple auto tool calls (#39626)

Signed-off-by: noobhappylife <aratar1991@hotmail.com>

noobHappylife committed Apr 14, 2026 at 13:28 UTC 25b3242d8b5c7cf5ddc2c0fe79e02a7ec7ba08c1

3 files changed +329 -21

tests/entrypoints/openai/responses/test_function_call.py

+50 -16

@@ -249,40 +249,74 @@ async def test_function_calling_with_streaming_expected_arguments(

249 "additionalProperties": False,

250 },

251 "strict": True,

252 - }

252 + },

253 + {

254 + "type": "function",

255 + "name": "get_time",

256 + "description": "Get current local time for provided location.",

257 + "parameters": {

258 + "type": "object",

259 + "properties": {

260 + "location": {"type": "string"},

261 + },

262 + "required": ["location"],

263 + "additionalProperties": False,

264 + },

265 + "strict": True,

266 + },

267 ]

268

269 stream_response = await client.responses.create(

270 model=model_name,

257 - input="Can you tell me what the current weather is in Berlin?",

271 + input=(

272 + "Use tools only. Call get_weather for Berlin and get_time for Tokyo. "

273 + "Do not answer directly."

274 + ),

275 tools=tools,

276 stream=True,

277 )

278

262 - tool_call_item = None

263 - completed_event = None

279 + tool_call_items = {}

280 + arguments_done_events = {}

281 + completed_events = {}

282 async for event in stream_response:

283 if (

284 event.type == "response.output_item.added"

285 and event.item.type == "function_call"

286 ):

269 - tool_call_item = event.item

270 - elif event.type == "response.function_call_arguments.delta" and tool_call_item:

287 + tool_call_items[event.output_index] = event.item

288 + elif event.type == "response.function_call_arguments.delta":

289 + tool_call_item = tool_call_items[event.output_index]

290 tool_call_item.arguments += event.delta

291 + elif event.type == "response.function_call_arguments.done":

292 + arguments_done_events[event.output_index] = event

293 elif (

294 event.type == "response.output_item.done"

295 and event.item.type == "function_call"

296 ):

276 - completed_event = event

277 - assert tool_call_item is not None

278 - assert tool_call_item.type == "function_call"

279 - assert tool_call_item.name == "get_weather"

280 - assert completed_event is not None

281 - assert tool_call_item.arguments == completed_event.item.arguments

282 - assert tool_call_item.name == completed_event.item.name

283 - args = json.loads(tool_call_item.arguments)

284 - assert "location" in args

285 - assert args["location"] is not None

297 + completed_events[event.output_index] = event

298 + assert len(tool_call_items) >= 2

299 + assert len(arguments_done_events) >= 2

300 + assert len(completed_events) >= 2

301 +

302 + tool_calls_by_name = {

303 + event.item.name: (

304 + tool_call_items[output_index],

305 + arguments_done_events[output_index],

306 + event.item,

307 + )

308 + for output_index, event in completed_events.items()

309 + }

310 + assert {"get_weather", "get_time"}.issubset(tool_calls_by_name)

311 + for added_item, arguments_done_event, completed_item in tool_calls_by_name.values():

312 + assert added_item.type == "function_call"

313 + assert added_item.arguments == arguments_done_event.arguments

314 + assert added_item.arguments == completed_item.arguments

315 + assert added_item.name == arguments_done_event.name

316 + assert added_item.name == completed_item.name

317 + args = json.loads(added_item.arguments)

318 + assert "location" in args

319 + assert args["location"] is not None

320

321

322 @pytest.mark.asyncio

tests/entrypoints/openai/responses/test_serving_responses.py

+196

@@ -27,7 +27,9 @@ from openai.types.responses.tool import (

27 import vllm.envs as envs

28 from vllm.entrypoints.mcp.tool_server import ToolServer

29 from vllm.entrypoints.openai.engine.protocol import (

30 + DeltaFunctionCall,

31 DeltaMessage,

32 + DeltaToolCall,

33 ErrorResponse,

34 RequestResponseMetadata,

35 )

@@ -928,3 +930,197 @@ class TestStreamingReasoningToContentTransition:

930 ]

931 assert len(item_done_events) == 1

932 assert isinstance(item_done_events[0].item, ResponseReasoningItem)

933 +

934 +

935 +class TestAutoToolStreaming:

936 + @staticmethod

937 + async def _collect_events(delta_sequence: list[DeltaMessage]):

938 + serving = _make_serving_instance_with_reasoning()

939 + _mock_parser_with_reasoning(serving, delta_sequence)

940 +

941 + contexts = [

942 + _make_simple_context_with_output("chunk", [i])

943 + for i in range(len(delta_sequence))

944 + ]

945 +

946 + async def result_generator():

947 + for ctx in contexts:

948 + yield ctx

949 +

950 + request = ResponsesRequest(

951 + input="hi",

952 + tools=[

953 + {

954 + "type": "function",

955 + "name": "get_weather",

956 + "description": "Get weather.",

957 + "parameters": {

958 + "type": "object",

959 + "properties": {"location": {"type": "string"}},

960 + "required": ["location"],

961 + "additionalProperties": False,

962 + },

963 + }

964 + ],

965 + tool_choice="auto",

966 + stream=True,

967 + )

968 + sampling_params = SamplingParams(max_tokens=64)

969 + metadata = RequestResponseMetadata(request_id="req")

970 + _identity_increment._counter = 0 # type: ignore

971 +

972 + events = []

973 + async for event in serving._process_simple_streaming_events(

974 + request=request,

975 + sampling_params=sampling_params,

976 + result_generator=result_generator(),

977 + context=SimpleContext(),

978 + model_name="test-model",

979 + tokenizer=MagicMock(),

980 + request_metadata=metadata,

981 + created_time=0,

982 + _increment_sequence_number_and_return=_identity_increment,

983 + ):

984 + events.append(event)

985 + return events

986 +

987 + @pytest.mark.skip_global_cleanup

988 + @pytest.mark.asyncio

989 + async def test_auto_multi_tool_streaming_opens_one_item_per_tool(self, monkeypatch):

990 + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)

991 +

992 + delta_sequence = [

993 + DeltaMessage(

994 + tool_calls=[

995 + DeltaToolCall(

996 + id="call_vienna",

997 + type="function",

998 + index=0,

999 + function=DeltaFunctionCall(

1000 + name="get_weather",

1001 + arguments="",

1002 + ),

1003 + )

1004 + ]

1005 + ),

1006 + DeltaMessage(

1007 + tool_calls=[

1008 + DeltaToolCall(

1009 + index=0,

1010 + function=DeltaFunctionCall(

1011 + arguments='{"location":"Vienna"}',

1012 + ),

1013 + )

1014 + ]

1015 + ),

1016 + DeltaMessage(

1017 + tool_calls=[

1018 + DeltaToolCall(

1019 + id="call_berlin",

1020 + type="function",

1021 + index=1,

1022 + function=DeltaFunctionCall(

1023 + name="get_weather",

1024 + arguments='{"location":"Berlin"}',

1025 + ),

1026 + )

1027 + ]

1028 + ),

1029 + ]

1030 + events = await self._collect_events(delta_sequence)

1031 +

1032 + function_items = [

1033 + event

1034 + for event in events

1035 + if event.type == "response.output_item.added"

1036 + and getattr(event.item, "type", None) == "function_call"

1037 + ]

1038 + assert len(function_items) == 2

1039 + assert [event.item.name for event in function_items] == [

1040 + "get_weather",

1041 + "get_weather",

1042 + ]

1043 + assert [event.output_index for event in function_items] == [0, 1]

1044 +

1045 + argument_deltas = [

1046 + event.delta

1047 + for event in events

1048 + if event.type == "response.function_call_arguments.delta"

1049 + ]

1050 + assert argument_deltas == [

1051 + '{"location":"Vienna"}',

1052 + '{"location":"Berlin"}',

1053 + ]

1054 +

1055 + argument_done = [

1056 + event

1057 + for event in events

1058 + if event.type == "response.function_call_arguments.done"

1059 + ]

1060 + assert [event.arguments for event in argument_done] == [

1061 + '{"location":"Vienna"}',

1062 + '{"location":"Berlin"}',

1063 + ]

1064 + assert [event.output_index for event in argument_done] == [0, 1]

1065 +

1066 + function_done = [

1067 + event

1068 + for event in events

1069 + if event.type == "response.output_item.done"

1070 + and getattr(event.item, "type", None) == "function_call"

1071 + ]

1072 + assert [event.item.arguments for event in function_done] == [

1073 + '{"location":"Vienna"}',

1074 + '{"location":"Berlin"}',

1075 + ]

1076 + assert [event.output_index for event in function_done] == [0, 1]

1077 +

1078 + @pytest.mark.skip_global_cleanup

1079 + @pytest.mark.asyncio

1080 + async def test_auto_tool_choice_first_delta_tool_call_does_not_duplicate_item(

1081 + self, monkeypatch

1082 + ):

1083 + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)

1084 +

1085 + delta_sequence = [

1086 + DeltaMessage(

1087 + tool_calls=[

1088 + DeltaToolCall(

1089 + id="call_test",

1090 + type="function",

1091 + index=0,

1092 + function=DeltaFunctionCall(

1093 + name="get_weather",

1094 + arguments="",

1095 + ),

1096 + )

1097 + ]

1098 + ),

1099 + DeltaMessage(

1100 + tool_calls=[

1101 + DeltaToolCall(

1102 + index=0,

1103 + function=DeltaFunctionCall(

1104 + arguments='{"location":"Berlin"}',

1105 + ),

1106 + )

1107 + ]

1108 + ),

1109 + ]

1110 + events = await self._collect_events(delta_sequence)

1111 +

1112 + function_items = [

1113 + event

1114 + for event in events

1115 + if event.type == "response.output_item.added"

1116 + and getattr(event.item, "type", None) == "function_call"

1117 + ]

1118 + assert len(function_items) == 1

1119 + assert function_items[0].item.name == "get_weather"

1120 +

1121 + argument_deltas = [

1122 + event.delta

1123 + for event in events

1124 + if event.type == "response.function_call_arguments.delta"

1125 + ]

1126 + assert "".join(argument_deltas) == '{"location":"Berlin"}'

vllm/entrypoints/openai/responses/serving.py

+83 -5

@@ -1341,6 +1341,7 @@ class OpenAIServingResponses(OpenAIServing):

1341 current_content_index = 0

1342 current_output_index = 0

1343 current_item_id = ""

1344 + current_tool_call_index: int | None = None

1345 parser = self.parser(tokenizer, request.tools) if self.parser else None

1346 first_delta_sent = False

1347 previous_delta_messages: list[DeltaMessage] = []

@@ -1368,6 +1369,7 @@ class OpenAIServingResponses(OpenAIServing):

1369 )

1370 if not delta_message:

1371 continue

1372 + tool_call_item_started = False

1373 if not first_delta_sent:

1374 current_item_id = random_uuid()

1375 if delta_message.tool_calls:

@@ -1384,6 +1386,7 @@ class OpenAIServingResponses(OpenAIServing):

1386 current_tool_call_name = delta_message.tool_calls[

1387 0

1388 ].function.name

1389 + current_tool_call_index = delta_message.tool_calls[0].index

1390 yield _increment_sequence_number_and_return(

1391 ResponseOutputItemAddedEvent(

1392 type="response.output_item.added",

@@ -1394,13 +1397,12 @@ class OpenAIServingResponses(OpenAIServing):

1397 id=current_item_id,

1398 call_id=current_tool_call_id,

1399 name=current_tool_call_name,

1397 - arguments=delta_message.tool_calls[

1398 - 0

1399 - ].function.arguments,

1400 + arguments="",

1401 status="in_progress",

1402 ),

1403 )

1404 )

1405 + tool_call_item_started = True

1406 elif delta_message.reasoning:

1407 yield _increment_sequence_number_and_return(

1408 ResponseOutputItemAddedEvent(

@@ -1572,6 +1574,79 @@ class OpenAIServingResponses(OpenAIServing):

1574 # reset previous delta messages

1575 previous_delta_messages = []

1576 if delta_message.tool_calls and delta_message.tool_calls[0].function:

1577 + tool_call = delta_message.tool_calls[0]

1578 + tool_call_function = tool_call.function

1579 + if (

1580 + current_tool_call_index is not None

1581 + and tool_call.index is not None

1582 + and tool_call.index != current_tool_call_index

1583 + and tool_call_function is not None

1584 + and tool_call_function.name is not None

1585 + ):

1586 + # From one tool call to another, finalize the previous

1587 + # function-call item before opening the next one.

1588 + parts = []

1589 + for pm in previous_delta_messages:

1590 + if pm.tool_calls:

1591 + previous_tool_call = pm.tool_calls[0]

1592 + if previous_tool_call.function is not None:

1593 + parts.append(

1594 + previous_tool_call.function.arguments or ""

1595 + )

1596 +

1597 + tool_call_arguments = "".join(parts)

1598 + yield _increment_sequence_number_and_return(

1599 + ResponseFunctionCallArgumentsDoneEvent(

1600 + type="response.function_call_arguments.done",

1601 + sequence_number=-1,

1602 + output_index=current_output_index,

1603 + item_id=current_item_id,

1604 + arguments=tool_call_arguments,

1605 + name=current_tool_call_name,

1606 + )

1607 + )

1608 + function_call_item = ResponseFunctionToolCall(

1609 + type="function_call",

1610 + name=current_tool_call_name,

1611 + arguments=tool_call_arguments,

1612 + status="completed",

1613 + id=current_item_id,

1614 + call_id=current_tool_call_id,

1615 + )

1616 + yield _increment_sequence_number_and_return(

1617 + ResponseOutputItemDoneEvent(

1618 + type="response.output_item.done",

1619 + sequence_number=-1,

1620 + output_index=current_output_index,

1621 + item=function_call_item,

1622 + )

1623 + )

1624 + # Reset previous delta messages so the next tool call

1625 + # does not reuse arguments from the completed item.

1626 + previous_delta_messages = []

1627 + current_output_index += 1

1628 + current_item_id = random_uuid()

1629 + current_tool_call_name = tool_call_function.name

1630 + current_tool_call_id = f"call_{random_uuid()}"

1631 + current_tool_call_index = tool_call.index

1632 + yield _increment_sequence_number_and_return(

1633 + ResponseOutputItemAddedEvent(

1634 + type="response.output_item.added",

1635 + sequence_number=-1,

1636 + output_index=current_output_index,

1637 + item=ResponseFunctionToolCallItem(

1638 + type="function_call",

1639 + id=current_item_id,

1640 + call_id=current_tool_call_id,

1641 + name=current_tool_call_name,

1642 + arguments="",

1643 + status="in_progress",

1644 + ),

1645 + )

1646 + )

1647 + current_content_index = 0

1648 + tool_call_item_started = True

1649 +

1650 if delta_message.tool_calls[0].function.arguments:

1651 yield _increment_sequence_number_and_return(

1652 ResponseFunctionCallArgumentsDeltaEvent(

@@ -1583,7 +1658,10 @@ class OpenAIServingResponses(OpenAIServing):

1658 )

1659 )

1660 # tool call initiated with no arguments

1586 - elif delta_message.tool_calls[0].function.name:

1661 + elif (

1662 + delta_message.tool_calls[0].function.name

1663 + and not tool_call_item_started

1664 + ):

1665 # send done with current content part

1666 # and add new function call item

1667 yield _increment_sequence_number_and_return(

@@ -1628,11 +1706,11 @@ class OpenAIServingResponses(OpenAIServing):

1706 )

1707 current_output_index += 1

1708 current_item_id = random_uuid()

1631 - assert delta_message.tool_calls[0].function is not None

1709 current_tool_call_name = delta_message.tool_calls[

1710 0

1711 ].function.name

1712 current_tool_call_id = f"call_{random_uuid()}"

1713 + current_tool_call_index = delta_message.tool_calls[0].index

1714 yield _increment_sequence_number_and_return(

1715 ResponseOutputItemAddedEvent(

1716 type="response.output_item.added",