A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
[Frontend] OpenAI Responses API supports Tool/Function calling with streaming (#29947)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Mar 12, 2026 at 15:03 UTC
9fe404ed046f0b5e9d254fd98e66c6d9e8f6a26c
3 files changed
+348
-20
tests/entrypoints/openai/test_serving_responses.py
+6
-1
@@ -659,9 +659,10 @@ class TestStreamingReasoningToContentTransition:
659
# Mock the reasoning parser on the serving instance
660
mock_parser = MagicMock()
661
mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
662
+ mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
663
serving.parser = MagicMock()
664
serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
664
-
665
+ serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
666
# Create contexts for each streaming chunk
667
contexts = [
668
_make_simple_context_with_output("chunk1", [10]),
@@ -739,8 +740,10 @@ class TestStreamingReasoningToContentTransition:
740
741
mock_parser = MagicMock()
742
mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
743
+ mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
744
serving.parser = MagicMock()
745
serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
746
+ serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
747
748
contexts = [
749
_make_simple_context_with_output("chunk1", [10]),
@@ -812,8 +815,10 @@ class TestStreamingReasoningToContentTransition:
815
816
mock_parser = MagicMock()
817
mock_parser.extract_reasoning_streaming = mock_extract_reasoning_streaming
818
+ mock_parser.extract_tool_calls_streaming = mock_extract_reasoning_streaming
819
serving.parser = MagicMock()
820
serving.parser.reasoning_parser_cls = MagicMock(return_value=mock_parser)
821
+ serving.parser.tool_parser_cls = MagicMock(return_value=mock_parser)
822
823
contexts = [
824
_make_simple_context_with_output("chunk1", [10]),
tests/v1/entrypoints/openai/serving_responses/test_function_call.py
+105
@@ -197,3 +197,108 @@ async def test_named_tool_use(client: openai.AsyncOpenAI):
197
response_2 = await client.responses.create(model=MODEL_NAME, input=input_messages)
198
# check the output
199
assert len(response_2.output_text) > 0
200
+
201
+
202
+@pytest.mark.asyncio
203
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
204
+async def test_function_calling_with_streaming_expected_arguments(
205
+ client: openai.AsyncOpenAI, model_name: str
206
+):
207
+ tools = [
208
+ {
209
+ "type": "function",
210
+ "name": "get_weather",
211
+ "description": "Get current temperature for provided location in celsius.",
212
+ "parameters": {
213
+ "type": "object",
214
+ "properties": {
215
+ "location": {"type": "string"},
216
+ },
217
+ "required": ["location"],
218
+ "additionalProperties": False,
219
+ },
220
+ "strict": True,
221
+ }
222
+ ]
223
+
224
+ stream_response = await client.responses.create(
225
+ model=model_name,
226
+ input="Can you tell me what the current weather is in Berlin?",
227
+ tools=tools,
228
+ stream=True,
229
+ )
230
+
231
+ tool_call_item = None
232
+ completed_event = None
233
+ async for event in stream_response:
234
+ if (
235
+ event.type == "response.output_item.added"
236
+ and event.item.type == "function_call"
237
+ ):
238
+ tool_call_item = event.item
239
+ elif event.type == "response.function_call_arguments.delta" and tool_call_item:
240
+ tool_call_item.arguments += event.delta
241
+ elif (
242
+ event.type == "response.output_item.done"
243
+ and event.item.type == "function_call"
244
+ ):
245
+ completed_event = event
246
+ assert tool_call_item is not None
247
+ assert tool_call_item.type == "function_call"
248
+ assert tool_call_item.name == "get_weather"
249
+ assert completed_event is not None
250
+ assert tool_call_item.arguments == completed_event.item.arguments
251
+ assert tool_call_item.name == completed_event.item.name
252
+ args = json.loads(tool_call_item.arguments)
253
+ assert "location" in args
254
+ assert args["location"] is not None
255
+
256
+
257
+@pytest.mark.asyncio
258
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
259
+async def test_function_calling_with_streaming_types(
260
+ client: openai.AsyncOpenAI, model_name: str
261
+):
262
+ # this links the "done" type with the "start" type
263
+ # so every "done" type should have a corresponding "start" type
264
+ # and every open block should be closed by the end of the stream
265
+ pairs_of_event_types = {
266
+ "response.completed": "response.created",
267
+ "response.output_item.done": "response.output_item.added",
268
+ "response.output_text.done": "response.output_text.delta",
269
+ "response.content_part.done": "response.content_part.added",
270
+ "response.reasoning_text.done": "response.reasoning_text.delta",
271
+ "response.reasoning_part.done": "response.reasoning_part.added",
272
+ "response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa
273
+ }
274
+
275
+ input_list = [
276
+ {
277
+ "role": "user",
278
+ "content": "Can you tell me what the current weather is in Berlin?",
279
+ }
280
+ ]
281
+ stream_response = await client.responses.create(
282
+ model=model_name,
283
+ input=input_list,
284
+ tools=tools,
285
+ stream=True,
286
+ )
287
+
288
+ stack_of_event_types = []
289
+ async for event in stream_response:
290
+ if event.type == "response.created":
291
+ stack_of_event_types.append(event.type)
292
+ elif event.type == "response.completed":
293
+ assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
294
+ stack_of_event_types.pop()
295
+ if event.type.endswith("added"):
296
+ stack_of_event_types.append(event.type)
297
+ elif event.type.endswith("delta"):
298
+ if stack_of_event_types[-1] == event.type:
299
+ continue
300
+ stack_of_event_types.append(event.type)
301
+ elif event.type.endswith("done"):
302
+ assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
303
+ stack_of_event_types.pop()
304
+ assert len(stack_of_event_types) == 0
vllm/entrypoints/openai/responses/serving.py
+237
-19
@@ -15,7 +15,10 @@ from fastapi import Request
15
from openai.types.responses import (
16
ResponseContentPartAddedEvent,
17
ResponseContentPartDoneEvent,
18
+ ResponseFunctionCallArgumentsDeltaEvent,
19
+ ResponseFunctionCallArgumentsDoneEvent,
20
ResponseFunctionToolCall,
21
+ ResponseFunctionToolCallItem,
22
ResponseOutputItem,
23
ResponseOutputItemAddedEvent,
24
ResponseOutputItemDoneEvent,
@@ -113,6 +116,7 @@ from vllm.parser import ParserManager
116
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
117
from vllm.tokenizers import TokenizerLike
118
from vllm.utils import random_uuid
119
+from vllm.utils.collection_utils import as_list
120
121
logger = init_logger(__name__)
122
@@ -1236,38 +1240,134 @@ class OpenAIServingResponses(OpenAIServing):
1240
reasoning_parser = None
1241
if self.parser and self.parser.reasoning_parser_cls:
1242
reasoning_parser = self.parser.reasoning_parser_cls(tokenizer)
1243
+ tool_parser = None
1244
+ if self.parser and self.parser.tool_parser_cls:
1245
+ tool_parser = self.parser.tool_parser_cls(tokenizer)
1246
+ reasoning_ended = False
1247
+ tool_call_text_started = False
1248
previous_text = ""
1249
previous_token_ids: list[int] = []
1250
+ prompt_is_reasoning_end = None
1251
first_delta_sent = False
1252
previous_delta_messages: list[DeltaMessage] = []
1253
async for ctx in result_generator:
1254
assert isinstance(ctx, SimpleContext)
1255
if ctx.last_output is None:
1256
continue
1257
+ if reasoning_parser and prompt_is_reasoning_end is None:
1258
+ prompt_is_reasoning_end = reasoning_parser.is_reasoning_end(
1259
+ ctx.last_output.prompt_token_ids
1260
+ )
1261
if ctx.last_output.outputs:
1262
output = ctx.last_output.outputs[0]
1263
# finish_reason='error' indicates a retryable error
1264
self._raise_if_error(output.finish_reason, request.request_id)
1251
- if reasoning_parser:
1265
+ delta_text = output.text
1266
+ delta_token_ids = as_list(output.token_ids)
1267
+ current_text = previous_text + delta_text
1268
+ current_token_ids = previous_token_ids + delta_token_ids
1269
+
1270
+ if reasoning_parser and tool_parser:
1271
+ if prompt_is_reasoning_end:
1272
+ reasoning_ended = True
1273
+ if not reasoning_ended:
1274
+ delta_message = reasoning_parser.extract_reasoning_streaming(
1275
+ previous_text=previous_text,
1276
+ current_text=current_text,
1277
+ delta_text=delta_text,
1278
+ previous_token_ids=previous_token_ids,
1279
+ current_token_ids=current_token_ids,
1280
+ delta_token_ids=delta_token_ids,
1281
+ )
1282
+ if reasoning_parser.is_reasoning_end(delta_token_ids):
1283
+ reasoning_ended = True
1284
+ current_token_ids = reasoning_parser.extract_content_ids(
1285
+ delta_token_ids
1286
+ )
1287
+ if delta_message and delta_message.content:
1288
+ current_text = delta_message.content
1289
+ delta_message.content = None
1290
+ else:
1291
+ current_text = ""
1292
+
1293
+ if reasoning_ended:
1294
+ if not tool_call_text_started:
1295
+ tool_call_text_started = True
1296
+ previous_text = ""
1297
+ previous_token_ids = []
1298
+ delta_text = current_text
1299
+ delta_token_ids = current_token_ids
1300
+
1301
+ delta_message = tool_parser.extract_tool_calls_streaming(
1302
+ previous_text=previous_text,
1303
+ current_text=current_text,
1304
+ delta_text=delta_text,
1305
+ previous_token_ids=previous_token_ids,
1306
+ current_token_ids=current_token_ids,
1307
+ delta_token_ids=delta_token_ids,
1308
+ request=request, # type: ignore[arg-type]
1309
+ )
1310
+ elif reasoning_parser:
1311
delta_message = reasoning_parser.extract_reasoning_streaming(
1312
previous_text=previous_text,
1254
- current_text=previous_text + output.text,
1255
- delta_text=output.text,
1313
+ current_text=current_text,
1314
+ delta_text=delta_text,
1315
+ previous_token_ids=previous_token_ids,
1316
+ current_token_ids=current_token_ids,
1317
+ delta_token_ids=delta_token_ids,
1318
+ )
1319
+ elif tool_parser:
1320
+ delta_message = tool_parser.extract_tool_calls_streaming(
1321
+ previous_text=previous_text,
1322
+ current_text=current_text,
1323
+ delta_text=delta_text,
1324
previous_token_ids=previous_token_ids,
1257
- current_token_ids=previous_token_ids + output.token_ids,
1258
- delta_token_ids=output.token_ids,
1325
+ current_token_ids=current_token_ids,
1326
+ delta_token_ids=delta_token_ids,
1327
+ request=request, # type: ignore[arg-type]
1328
)
1329
else:
1330
delta_message = DeltaMessage(
1331
content=output.text,
1332
)
1264
- previous_text += output.text
1265
- previous_token_ids += output.token_ids
1333
+ previous_text = current_text
1334
+ previous_token_ids = current_token_ids
1335
if not delta_message:
1336
continue
1337
if not first_delta_sent:
1269
- current_item_id = str(uuid.uuid4())
1270
- if delta_message.reasoning:
1338
+ current_item_id = random_uuid()
1339
+ if delta_message.tool_calls:
1340
+ current_tool_call_id = f"call_{random_uuid()}"
1341
+ assert len(delta_message.tool_calls) == 1, (
1342
+ "Multiple tool calls in one delta is not supported"
1343
+ )
1344
+ assert delta_message.tool_calls[0].function is not None, (
1345
+ "Tool call without function is not supported"
1346
+ )
1347
+ assert delta_message.tool_calls[0].function.name is not None, (
1348
+ "Tool call without function name is not supported"
1349
+ )
1350
+ current_tool_call_name = delta_message.tool_calls[
1351
+ 0
1352
+ ].function.name
1353
+ yield _increment_sequence_number_and_return(
1354
+ ResponseOutputItemAddedEvent(
1355
+ type="response.output_item.added",
1356
+ sequence_number=-1,
1357
+ output_index=current_output_index,
1358
+ item=ResponseFunctionToolCallItem(
1359
+ type="function_call",
1360
+ id=current_item_id,
1361
+ call_id=current_tool_call_id,
1362
+ name=current_tool_call_name,
1363
+ arguments=delta_message.tool_calls[
1364
+ 0
1365
+ ].function.arguments,
1366
+ status="in_progress",
1367
+ ),
1368
+ )
1369
+ )
1370
+ elif delta_message.reasoning:
1371
yield _increment_sequence_number_and_return(
1372
ResponseOutputItemAddedEvent(
1373
type="response.output_item.added",
@@ -1294,7 +1394,7 @@ class OpenAIServingResponses(OpenAIServing):
1394
),
1395
)
1396
)
1297
- else:
1397
+ elif not delta_message.tool_calls:
1398
yield _increment_sequence_number_and_return(
1399
ResponseOutputItemAddedEvent(
1400
type="response.output_item.added",
@@ -1325,7 +1425,6 @@ class OpenAIServingResponses(OpenAIServing):
1425
)
1426
)
1427
first_delta_sent = True
1328
- # todo(kebe7jun) tool call support
1428
1429
# check delta message and previous delta message are
1430
# same as content or reasoning content
@@ -1438,8 +1537,87 @@ class OpenAIServingResponses(OpenAIServing):
1537
)
1538
# reset previous delta messages
1539
previous_delta_messages = []
1441
-
1442
- if delta_message.reasoning is not None:
1540
+ if delta_message.tool_calls and delta_message.tool_calls[0].function:
1541
+ if delta_message.tool_calls[0].function.arguments:
1542
+ yield _increment_sequence_number_and_return(
1543
+ ResponseFunctionCallArgumentsDeltaEvent(
1544
+ type="response.function_call_arguments.delta",
1545
+ sequence_number=-1,
1546
+ output_index=current_output_index,
1547
+ item_id=current_item_id,
1548
+ delta=delta_message.tool_calls[0].function.arguments,
1549
+ )
1550
+ )
1551
+ # tool call initiated with no arguments
1552
+ elif delta_message.tool_calls[0].function.name:
1553
+ # send done with current content part
1554
+ # and add new function call item
1555
+ yield _increment_sequence_number_and_return(
1556
+ ResponseTextDoneEvent(
1557
+ type="response.output_text.done",
1558
+ sequence_number=-1,
1559
+ output_index=current_output_index,
1560
+ content_index=current_content_index,
1561
+ text="",
1562
+ logprobs=[],
1563
+ item_id=current_item_id,
1564
+ )
1565
+ )
1566
+ yield _increment_sequence_number_and_return(
1567
+ ResponseContentPartDoneEvent(
1568
+ type="response.content_part.done",
1569
+ sequence_number=-1,
1570
+ item_id=current_item_id,
1571
+ output_index=current_output_index,
1572
+ content_index=current_content_index,
1573
+ part=ResponseOutputText(
1574
+ type="output_text",
1575
+ text="",
1576
+ annotations=[],
1577
+ logprobs=[],
1578
+ ),
1579
+ )
1580
+ )
1581
+ yield _increment_sequence_number_and_return(
1582
+ ResponseOutputItemDoneEvent(
1583
+ type="response.output_item.done",
1584
+ sequence_number=-1,
1585
+ output_index=current_output_index,
1586
+ item=ResponseOutputMessage(
1587
+ id=current_item_id,
1588
+ type="message",
1589
+ role="assistant",
1590
+ content=[],
1591
+ status="completed",
1592
+ ),
1593
+ )
1594
+ )
1595
+ current_output_index += 1
1596
+ current_item_id = random_uuid()
1597
+ assert delta_message.tool_calls[0].function is not None
1598
+ current_tool_call_name = delta_message.tool_calls[
1599
+ 0
1600
+ ].function.name
1601
+ current_tool_call_id = f"call_{random_uuid()}"
1602
+ yield _increment_sequence_number_and_return(
1603
+ ResponseOutputItemAddedEvent(
1604
+ type="response.output_item.added",
1605
+ sequence_number=-1,
1606
+ output_index=current_output_index,
1607
+ item=ResponseFunctionToolCallItem(
1608
+ type="function_call",
1609
+ id=current_item_id,
1610
+ call_id=current_tool_call_id,
1611
+ name=current_tool_call_name,
1612
+ arguments="",
1613
+ status="in_progress",
1614
+ ),
1615
+ )
1616
+ )
1617
+ # skip content part for tool call
1618
+ current_content_index = 1
1619
+ continue
1620
+ elif delta_message.reasoning is not None:
1621
yield _increment_sequence_number_and_return(
1622
ResponseReasoningTextDeltaEvent(
1623
type="response.reasoning_text.delta",
@@ -1450,7 +1628,7 @@ class OpenAIServingResponses(OpenAIServing):
1628
delta=delta_message.reasoning,
1629
)
1630
)
1453
- elif delta_message.content is not None:
1631
+ elif delta_message.content:
1632
yield _increment_sequence_number_and_return(
1633
ResponseTextDeltaEvent(
1634
type="response.output_text.delta",
@@ -1473,8 +1651,50 @@ class OpenAIServingResponses(OpenAIServing):
1651
)
1652
1653
previous_delta_messages.append(delta_message)
1654
+
1655
if previous_delta_messages:
1477
- if previous_delta_messages[-1].reasoning is not None:
1656
+ parts = []
1657
+ for pm in previous_delta_messages:
1658
+ if pm.tool_calls:
1659
+ assert len(pm.tool_calls) == 1, (
1660
+ "Multiple tool calls in one delta is not supported"
1661
+ )
1662
+ assert pm.tool_calls[0].function is not None, (
1663
+ "Tool call without function is not supported"
1664
+ )
1665
+ parts.append(pm.tool_calls[0].function.arguments or "")
1666
+
1667
+ tool_call_arguments = "".join(parts)
1668
+ if tool_call_arguments:
1669
+ yield _increment_sequence_number_and_return(
1670
+ ResponseFunctionCallArgumentsDoneEvent(
1671
+ type="response.function_call_arguments.done",
1672
+ sequence_number=-1,
1673
+ output_index=current_output_index,
1674
+ item_id=current_item_id,
1675
+ arguments=tool_call_arguments,
1676
+ name=current_tool_call_name,
1677
+ )
1678
+ )
1679
+ current_content_index = 0
1680
+ function_call_item = ResponseFunctionToolCall(
1681
+ type="function_call",
1682
+ name=current_tool_call_name,
1683
+ arguments=tool_call_arguments,
1684
+ status="completed",
1685
+ id=current_item_id,
1686
+ call_id=current_tool_call_id,
1687
+ )
1688
+ yield _increment_sequence_number_and_return(
1689
+ ResponseOutputItemDoneEvent(
1690
+ type="response.output_item.done",
1691
+ sequence_number=-1,
1692
+ output_index=current_output_index,
1693
+ item=function_call_item,
1694
+ )
1695
+ )
1696
+
1697
+ elif previous_delta_messages[-1].reasoning is not None:
1698
reason_content = "".join(
1699
pm.reasoning
1700
for pm in previous_delta_messages
@@ -1523,11 +1743,9 @@ class OpenAIServingResponses(OpenAIServing):
1743
item=reasoning_item,
1744
)
1745
)
1526
- elif previous_delta_messages[-1].content is not None:
1746
+ elif previous_delta_messages[-1].content:
1747
final_content = "".join(
1528
- pm.content
1529
- for pm in previous_delta_messages
1530
- if pm.content is not None
1748
+ pm.content for pm in previous_delta_messages if pm.content
1749
)
1750
yield _increment_sequence_number_and_return(
1751
ResponseTextDoneEvent(