A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 21:30 UTC
[Bugfix] Preserve tool call id/type/name in streaming finish chunk (#31438)
Signed-off-by: amittell <mittell@me.com> Signed-off-by: Alex Mittell <mittell@me.com>
amittell committed
Dec 29, 2025 at 08:10 UTC
9c884faa95f0c58ff2831278cc65a68a48aef679
2 files changed
+170
-9
tests/entrypoints/openai/test_serving_chat.py
+139
@@ -1506,3 +1506,142 @@ async def test_tool_choice_validation_without_parser():
1506
assert isinstance(response_named, ErrorResponse)
1507
assert "tool_choice" in response_named.error.message
1508
assert "--tool-call-parser" in response_named.error.message
1509
+
1510
+
1511
+class TestCreateRemainingArgsDelta:
1512
+ """Tests for _create_remaining_args_delta helper function.
1513
+
1514
+ This helper is used when streaming tool calls to preserve id/type/name
1515
+ fields in the finish chunk, which would otherwise be lost.
1516
+ """
1517
+
1518
+ def test_preserves_id_type_name(self):
1519
+ """Test that id, type, and name are preserved from original delta."""
1520
+ from vllm.entrypoints.openai.protocol import (
1521
+ DeltaFunctionCall,
1522
+ DeltaMessage,
1523
+ DeltaToolCall,
1524
+ )
1525
+ from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1526
+
1527
+ original_delta = DeltaMessage(
1528
+ tool_calls=[
1529
+ DeltaToolCall(
1530
+ index=0,
1531
+ id="call_abc123",
1532
+ type="function",
1533
+ function=DeltaFunctionCall(
1534
+ name="get_weather",
1535
+ arguments='{"location": "Paris"}',
1536
+ ),
1537
+ )
1538
+ ]
1539
+ )
1540
+
1541
+ result = OpenAIServingChat._create_remaining_args_delta(
1542
+ original_delta, '", "unit": "celsius"}', 0
1543
+ )
1544
+
1545
+ assert len(result.tool_calls) == 1
1546
+ tc = result.tool_calls[0]
1547
+ assert tc.index == 0
1548
+ assert tc.id == "call_abc123"
1549
+ assert tc.type == "function"
1550
+ assert tc.function.name == "get_weather"
1551
+ assert tc.function.arguments == '", "unit": "celsius"}'
1552
+
1553
+ def test_matches_by_index(self):
1554
+ """Test that the correct tool call is matched by index."""
1555
+ from vllm.entrypoints.openai.protocol import (
1556
+ DeltaFunctionCall,
1557
+ DeltaMessage,
1558
+ DeltaToolCall,
1559
+ )
1560
+ from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1561
+
1562
+ original_delta = DeltaMessage(
1563
+ tool_calls=[
1564
+ DeltaToolCall(
1565
+ index=0,
1566
+ id="call_first",
1567
+ type="function",
1568
+ function=DeltaFunctionCall(name="func_a", arguments="{}"),
1569
+ ),
1570
+ DeltaToolCall(
1571
+ index=1,
1572
+ id="call_second",
1573
+ type="function",
1574
+ function=DeltaFunctionCall(name="func_b", arguments="{}"),
1575
+ ),
1576
+ ]
1577
+ )
1578
+
1579
+ result = OpenAIServingChat._create_remaining_args_delta(
1580
+ original_delta, '{"extra": true}', 1
1581
+ )
1582
+
1583
+ assert len(result.tool_calls) == 1
1584
+ tc = result.tool_calls[0]
1585
+ assert tc.index == 1
1586
+ assert tc.id == "call_second"
1587
+ assert tc.function.name == "func_b"
1588
+
1589
+ def test_no_matching_tool_call(self):
1590
+ """Test graceful handling when no matching tool call is found."""
1591
+ from vllm.entrypoints.openai.protocol import (
1592
+ DeltaFunctionCall,
1593
+ DeltaMessage,
1594
+ DeltaToolCall,
1595
+ )
1596
+ from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1597
+
1598
+ original_delta = DeltaMessage(
1599
+ tool_calls=[
1600
+ DeltaToolCall(
1601
+ index=0,
1602
+ id="call_zero",
1603
+ type="function",
1604
+ function=DeltaFunctionCall(name="func", arguments="{}"),
1605
+ )
1606
+ ]
1607
+ )
1608
+
1609
+ result = OpenAIServingChat._create_remaining_args_delta(
1610
+ original_delta, '{"arg": 1}', 5
1611
+ )
1612
+
1613
+ assert len(result.tool_calls) == 1
1614
+ tc = result.tool_calls[0]
1615
+ assert tc.index == 5
1616
+ assert tc.id is None
1617
+ assert tc.type is None
1618
+ assert tc.function.name is None
1619
+ assert tc.function.arguments == '{"arg": 1}'
1620
+
1621
+ def test_function_is_none(self):
1622
+ """Test handling when original tool call has no function."""
1623
+ from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
1624
+ from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1625
+
1626
+ original_delta = DeltaMessage(
1627
+ tool_calls=[
1628
+ DeltaToolCall(
1629
+ index=0,
1630
+ id="call_nofunc",
1631
+ type="function",
1632
+ function=None,
1633
+ )
1634
+ ]
1635
+ )
1636
+
1637
+ result = OpenAIServingChat._create_remaining_args_delta(
1638
+ original_delta, '{"data": "value"}', 0
1639
+ )
1640
+
1641
+ assert len(result.tool_calls) == 1
1642
+ tc = result.tool_calls[0]
1643
+ assert tc.index == 0
1644
+ assert tc.id == "call_nofunc"
1645
+ assert tc.type == "function"
1646
+ assert tc.function.name is None
1647
+ assert tc.function.arguments == '{"data": "value"}'
vllm/entrypoints/openai/serving_chat.py
+31
-9
@@ -1208,15 +1208,8 @@ class OpenAIServingChat(OpenAIServing):
1208
# check to see if there's anything left to stream
1209
remaining_call = expected_call.replace(actual_call, "", 1)
1210
# set that as a delta message
1211
- delta_message = DeltaMessage(
1212
- tool_calls=[
1213
- DeltaToolCall(
1214
- index=index,
1215
- function=DeltaFunctionCall(
1216
- arguments=remaining_call
1217
- ).model_dump(exclude_none=True),
1218
- )
1219
- ]
1211
+ delta_message = self._create_remaining_args_delta(
1212
+ delta_message, remaining_call, index
1213
)
1214
1215
# Send the finish response for each request.n only once
@@ -1803,6 +1796,35 @@ class OpenAIServingChat(OpenAIServing):
1796
and delta_message.tool_calls[0].function.arguments is not None
1797
)
1798
1799
+ @staticmethod
1800
+ def _create_remaining_args_delta(
1801
+ delta_message: DeltaMessage,
1802
+ remaining_call: str,
1803
+ index: int,
1804
+ ) -> DeltaMessage:
1805
+ """
1806
+ Create a delta message for remaining tool arguments, preserving
1807
+ id/type/name from the original delta.
1808
+ """
1809
+ original_tc = next(
1810
+ (tc for tc in delta_message.tool_calls if tc.index == index),
1811
+ None,
1812
+ )
1813
+ original_fn = original_tc.function if original_tc else None
1814
+ return DeltaMessage(
1815
+ tool_calls=[
1816
+ DeltaToolCall(
1817
+ index=index,
1818
+ id=original_tc.id if original_tc else None,
1819
+ type=original_tc.type if original_tc else None,
1820
+ function=DeltaFunctionCall(
1821
+ name=original_fn.name if original_fn else None,
1822
+ arguments=remaining_call,
1823
+ ),
1824
+ )
1825
+ ]
1826
+ )
1827
+
1828
def _make_request_with_harmony(
1829
self,
1830
request: ChatCompletionRequest,