@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3683fe6c06

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 19:45 UTC

[Bugfix] Fix shared-object aliasing in n>1 streaming with tool calls (#38158)

Signed-off-by: Yifan Zong <yzong@redhat.com> Signed-off-by: Yifan <yzong@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

yzong-rh committed Mar 30, 2026 at 06:12 UTC 3683fe6c0651fe54a0201552ae7dfb7acb1e0cea

2 files changed +168 -2

tests/entrypoints/openai/chat_completion/test_serving_chat.py

+165

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import asyncio

4 +import json

5 from contextlib import suppress

6 from dataclasses import dataclass, field

7 from typing import Any

@@ -1792,6 +1793,170 @@ async def test_tool_choice_validation_without_parser():

1793 assert "--tool-call-parser" in response_named.error.message

1794

1795

1796 +@pytest.mark.asyncio

1797 +async def test_streaming_n_gt1_independent_tool_parsers():

1798 + """n>1 streaming must use independent parser instances

1799 + and token-id histories per choice.

1800 + """

1801 + mock_engine = MagicMock(spec=AsyncLLM)

1802 + mock_engine.errored = False

1803 + mock_engine.model_config = MockModelConfig()

1804 + mock_engine.input_processor = MagicMock()

1805 + mock_engine.io_processor = MagicMock()

1806 + mock_engine.renderer = _build_renderer(mock_engine.model_config)

1807 +

1808 + models = OpenAIServingModels(

1809 + engine_client=mock_engine,

1810 + base_model_paths=BASE_MODEL_PATHS,

1811 + )

1812 + openai_serving_render = _build_serving_render(mock_engine, models.registry)

1813 +

1814 + serving_chat = OpenAIServingChat(

1815 + mock_engine,

1816 + models,

1817 + response_role="assistant",

1818 + openai_serving_render=openai_serving_render,

1819 + chat_template=CHAT_TEMPLATE,

1820 + chat_template_content_format="auto",

1821 + request_logger=None,

1822 + enable_auto_tools=True,

1823 + tool_parser="hermes",

1824 + )

1825 +

1826 + tokenizer = get_tokenizer(MODEL_NAME)

1827 +

1828 + tools = [

1829 + {

1830 + "type": "function",

1831 + "function": {

1832 + "name": "get_weather",

1833 + "description": "Get weather",

1834 + "parameters": {

1835 + "type": "object",

1836 + "properties": {"city": {"type": "string"}},

1837 + "required": ["city"],

1838 + },

1839 + },

1840 + }

1841 + ]

1842 +

1843 + num_choices = 2

1844 +

1845 + request = ChatCompletionRequest(

1846 + model=MODEL_NAME,

1847 + messages=[{"role": "user", "content": "test"}],

1848 + n=num_choices,

1849 + stream=True,

1850 + tools=tools,

1851 + tool_choice="auto",

1852 + )

1853 +

1854 + tool_call_text = (

1855 + "<tool_call>\n"

1856 + '{"name": "get_weather", "arguments": {"city": "Tokyo"}}\n'

1857 + "</tool_call>"

1858 + )

1859 + all_token_ids = tokenizer.encode(tool_call_text, add_special_tokens=False)

1860 +

1861 + # Compute proper delta text for each token so that concatenated deltas

1862 + # reproduce the original string exactly.

1863 + steps: list[tuple[str, int]] = []

1864 + prev_decoded = ""

1865 + for i, tid in enumerate(all_token_ids):

1866 + decoded_so_far = tokenizer.decode(all_token_ids[: i + 1])

1867 + delta = decoded_so_far[len(prev_decoded) :]

1868 + steps.append((delta, tid))

1869 + prev_decoded = decoded_so_far

1870 +

1871 + async def result_generator():

1872 + for delta_text, token_id in steps:

1873 + yield RequestOutput(

1874 + request_id="test-req",

1875 + prompt="test",

1876 + prompt_token_ids=[1, 2, 3],

1877 + prompt_logprobs=None,

1878 + outputs=[

1879 + CompletionOutput(

1880 + index=choice_idx,

1881 + text=delta_text,

1882 + token_ids=[token_id],

1883 + cumulative_logprob=0.0,

1884 + logprobs=None,

1885 + )

1886 + for choice_idx in range(num_choices)

1887 + ],

1888 + finished=False,

1889 + )

1890 + # Final output with finish_reason

1891 + yield RequestOutput(

1892 + request_id="test-req",

1893 + prompt="test",

1894 + prompt_token_ids=[1, 2, 3],

1895 + prompt_logprobs=None,

1896 + outputs=[

1897 + CompletionOutput(

1898 + index=choice_idx,

1899 + text="",

1900 + token_ids=[],

1901 + cumulative_logprob=0.0,

1902 + logprobs=None,

1903 + finish_reason="stop",

1904 + )

1905 + for choice_idx in range(num_choices)

1906 + ],

1907 + finished=True,

1908 + )

1909 +

1910 + # Collect tool-call deltas per choice from the SSE stream.

1911 + tc_deltas_by_choice: dict[int, list[dict]] = {i: [] for i in range(num_choices)}

1912 + async for chunk_str in serving_chat.chat_completion_stream_generator(

1913 + request=request,

1914 + result_generator=result_generator(),

1915 + request_id="test-req",

1916 + model_name=MODEL_NAME,

1917 + conversation=[],

1918 + tokenizer=tokenizer,

1919 + request_metadata=RequestResponseMetadata(

1920 + request_id="test-req",

1921 + model_name=MODEL_NAME,

1922 + ),

1923 + ):

1924 + if not chunk_str.strip() or "data: [DONE]" in chunk_str:

1925 + continue

1926 + if chunk_str.startswith("data: "):

1927 + data = json.loads(chunk_str[6:].strip())

1928 + for choice in data.get("choices", []):

1929 + idx = choice["index"]

1930 + delta = choice.get("delta", {})

1931 + if delta.get("tool_calls"):

1932 + for tc in delta["tool_calls"]:

1933 + tc_deltas_by_choice[idx].append(tc)

1934 +

1935 + # Both choices must independently produce the correct tool call.

1936 + for choice_idx in range(num_choices):

1937 + deltas = tc_deltas_by_choice[choice_idx]

1938 + assert len(deltas) > 0, (

1939 + f"Choice {choice_idx}: expected tool-call deltas but got none"

1940 + )

1941 +

1942 + name = None

1943 + args_buf = ""

1944 + for tc in deltas:

1945 + fn = tc.get("function", {})

1946 + if fn.get("name"):

1947 + name = fn["name"]

1948 + if fn.get("arguments"):

1949 + args_buf += fn["arguments"]

1950 +

1951 + assert name == "get_weather", (

1952 + f"Choice {choice_idx}: expected 'get_weather', got {name!r}"

1953 + )

1954 + parsed_args = json.loads(args_buf)

1955 + assert parsed_args == {"city": "Tokyo"}, (

1956 + f"Choice {choice_idx}: expected {{'city': 'Tokyo'}}, got {parsed_args}"

1957 + )

1958 +

1959 +

1960 class TestCreateRemainingArgsDelta:

1961 """Tests for _create_remaining_args_delta helper function.

1962

vllm/entrypoints/openai/chat_completion/serving.py

+3 -2

@@ -548,7 +548,7 @@ class OpenAIServingChat(OpenAIServing):

548 # all_previous_token_ids will not be used twice in the same iteration.

549 if tool_choice_auto or reasoning_parser:

550 # These are only required in "auto" tool choice case

551 - all_previous_token_ids = [[]] * num_choices

551 + all_previous_token_ids = [[] for _ in range(num_choices)]

552 # For reasoning parser and tool call all enabled

553 added_content_delta_arr = [False] * num_choices

554 reasoning_end_arr = [False] * num_choices

@@ -566,7 +566,8 @@ class OpenAIServingChat(OpenAIServing):

566

567 tool_parsers: list[ToolParser | None] = [

568 self.tool_parser(tokenizer, request.tools)

569 - ] * num_choices

569 + for _ in range(num_choices)

570 + ]

571 else:

572 tool_parsers = [None] * num_choices

573 except Exception as e: