@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 146f44b77d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 4, 15:00 UTC

[Frontend]Responses API supports Tool/Function calling with streaming with required (#40700)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

Chauncey committed Apr 28, 2026 at 10:36 UTC 146f44b77d079f5a16fe7094fa0dde6b1be95f38

6 files changed +932 -718

tests/entrypoints/openai/responses/test_function_call.py

+156 -1

@@ -321,12 +321,100 @@ async def test_function_calling_with_streaming_expected_arguments(

321

322 @pytest.mark.asyncio

323 @pytest.mark.parametrize("model_name", [MODEL_NAME])

324 +@pytest.mark.parametrize(

325 + "tool_choice",

326 + ["auto", "required"],

327 +)

328 async def test_function_calling_with_streaming_types(

325 - client: openai.AsyncOpenAI, model_name: str

329 + client: openai.AsyncOpenAI, model_name: str, tool_choice

330 ):

331 # this links the "done" type with the "start" type

332 # so every "done" type should have a corresponding "start" type

333 # and every open block should be closed by the end of the stream

334 + #

335 + # stream of events for a response with function call could look like this:

336 + # option1: reasoning -> content(option) -> function_call

337 + # response.created

338 + # -> response.in_progress

339 + # -> response.output_item.added

340 + # -> response.reasoning_part.added

341 + # -> response.reasoning_text.delta

342 + # ....

343 + # -> response.reasoning_text.delta

344 + # -> response.reasoning_text.done

345 + # -> response.reasoning_part.done

346 + # -> response.output_item.done

347 + # -> response.output_item.added

348 + # -> response.content_part.added

349 + # -> response.output_text.delta

350 + # ...

351 + # -> response.output_text.delta

352 + # -> response.output_text.done

353 + # -> response.content_part.done

354 + # -> response.output_item.done

355 + # -> response.output_item.added

356 + # -> response.function_call_arguments.delta

357 + # ...

358 + # -> response.function_call_arguments.delta

359 + # -> response.function_call_arguments.done

360 + # -> response.output_item.done

361 + # -> response.completed

362 + #

363 + #

364 + # option2: reasoning -> content

365 + # response.created

366 + # -> response.in_progress

367 + # -> response.output_item.added

368 + # -> response.reasoning_part.added

369 + # -> response.reasoning_text.delta

370 + # ....

371 + # -> response.reasoning_text.delta

372 + # -> response.reasoning_text.done

373 + # -> response.reasoning_part.done

374 + # -> response.output_item.done

375 + # -> response.output_item.added

376 + # -> response.content_part.added

377 + # -> response.output_text.delta

378 + # ..

379 + # -> response.output_text.delta

380 + # -> response.output_text.done

381 + # -> response.content_part.done

382 + # -> response.output_item.done

383 + # -> response.completed

384 + #

385 + # option3: content

386 + #

387 + # response.created

388 + # -> response.in_progress

389 + # -> response.output_item.added

390 + # -> response.content_part.added

391 + # -> response.output_text.delta

392 + # ...

393 + # -> response.output_text.delta

394 + # -> response.output_text.done

395 + # -> response.content_part.done

396 + # -> response.output_item.done

397 + # -> response.completed

398 + #

399 + # option4: content -> function_call

400 + # response.created

401 + # -> response.in_progress

402 + # -> response.output_item.added

403 + # -> response.content_part.added

404 + # -> response.output_text.delta

405 + # ...

406 + # -> response.output_text.delta

407 + # -> response.output_text.done

408 + # -> response.content_part.done

409 + # -> response.output_item.done

410 + # -> response.output_item.added

411 + # -> response.function_call_arguments.delta

412 + # ...

413 + # -> response.function_call_arguments.delta

414 + # -> response.function_call_arguments.done

415 + # -> response.output_item.done

416 + # -> response.completed

417 +

418 pairs_of_event_types = {

419 "response.completed": "response.created",

420 "response.output_item.done": "response.output_item.added",

@@ -347,6 +435,7 @@ async def test_function_calling_with_streaming_types(

435 model=model_name,

436 input=input_list,

437 tools=tools,

438 + tool_choice=tool_choice,

439 stream=True,

440 )

441

@@ -367,3 +456,69 @@ async def test_function_calling_with_streaming_types(

456 assert stack_of_event_types[-1] == pairs_of_event_types[event.type]

457 stack_of_event_types.pop()

458 assert len(stack_of_event_types) == 0

459 +

460 +

461 +@pytest.mark.asyncio

462 +@pytest.mark.parametrize("model_name", [MODEL_NAME])

463 +@pytest.mark.parametrize(

464 + "tool_choice",

465 + ["required", "auto"],

466 +)

467 +async def test_function_calling_with_streaming_forced_tool_choice(

468 + client: openai.AsyncOpenAI, model_name: str, tool_choice: str

469 +):

470 + tools = [

471 + {

472 + "type": "function",

473 + "name": "get_weather",

474 + "description": "Get current temperature for provided location in celsius.",

475 + "parameters": {

476 + "type": "object",

477 + "properties": {

478 + "location": {"type": "string"},

479 + },

480 + "required": ["location"],

481 + "additionalProperties": False,

482 + },

483 + "strict": True,

484 + }

485 + ]

486 +

487 + stream_response = await client.responses.create(

488 + model=model_name,

489 + input="Call the get_weather function for Berlin and do not answer directly.",

490 + tools=tools,

491 + tool_choice=tool_choice,

492 + stream=True,

493 + )

494 +

495 + tool_call_item = None

496 + completed_event = None

497 + text_deltas = []

498 + async for event in stream_response:

499 + if (

500 + event.type == "response.output_item.added"

501 + and event.item.type == "function_call"

502 + ):

503 + tool_call_item = event.item

504 + elif event.type == "response.output_text.delta":

505 + text_deltas.append(event.delta)

506 + elif event.type == "response.function_call_arguments.delta" and tool_call_item:

507 + tool_call_item.arguments += event.delta

508 + elif (

509 + event.type == "response.output_item.done"

510 + and event.item.type == "function_call"

511 + ):

512 + completed_event = event

513 +

514 + assert tool_call_item is not None

515 + assert tool_call_item.type == "function_call"

516 + assert tool_call_item.name == "get_weather"

517 + assert completed_event is not None

518 + assert tool_call_item.arguments == completed_event.item.arguments

519 + assert tool_call_item.name == completed_event.item.name

520 + args = json.loads(tool_call_item.arguments)

521 + assert "location" in args

522 + assert args["location"] is not None

523 + # Forced tool choice should not leak tool-call JSON via output_text delta.

524 + assert "".join(text_deltas).strip() == ""

vllm/entrypoints/openai/chat_completion/serving.py

+9 -134

@@ -9,10 +9,7 @@ from collections.abc import Sequence as GenericSequence

9 from http import HTTPStatus

10 from typing import TYPE_CHECKING, Any, Final

12 -import partial_json_parser

13 -import regex as re

12 from fastapi import Request

15 -from partial_json_parser.core.options import Allow

14 from vllm.engine.protocol import EngineClient

15 from vllm.entrypoints.chat_utils import (

@@ -73,7 +70,7 @@ from vllm.reasoning import ReasoningParser

70 from vllm.renderers import ChatParams

71 from vllm.sampling_params import BeamSearchParams, SamplingParams

72 from vllm.tokenizers import TokenizerLike

76 -from vllm.tool_parsers.utils import partial_json_loads

73 +from vllm.tool_parsers.streaming import extract_required_tool_call_streaming

74 from vllm.utils.collection_utils import as_list

75 from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser

@@ -383,45 +380,6 @@ class OpenAIServingChat(OpenAIServing):

380 return self.response_role

381 return request.messages[-1]["role"]

382

386 - @staticmethod

387 - def _bracket_level(s: str, opening="{", closing="}") -> int:

388 - """

389 - Calculate the current level of nested brackets in a given string.

390 - """

391 - level = 0

392 - for char in s:

393 - if char == opening:

394 - level += 1

395 - elif char == closing:

396 - level -= 1

397 - return level

398 -

399 - @staticmethod

400 - def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:

401 - # remove last '},' of the tool definition stemming from the

402 - # "name"/"parameters" outer object or closing ']' of the tool list

403 - # count occurrences of opening and closing curly braces and

404 - # once level 0 is reached stop outputting text

405 - # if 0 is reached while parsing the delta_text we know the current

406 - # tool will finish in this current iteration

407 - bracket_level = OpenAIServingChat._bracket_level(previous_text)

408 - updated_delta, passed_zero = "", False

409 - for c in delta_text:

410 - if c == "{":

411 - bracket_level += 1

412 - passed_zero = bracket_level == 0

413 - elif c == "}":

414 - bracket_level -= 1

415 - passed_zero = bracket_level == 0

416 -

417 - if bracket_level != 0:

418 - updated_delta += c

419 - else:

420 - # if a comma is reached at level 0 we can stop

421 - if c == ",":

422 - break

423 - return updated_delta, passed_zero

424 -

383 def extract_tool_call_required_streaming(

384 self,

385 previous_text: str,

@@ -430,97 +388,14 @@ class OpenAIServingChat(OpenAIServing):

388 function_name_returned: bool,

389 tool_call_idx: int | None = None,

390 ) -> tuple[DeltaMessage | None, bool]:

433 - if current_text is None or current_text == "":

434 - # if the current text is empty, we cannot parse it

435 - return None, function_name_returned

436 - try:

437 - flags = Allow.ALL

438 - obj, _ = partial_json_loads(current_text, flags)

439 - except (

440 - partial_json_parser.core.exceptions.MalformedJSON,

441 - json.JSONDecodeError,

442 - ):

443 - logger.debug("not enough tokens to parse into JSON yet")

444 - obj = None

445 -

446 - # check if the current text is a valid array

447 - # containing a partial tool calling object

448 - # if not repeat

449 - if obj is None or not isinstance(obj, list) or not len(obj) > 0:

450 - function_name_returned = False

451 - delta_message = None

452 - else:

453 - _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(

454 - delta_text, previous_text

455 - )

456 - # take the last tool call from the generated list

457 - current_tool_call = obj[-1]

458 -

459 - # once parameters have been generated the name is complete as well

460 - if not finishes_previous_tool and (

461 - "name" not in current_tool_call or "parameters" not in current_tool_call

462 - ):

463 - function_name_returned = False

464 - delta_message = None

465 - else:

466 - if not function_name_returned:

467 - # get partly generated arguments from the latest tool call

468 - param_match = re.search(

469 - r'.*"parameters":\s*(.*)', current_text, re.DOTALL

470 - )

471 - arguments = param_match.group(1) if param_match else ""

472 - arguments, _ = OpenAIServingChat._filter_delta_text(

473 - arguments, previous_text

474 - )

475 -

476 - # if this iteration finishes a previous tool call but a

477 - # new incomplete tool is already generated, take the

478 - # previous from the list

479 - if finishes_previous_tool and "parameters" not in current_tool_call:

480 - current_tool_call = obj[-2]

481 -

482 - function_name_returned = True

483 - tool_call_id = make_tool_call_id(

484 - id_type=self.tool_call_id_type,

485 - func_name=current_tool_call["name"],

486 - idx=tool_call_idx,

487 - )

488 - delta_message = DeltaMessage(

489 - tool_calls=[

490 - DeltaToolCall(

491 - id=tool_call_id,

492 - function=DeltaFunctionCall(

493 - name=current_tool_call["name"], arguments=arguments

494 - ),

495 - index=len(obj) - 1,

496 - type="function",

497 - )

498 - ]

499 - )

500 -

501 - else:

502 - delta_text, _ = OpenAIServingChat._filter_delta_text(

503 - delta_text, previous_text

504 - )

505 -

506 - if delta_text != "":

507 - delta_message = DeltaMessage(

508 - tool_calls=[

509 - DeltaToolCall(

510 - function=DeltaFunctionCall(

511 - # OpenAI API returns None

512 - # instead of name every time

513 - name=None,

514 - arguments=delta_text,

515 - ),

516 - index=len(obj) - 1,

517 - )

518 - ]

519 - )

520 - else:

521 - delta_message = None

522 -

523 - return delta_message, function_name_returned

391 + return extract_required_tool_call_streaming(

392 + previous_text=previous_text,

393 + current_text=current_text,

394 + delta_text=delta_text,

395 + function_name_returned=function_name_returned,

396 + tool_call_idx=tool_call_idx,

397 + tool_call_id_type=self.tool_call_id_type,

398 + )

399

400 async def chat_completion_stream_generator(

401 self,

vllm/entrypoints/openai/responses/serving.py

+45 -573

@@ -3,7 +3,6 @@

4 import asyncio

5 import time

6 -import uuid

6 from collections import deque

7 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence

8 from contextlib import AsyncExitStack

@@ -13,29 +12,14 @@ from typing import Any, Final

13 from fastapi import Request

14 from openai.types.responses import (

16 - ResponseContentPartAddedEvent,

17 - ResponseContentPartDoneEvent,

18 - ResponseFunctionCallArgumentsDeltaEvent,

19 - ResponseFunctionCallArgumentsDoneEvent,

15 ResponseFunctionToolCall,

21 - ResponseFunctionToolCallItem,

16 ResponseOutputItem,

23 - ResponseOutputItemAddedEvent,

24 - ResponseOutputItemDoneEvent,

17 ResponseOutputMessage,

18 ResponseOutputText,

27 - ResponseReasoningItem,

28 - ResponseReasoningTextDeltaEvent,

29 - ResponseReasoningTextDoneEvent,

19 ResponseStatus,

31 - ResponseTextDeltaEvent,

32 - ResponseTextDoneEvent,

20 response_text_delta_event,

21 )

22 from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob

36 -from openai.types.responses.response_reasoning_item import (

37 - Content as ResponseReasoningTextContent,

38 -)

23 from openai.types.responses.tool import Mcp, Tool

24 from openai_harmony import Message as OpenAIHarmonyMessage

25 from pydantic import TypeAdapter

@@ -89,15 +73,15 @@ from vllm.entrypoints.openai.responses.protocol import (

73 ResponseInProgressEvent,

74 ResponseInputOutputItem,

75 ResponseInputOutputMessage,

92 - ResponseReasoningPartAddedEvent,

93 - ResponseReasoningPartDoneEvent,

76 ResponsesRequest,

77 ResponsesResponse,

78 ResponseUsage,

79 StreamingResponsesResponse,

80 )

81 from vllm.entrypoints.openai.responses.streaming_events import (

82 + SimpleStreamingEventProcessor,

83 StreamingState,

84 + _StateType,

85 emit_content_delta_events,

86 emit_previous_item_done_events,

87 emit_tool_action_events,

@@ -1353,571 +1337,59 @@ class OpenAIServingResponses(OpenAIServing):

1337 [StreamingResponsesResponse], StreamingResponsesResponse

1338 ],

1339 ) -> AsyncGenerator[StreamingResponsesResponse, None]:

1356 - current_content_index = 0

1357 - current_output_index = 0

1358 - current_item_id = ""

1359 - current_tool_call_index: int | None = None

1340 + processor = SimpleStreamingEventProcessor()

1341 parser = self.parser(tokenizer, request.tools) if self.parser else None

1361 - first_delta_sent = False

1362 - previous_delta_messages: list[DeltaMessage] = []

1342 +

1343 + def _get_logprobs(

1344 + output: CompletionOutput,

1345 + ) -> list[response_text_delta_event.Logprob]:

1346 + if not request.is_include_output_logprobs():

1347 + return []

1348 + return self._create_stream_response_logprobs(

1349 + token_ids=output.token_ids,

1350 + logprobs=output.logprobs,

1351 + tokenizer=tokenizer,

1352 + top_logprobs=request.top_logprobs,

1353 + )

1354 +

1355 async for ctx in result_generator:

1356 assert isinstance(ctx, SimpleContext)

1365 - if ctx.last_output is None:

1357 + if ctx.last_output is None or not ctx.last_output.outputs:

1358 continue

1367 - if ctx.last_output.outputs:

1368 - output = ctx.last_output.outputs[0]

1369 - # finish_reason='error' indicates a retryable error

1370 - self._raise_if_error(output.finish_reason, request.request_id)

1371 - delta_text = output.text

1372 - delta_token_ids = as_list(output.token_ids)

1373 -

1374 - if parser:

1375 - delta_message = parser.parse_delta(

1376 - delta_text=delta_text,

1377 - delta_token_ids=delta_token_ids,

1378 - request=request,

1379 - prompt_token_ids=ctx.last_output.prompt_token_ids,

1380 - )

1381 - else:

1382 - delta_message = DeltaMessage(

1383 - content=output.text,

1384 - )

1385 - if not delta_message:

1386 - continue

1387 - tool_call_item_started = False

1388 - if not first_delta_sent:

1389 - current_item_id = random_uuid()

1390 - if delta_message.tool_calls:

1391 - current_tool_call_id = f"call_{random_uuid()}"

1392 - assert len(delta_message.tool_calls) == 1, (

1393 - "Multiple tool calls in one delta is not supported"

1394 - )

1395 - assert delta_message.tool_calls[0].function is not None, (

1396 - "Tool call without function is not supported"

1397 - )

1398 - assert delta_message.tool_calls[0].function.name is not None, (

1399 - "Tool call without function name is not supported"

1400 - )

1401 - current_tool_call_name = delta_message.tool_calls[

1402 - 0

1403 - ].function.name

1404 - current_tool_call_index = delta_message.tool_calls[0].index

1405 - yield _increment_sequence_number_and_return(

1406 - ResponseOutputItemAddedEvent(

1407 - type="response.output_item.added",

1408 - sequence_number=-1,

1409 - output_index=current_output_index,

1410 - item=ResponseFunctionToolCallItem(

1411 - type="function_call",

1412 - id=current_item_id,

1413 - call_id=current_tool_call_id,

1414 - name=current_tool_call_name,

1415 - arguments="",

1416 - status="in_progress",

1417 - ),

1418 - )

1419 - )

1420 - tool_call_item_started = True

1421 - elif delta_message.reasoning:

1422 - yield _increment_sequence_number_and_return(

1423 - ResponseOutputItemAddedEvent(

1424 - type="response.output_item.added",

1425 - sequence_number=-1,

1426 - output_index=current_output_index,

1427 - item=ResponseReasoningItem(

1428 - type="reasoning",

1429 - id=current_item_id,

1430 - summary=[],

1431 - status="in_progress",

1432 - ),

1433 - )

1434 - )

1435 - yield _increment_sequence_number_and_return(

1436 - ResponseReasoningPartAddedEvent(

1437 - type="response.reasoning_part.added",

1438 - sequence_number=-1,

1439 - output_index=current_output_index,

1440 - item_id=current_item_id,

1441 - content_index=current_content_index,

1442 - part=ResponseReasoningTextContent(

1443 - text="",

1444 - type="reasoning_text",

1445 - ),

1446 - )

1447 - )

1448 - elif not delta_message.tool_calls:

1449 - yield _increment_sequence_number_and_return(

1450 - ResponseOutputItemAddedEvent(

1451 - type="response.output_item.added",

1452 - sequence_number=-1,

1453 - output_index=current_output_index,

1454 - item=ResponseOutputMessage(

1455 - id=current_item_id,

1456 - type="message",

1457 - role="assistant",

1458 - content=[],

1459 - status="in_progress",

1460 - ),

1461 - )

1462 - )

1463 - yield _increment_sequence_number_and_return(

1464 - ResponseContentPartAddedEvent(

1465 - type="response.content_part.added",

1466 - sequence_number=-1,

1467 - output_index=current_output_index,

1468 - item_id=current_item_id,

1469 - content_index=current_content_index,

1470 - part=ResponseOutputText(

1471 - type="output_text",

1472 - text="",

1473 - annotations=[],

1474 - logprobs=[],

1475 - ),

1476 - )

1477 - )

1478 - first_delta_sent = True

1359

1480 - # check delta message and previous delta message are

1481 - # same as content or reasoning content

1482 - if (

1483 - previous_delta_messages

1484 - and previous_delta_messages[-1].reasoning is not None

1485 - and delta_message.content is not None

1486 - ):

1487 - # from reasoning to normal content, send done

1488 - # event for reasoning

1489 - reason_content = "".join(

1490 - pm.reasoning

1491 - for pm in previous_delta_messages

1492 - if pm.reasoning is not None

1493 - )

1360 + output = ctx.last_output.outputs[0]

1361 + self._raise_if_error(output.finish_reason, request.request_id)

1362 + delta_text = output.text

1363 + delta_token_ids = as_list(output.token_ids)

1364 +

1365 + if parser:

1366 + delta_message = parser.parse_delta(

1367 + delta_text=delta_text,

1368 + delta_token_ids=delta_token_ids,

1369 + request=request,

1370 + prompt_token_ids=ctx.last_output.prompt_token_ids,

1371 + )

1372 + else:

1373 + delta_message = DeltaMessage(content=output.text)

1374

1495 - # delta message could have both reasoning and

1496 - # content. Include current delta's reasoning in the

1497 - # finalization since it may carry the tail end of

1498 - # reasoning text (e.g. when reasoning end and

1499 - # content start arrive in the same delta).

1500 - if delta_message.reasoning is not None:

1501 - yield _increment_sequence_number_and_return(

1502 - ResponseReasoningTextDeltaEvent(

1503 - type="response.reasoning_text.delta",

1504 - sequence_number=-1,

1505 - content_index=current_content_index,

1506 - output_index=current_output_index,

1507 - item_id=current_item_id,

1508 - delta=delta_message.reasoning,

1509 - )

1510 - )

1511 - reason_content += delta_message.reasoning

1512 - delta_message = DeltaMessage(content=delta_message.content)

1513 -

1514 - yield _increment_sequence_number_and_return(

1515 - ResponseReasoningTextDoneEvent(

1516 - type="response.reasoning_text.done",

1517 - item_id=current_item_id,

1518 - sequence_number=-1,

1519 - output_index=current_output_index,

1520 - content_index=current_content_index,

1521 - text=reason_content,

1522 - )

1523 - )

1524 - yield _increment_sequence_number_and_return(

1525 - ResponseReasoningPartDoneEvent(

1526 - type="response.reasoning_part.done",

1527 - sequence_number=-1,

1528 - item_id=current_item_id,

1529 - output_index=current_output_index,

1530 - content_index=current_content_index,

1531 - part=ResponseReasoningTextContent(

1532 - text=reason_content,

1533 - type="reasoning_text",

1534 - ),

1535 - )

1536 - )

1537 - current_content_index = 0

1538 - reasoning_item = ResponseReasoningItem(

1539 - type="reasoning",

1540 - content=[

1541 - ResponseReasoningTextContent(

1542 - text=reason_content,

1543 - type="reasoning_text",

1544 - ),

1545 - ],

1546 - status="completed",

1547 - id=current_item_id,

1548 - summary=[],

1549 - )

1550 - yield _increment_sequence_number_and_return(

1551 - ResponseOutputItemDoneEvent(

1552 - type="response.output_item.done",

1553 - sequence_number=-1,

1554 - output_index=current_output_index,

1555 - item=reasoning_item,

1556 - )

1557 - )

1558 - current_output_index += 1

1559 - current_item_id = str(uuid.uuid4())

1560 - yield _increment_sequence_number_and_return(

1561 - ResponseOutputItemAddedEvent(

1562 - type="response.output_item.added",

1563 - sequence_number=-1,

1564 - output_index=current_output_index,

1565 - item=ResponseOutputMessage(

1566 - id=current_item_id,

1567 - type="message",

1568 - role="assistant",

1569 - content=[],

1570 - status="in_progress",

1571 - ),

1572 - )

1573 - )

1574 - yield _increment_sequence_number_and_return(

1575 - ResponseContentPartAddedEvent(

1576 - type="response.content_part.added",

1577 - sequence_number=-1,

1578 - output_index=current_output_index,

1579 - item_id=current_item_id,

1580 - content_index=current_content_index,

1581 - part=ResponseOutputText(

1582 - type="output_text",

1583 - text="",

1584 - annotations=[],

1585 - logprobs=[],

1586 - ),

1587 - )

1588 - )

1589 - # reset previous delta messages

1590 - previous_delta_messages = []

1591 - if delta_message.tool_calls and delta_message.tool_calls[0].function:

1592 - tool_call = delta_message.tool_calls[0]

1593 - tool_call_function = tool_call.function

1594 - if (

1595 - current_tool_call_index is not None

1596 - and tool_call.index is not None

1597 - and tool_call.index != current_tool_call_index

1598 - and tool_call_function is not None

1599 - and tool_call_function.name is not None

1600 - ):

1601 - # From one tool call to another, finalize the previous

1602 - # function-call item before opening the next one.

1603 - parts = []

1604 - for pm in previous_delta_messages:

1605 - if pm.tool_calls:

1606 - previous_tool_call = pm.tool_calls[0]

1607 - if previous_tool_call.function is not None:

1608 - parts.append(

1609 - previous_tool_call.function.arguments or ""

1610 - )

1611 -

1612 - tool_call_arguments = "".join(parts)

1613 - yield _increment_sequence_number_and_return(

1614 - ResponseFunctionCallArgumentsDoneEvent(

1615 - type="response.function_call_arguments.done",

1616 - sequence_number=-1,

1617 - output_index=current_output_index,

1618 - item_id=current_item_id,

1619 - arguments=tool_call_arguments,

1620 - name=current_tool_call_name,

1621 - )

1622 - )

1623 - function_call_item = ResponseFunctionToolCall(

1624 - type="function_call",

1625 - name=current_tool_call_name,

1626 - arguments=tool_call_arguments,

1627 - status="completed",

1628 - id=current_item_id,

1629 - call_id=current_tool_call_id,

1630 - )

1631 - yield _increment_sequence_number_and_return(

1632 - ResponseOutputItemDoneEvent(

1633 - type="response.output_item.done",

1634 - sequence_number=-1,

1635 - output_index=current_output_index,

1636 - item=function_call_item,

1637 - )

1638 - )

1639 - # Reset previous delta messages so the next tool call

1640 - # does not reuse arguments from the completed item.

1641 - previous_delta_messages = []

1642 - current_output_index += 1

1643 - current_item_id = random_uuid()

1644 - current_tool_call_name = tool_call_function.name

1645 - current_tool_call_id = f"call_{random_uuid()}"

1646 - current_tool_call_index = tool_call.index

1647 - yield _increment_sequence_number_and_return(

1648 - ResponseOutputItemAddedEvent(

1649 - type="response.output_item.added",

1650 - sequence_number=-1,

1651 - output_index=current_output_index,

1652 - item=ResponseFunctionToolCallItem(

1653 - type="function_call",

1654 - id=current_item_id,

1655 - call_id=current_tool_call_id,

1656 - name=current_tool_call_name,

1657 - arguments="",

1658 - status="in_progress",

1659 - ),

1660 - )

1661 - )

1662 - current_content_index = 0

1663 - tool_call_item_started = True

1664 -

1665 - if delta_message.tool_calls[0].function.arguments:

1666 - yield _increment_sequence_number_and_return(

1667 - ResponseFunctionCallArgumentsDeltaEvent(

1668 - type="response.function_call_arguments.delta",

1669 - sequence_number=-1,

1670 - output_index=current_output_index,

1671 - item_id=current_item_id,

1672 - delta=delta_message.tool_calls[0].function.arguments,

1673 - )

1674 - )

1675 - # tool call initiated with no arguments

1676 - elif (

1677 - delta_message.tool_calls[0].function.name

1678 - and not tool_call_item_started

1679 - ):

1680 - # send done with current content part

1681 - # and add new function call item

1682 - yield _increment_sequence_number_and_return(

1683 - ResponseTextDoneEvent(

1684 - type="response.output_text.done",

1685 - sequence_number=-1,

1686 - output_index=current_output_index,

1687 - content_index=current_content_index,

1688 - text="",

1689 - logprobs=[],

1690 - item_id=current_item_id,

1691 - )

1692 - )

1693 - yield _increment_sequence_number_and_return(

1694 - ResponseContentPartDoneEvent(

1695 - type="response.content_part.done",

1696 - sequence_number=-1,

1697 - item_id=current_item_id,

1698 - output_index=current_output_index,

1699 - content_index=current_content_index,

1700 - part=ResponseOutputText(

1701 - type="output_text",

1702 - text="",

1703 - annotations=[],

1704 - logprobs=[],

1705 - ),

1706 - )

1707 - )

1708 - yield _increment_sequence_number_and_return(

1709 - ResponseOutputItemDoneEvent(

1710 - type="response.output_item.done",

1711 - sequence_number=-1,

1712 - output_index=current_output_index,

1713 - item=ResponseOutputMessage(

1714 - id=current_item_id,

1715 - type="message",

1716 - role="assistant",

1717 - content=[],

1718 - status="completed",

1719 - ),

1720 - )

1721 - )

1722 - current_output_index += 1

1723 - current_item_id = random_uuid()

1724 - current_tool_call_name = delta_message.tool_calls[

1725 - 0

1726 - ].function.name

1727 - current_tool_call_id = f"call_{random_uuid()}"

1728 - current_tool_call_index = delta_message.tool_calls[0].index

1729 - yield _increment_sequence_number_and_return(

1730 - ResponseOutputItemAddedEvent(

1731 - type="response.output_item.added",

1732 - sequence_number=-1,

1733 - output_index=current_output_index,

1734 - item=ResponseFunctionToolCallItem(

1735 - type="function_call",

1736 - id=current_item_id,

1737 - call_id=current_tool_call_id,

1738 - name=current_tool_call_name,

1739 - arguments="",

1740 - status="in_progress",

1741 - ),

1742 - )

1743 - )

1744 - # skip content part for tool call

1745 - current_content_index = 1

1746 - continue

1747 - elif delta_message.reasoning is not None:

1748 - yield _increment_sequence_number_and_return(

1749 - ResponseReasoningTextDeltaEvent(

1750 - type="response.reasoning_text.delta",

1751 - sequence_number=-1,

1752 - content_index=current_content_index,

1753 - output_index=current_output_index,

1754 - item_id=current_item_id,

1755 - delta=delta_message.reasoning,

1756 - )

1757 - )

1758 - elif delta_message.content:

1759 - yield _increment_sequence_number_and_return(

1760 - ResponseTextDeltaEvent(

1761 - type="response.output_text.delta",

1762 - sequence_number=-1,

1763 - content_index=current_content_index,

1764 - output_index=current_output_index,

1765 - item_id=current_item_id,

1766 - delta=delta_message.content,

1767 - logprobs=(

1768 - self._create_stream_response_logprobs(

1769 - token_ids=output.token_ids,

1770 - logprobs=output.logprobs,

1771 - tokenizer=tokenizer,

1772 - top_logprobs=request.top_logprobs,

1773 - )

1774 - if request.is_include_output_logprobs()

1775 - else []

1776 - ),

1777 - )

1778 - )

1375 + if not delta_message:

1376 + continue

1377

1780 - previous_delta_messages.append(delta_message)

1378 + target_state, tool_call = processor.resolve_target_state(delta_message)

1379 + if target_state == _StateType.NONE:

1380 + continue

1381

1782 - if previous_delta_messages:

1783 - parts = []

1784 - for pm in previous_delta_messages:

1785 - if pm.tool_calls:

1786 - assert len(pm.tool_calls) == 1, (

1787 - "Multiple tool calls in one delta is not supported"

1788 - )

1789 - assert pm.tool_calls[0].function is not None, (

1790 - "Tool call without function is not supported"

1791 - )

1792 - parts.append(pm.tool_calls[0].function.arguments or "")

1382 + if processor.needs_transition(target_state, tool_call):

1383 + for event in processor.close_current():

1384 + yield _increment_sequence_number_and_return(event)

1385 + for event in processor.open(target_state, tool_call):

1386 + yield _increment_sequence_number_and_return(event)

1387

1794 - tool_call_arguments = "".join(parts)

1795 - if tool_call_arguments:

1796 - yield _increment_sequence_number_and_return(

1797 - ResponseFunctionCallArgumentsDoneEvent(

1798 - type="response.function_call_arguments.done",

1799 - sequence_number=-1,

1800 - output_index=current_output_index,

1801 - item_id=current_item_id,

1802 - arguments=tool_call_arguments,

1803 - name=current_tool_call_name,

1804 - )

1805 - )

1806 - current_content_index = 0

1807 - function_call_item = ResponseFunctionToolCall(

1808 - type="function_call",

1809 - name=current_tool_call_name,

1810 - arguments=tool_call_arguments,

1811 - status="completed",

1812 - id=current_item_id,

1813 - call_id=current_tool_call_id,

1814 - )

1815 - yield _increment_sequence_number_and_return(

1816 - ResponseOutputItemDoneEvent(

1817 - type="response.output_item.done",

1818 - sequence_number=-1,

1819 - output_index=current_output_index,

1820 - item=function_call_item,

1821 - )

1822 - )

1388 + for event in processor.emit_delta(delta_message, output, _get_logprobs):

1389 + yield _increment_sequence_number_and_return(event)

1390

1824 - elif previous_delta_messages[-1].reasoning is not None:

1825 - reason_content = "".join(

1826 - pm.reasoning

1827 - for pm in previous_delta_messages

1828 - if pm.reasoning is not None

1829 - )

1830 - yield _increment_sequence_number_and_return(

1831 - ResponseReasoningTextDoneEvent(

1832 - type="response.reasoning_text.done",

1833 - item_id=current_item_id,

1834 - sequence_number=-1,

1835 - output_index=current_output_index,

1836 - content_index=current_content_index,

1837 - text=reason_content,

1838 - )

1839 - )

1840 - yield _increment_sequence_number_and_return(

1841 - ResponseReasoningPartDoneEvent(

1842 - type="response.reasoning_part.done",

1843 - sequence_number=-1,

1844 - item_id=current_item_id,

1845 - output_index=current_output_index,

1846 - content_index=current_content_index,

1847 - part=ResponseReasoningTextContent(

1848 - text=reason_content,

1849 - type="reasoning_text",

1850 - ),

1851 - )

1852 - )

1853 - reasoning_item = ResponseReasoningItem(

1854 - type="reasoning",

1855 - content=[

1856 - ResponseReasoningTextContent(

1857 - text=reason_content,

1858 - type="reasoning_text",

1859 - ),

1860 - ],

1861 - status="completed",

1862 - id=current_item_id,

1863 - summary=[],

1864 - )

1865 - yield _increment_sequence_number_and_return(

1866 - ResponseOutputItemDoneEvent(

1867 - type="response.output_item.done",

1868 - sequence_number=-1,

1869 - output_index=current_output_index,

1870 - item=reasoning_item,

1871 - )

1872 - )

1873 - elif previous_delta_messages[-1].content:

1874 - final_content = "".join(

1875 - pm.content for pm in previous_delta_messages if pm.content

1876 - )

1877 - yield _increment_sequence_number_and_return(

1878 - ResponseTextDoneEvent(

1879 - type="response.output_text.done",

1880 - sequence_number=-1,

1881 - output_index=current_output_index,

1882 - content_index=current_content_index,

1883 - text=final_content,

1884 - logprobs=[],

1885 - item_id=current_item_id,

1886 - )

1887 - )

1888 - part = ResponseOutputText(

1889 - text=final_content,

1890 - type="output_text",

1891 - annotations=[],

1892 - )

1893 - yield _increment_sequence_number_and_return(

1894 - ResponseContentPartDoneEvent(

1895 - type="response.content_part.done",

1896 - sequence_number=-1,

1897 - item_id=current_item_id,

1898 - output_index=current_output_index,

1899 - content_index=current_content_index,

1900 - part=part,

1901 - )

1902 - )

1903 - item = ResponseOutputMessage(

1904 - type="message",

1905 - role="assistant",

1906 - content=[

1907 - part,

1908 - ],

1909 - status="completed",

1910 - id=current_item_id,

1911 - summary=[],

1912 - )

1913 - yield _increment_sequence_number_and_return(

1914 - ResponseOutputItemDoneEvent(

1915 - type="response.output_item.done",

1916 - sequence_number=-1,

1917 - output_index=current_output_index,

1918 - item=item,

1919 - )

1920 - )

1391 + for event in processor.close_current():

1392 + yield _increment_sequence_number_and_return(event)

1393

1394 async def _process_harmony_streaming_events(

1395 self,

vllm/entrypoints/openai/responses/streaming_events.py

+462 -2

@@ -16,8 +16,10 @@ The file is organized as:

16 """

18 import json

19 -from dataclasses import dataclass

20 -from typing import Final

19 +from collections.abc import Callable

20 +from dataclasses import dataclass, field

21 +from enum import Enum, auto

22 +from typing import Any, ClassVar, Final, NamedTuple

24 from openai.types.responses import (

25 ResponseCodeInterpreterCallCodeDeltaEvent,

@@ -31,6 +33,7 @@ from openai.types.responses import (

33 ResponseFunctionCallArgumentsDeltaEvent,

34 ResponseFunctionCallArgumentsDoneEvent,

35 ResponseFunctionToolCall,

36 + ResponseFunctionToolCallItem,

37 ResponseFunctionWebSearch,

38 ResponseMcpCallArgumentsDeltaEvent,

39 ResponseMcpCallArgumentsDoneEvent,

@@ -49,6 +52,7 @@ from openai.types.responses import (

52 ResponseWebSearchCallInProgressEvent,

53 ResponseWebSearchCallSearchingEvent,

54 response_function_web_search,

55 + response_text_delta_event,

56 )

57 from openai.types.responses.response_output_item import McpCall

58 from openai.types.responses.response_reasoning_item import (

@@ -57,12 +61,14 @@ from openai.types.responses.response_reasoning_item import (

61 from openai_harmony import Message as HarmonyMessage

63 from vllm.entrypoints.mcp.tool_server import ToolServer

64 +from vllm.entrypoints.openai.engine.protocol import DeltaMessage

65 from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext

66 from vllm.entrypoints.openai.responses.protocol import (

67 ResponseReasoningPartAddedEvent,

68 ResponseReasoningPartDoneEvent,

69 StreamingResponsesResponse,

70 )

71 +from vllm.outputs import CompletionOutput

72 from vllm.utils import random_uuid

74 TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {

@@ -796,3 +802,457 @@ def emit_tool_action_events(

802 )

803

804 return events

805 +

806 +

807 +# =====================================================================

808 +# Simple streaming helpers

809 +# =====================================================================

810 +

811 +

812 +class _StateType(Enum):

813 + NONE = auto()

814 + CONTENT = auto()

815 + REASONING = auto()

816 + TOOL_CALL = auto()

817 +

818 +

819 +@dataclass

820 +class SimpleStreamingState:

821 + output_index: int = 0

822 + current_item_id: str = ""

823 + content_index: int = 0

824 + accumulated_text: str = ""

825 + tool_call_id: str = ""

826 + tool_call_name: str = ""

827 + tool_call_index: int | None = None

828 + has_emitted_tool_call_delta: bool = False

829 + current_state: _StateType = field(default_factory=lambda: _StateType.NONE)

830 +

831 +

832 +def emit_simple_content_open(

833 + state: SimpleStreamingState,

834 +) -> list[StreamingResponsesResponse]:

835 + state.current_state = _StateType.CONTENT

836 + state.current_item_id = random_uuid()

837 + state.content_index = 0

838 + state.accumulated_text = ""

839 + return [

840 + ResponseOutputItemAddedEvent(

841 + type="response.output_item.added",

842 + sequence_number=-1,

843 + output_index=state.output_index,

844 + item=ResponseOutputMessage(

845 + id=state.current_item_id,

846 + type="message",

847 + role="assistant",

848 + content=[],

849 + status="in_progress",

850 + ),

851 + ),

852 + ResponseContentPartAddedEvent(

853 + type="response.content_part.added",

854 + sequence_number=-1,

855 + output_index=state.output_index,

856 + item_id=state.current_item_id,

857 + content_index=state.content_index,

858 + part=ResponseOutputText(

859 + type="output_text",

860 + text="",

861 + annotations=[],

862 + logprobs=[],

863 + ),

864 + ),

865 + ]

866 +

867 +

868 +def emit_simple_content_delta(

869 + state: SimpleStreamingState,

870 + delta: str,

871 + logprobs: list[response_text_delta_event.Logprob] | None = None,

872 +) -> list[StreamingResponsesResponse]:

873 + state.accumulated_text += delta

874 + return [

875 + ResponseTextDeltaEvent(

876 + type="response.output_text.delta",

877 + sequence_number=-1,

878 + content_index=state.content_index,

879 + output_index=state.output_index,

880 + item_id=state.current_item_id,

881 + delta=delta,

882 + logprobs=logprobs or [],

883 + )

884 + ]

885 +

886 +

887 +def emit_simple_content_done(

888 + state: SimpleStreamingState,

889 +) -> list[StreamingResponsesResponse]:

890 + part = ResponseOutputText(

891 + type="output_text",

892 + text=state.accumulated_text,

893 + annotations=[],

894 + )

895 + events: list[StreamingResponsesResponse] = [

896 + ResponseTextDoneEvent(

897 + type="response.output_text.done",

898 + sequence_number=-1,

899 + output_index=state.output_index,

900 + content_index=state.content_index,

901 + text=state.accumulated_text,

902 + logprobs=[],

903 + item_id=state.current_item_id,

904 + ),

905 + ResponseContentPartDoneEvent(

906 + type="response.content_part.done",

907 + sequence_number=-1,

908 + item_id=state.current_item_id,

909 + output_index=state.output_index,

910 + content_index=state.content_index,

911 + part=part,

912 + ),

913 + ResponseOutputItemDoneEvent(

914 + type="response.output_item.done",

915 + sequence_number=-1,

916 + output_index=state.output_index,

917 + item=ResponseOutputMessage(

918 + id=state.current_item_id,

919 + type="message",

920 + role="assistant",

921 + content=[part] if state.accumulated_text else [],

922 + status="completed",

923 + summary=[],

924 + ),

925 + ),

926 + ]

927 + state.output_index += 1

928 + state.current_state = _StateType.NONE

929 + return events

930 +

931 +

932 +def emit_simple_reasoning_open(

933 + state: SimpleStreamingState,

934 +) -> list[StreamingResponsesResponse]:

935 + state.current_state = _StateType.REASONING

936 + state.current_item_id = random_uuid()

937 + state.content_index = 0

938 + state.accumulated_text = ""

939 + return [

940 + ResponseOutputItemAddedEvent(

941 + type="response.output_item.added",

942 + sequence_number=-1,

943 + output_index=state.output_index,

944 + item=ResponseReasoningItem(

945 + type="reasoning",

946 + id=state.current_item_id,

947 + summary=[],

948 + status="in_progress",

949 + ),

950 + ),

951 + ResponseReasoningPartAddedEvent(

952 + type="response.reasoning_part.added",

953 + sequence_number=-1,

954 + output_index=state.output_index,

955 + item_id=state.current_item_id,

956 + content_index=state.content_index,

957 + part=ResponseReasoningTextContent(

958 + text="",

959 + type="reasoning_text",

960 + ),

961 + ),

962 + ]

963 +

964 +

965 +def emit_simple_reasoning_delta(

966 + state: SimpleStreamingState,

967 + delta: str,

968 +) -> list[StreamingResponsesResponse]:

969 + state.accumulated_text += delta

970 + return [

971 + ResponseReasoningTextDeltaEvent(

972 + type="response.reasoning_text.delta",

973 + item_id=state.current_item_id,

974 + sequence_number=-1,

975 + output_index=state.output_index,

976 + content_index=state.content_index,

977 + delta=delta,

978 + )

979 + ]

980 +

981 +

982 +def emit_simple_reasoning_done(

983 + state: SimpleStreamingState,

984 +) -> list[StreamingResponsesResponse]:

985 + part = ResponseReasoningTextContent(

986 + text=state.accumulated_text,

987 + type="reasoning_text",

988 + )

989 + events: list[StreamingResponsesResponse] = [

990 + ResponseReasoningTextDoneEvent(

991 + type="response.reasoning_text.done",

992 + item_id=state.current_item_id,

993 + sequence_number=-1,

994 + output_index=state.output_index,

995 + content_index=state.content_index,

996 + text=state.accumulated_text,

997 + ),

998 + ResponseReasoningPartDoneEvent(

999 + type="response.reasoning_part.done",

1000 + sequence_number=-1,

1001 + item_id=state.current_item_id,

1002 + output_index=state.output_index,

1003 + content_index=state.content_index,

1004 + part=part,

1005 + ),

1006 + ResponseOutputItemDoneEvent(

1007 + type="response.output_item.done",

1008 + sequence_number=-1,

1009 + output_index=state.output_index,

1010 + item=ResponseReasoningItem(

1011 + type="reasoning",

1012 + content=[part],

1013 + status="completed",

1014 + id=state.current_item_id,

1015 + summary=[],

1016 + ),

1017 + ),

1018 + ]

1019 + state.output_index += 1

1020 + state.current_state = _StateType.NONE

1021 + return events

1022 +

1023 +

1024 +def emit_simple_tool_call_open(

1025 + state: SimpleStreamingState,

1026 + name: str,

1027 + index: int | None,

1028 +) -> list[StreamingResponsesResponse]:

1029 + state.current_state = _StateType.TOOL_CALL

1030 + state.current_item_id = random_uuid()

1031 + state.tool_call_id = f"call_{random_uuid()}"

1032 + state.tool_call_name = name

1033 + state.tool_call_index = index

1034 + state.accumulated_text = ""

1035 + state.has_emitted_tool_call_delta = False

1036 + return [

1037 + ResponseOutputItemAddedEvent(

1038 + type="response.output_item.added",

1039 + sequence_number=-1,

1040 + output_index=state.output_index,

1041 + item=ResponseFunctionToolCallItem(

1042 + type="function_call",

1043 + id=state.current_item_id,

1044 + call_id=state.tool_call_id,

1045 + name=name,

1046 + arguments="",

1047 + status="in_progress",

1048 + ),

1049 + ),

1050 + ]

1051 +

1052 +

1053 +def emit_simple_tool_call_delta(

1054 + state: SimpleStreamingState,

1055 + delta: str,

1056 +) -> list[StreamingResponsesResponse]:

1057 + state.accumulated_text += delta

1058 + state.has_emitted_tool_call_delta = True

1059 + return [

1060 + ResponseFunctionCallArgumentsDeltaEvent(

1061 + type="response.function_call_arguments.delta",

1062 + sequence_number=-1,

1063 + output_index=state.output_index,

1064 + item_id=state.current_item_id,

1065 + delta=delta,

1066 + )

1067 + ]

1068 +

1069 +

1070 +def emit_simple_tool_call_done(

1071 + state: SimpleStreamingState,

1072 +) -> list[StreamingResponsesResponse]:

1073 + events: list[StreamingResponsesResponse] = []

1074 + if state.has_emitted_tool_call_delta:

1075 + events.append(

1076 + ResponseFunctionCallArgumentsDoneEvent(

1077 + type="response.function_call_arguments.done",

1078 + sequence_number=-1,

1079 + output_index=state.output_index,

1080 + item_id=state.current_item_id,

1081 + arguments=state.accumulated_text,

1082 + name=state.tool_call_name,

1083 + )

1084 + )

1085 + events.append(

1086 + ResponseOutputItemDoneEvent(

1087 + type="response.output_item.done",

1088 + sequence_number=-1,

1089 + output_index=state.output_index,

1090 + item=ResponseFunctionToolCall(

1091 + type="function_call",

1092 + name=state.tool_call_name,

1093 + arguments=state.accumulated_text,

1094 + status="completed",

1095 + id=state.current_item_id,

1096 + call_id=state.tool_call_id,

1097 + ),

1098 + ),

1099 + )

1100 + state.output_index += 1

1101 + state.current_state = _StateType.NONE

1102 + return events

1103 +

1104 +

1105 +class _StateHandlers(NamedTuple):

1106 + """Tuple for each state: open(start), delta(chunk), done(finish)."""

1107 +

1108 + open_fn: Callable[..., list[StreamingResponsesResponse]]

1109 + delta_fn: Callable[..., list[StreamingResponsesResponse]]

1110 + done_fn: Callable[..., list[StreamingResponsesResponse]]

1111 +

1112 +

1113 +class SimpleStreamingEventProcessor:

1114 + """

1115 + State-machine processor for the simple (non-Harmony) streaming path.

1116 +

1117 + Core flow:

1118 + 1. Resolve the target state from the delta_message

1119 + (CONTENT / REASONING / TOOL_CALL).

1120 + 2. If the target state differs from the current one,

1121 + close_current() then open() the new state.

1122 + 3. emit_delta() produces the incremental events for the state.

1123 +

1124 + State lifecycle:

1125 + open() -> repeated emit_delta() -> close_current()

1126 + """

1127 +

1128 + _STATE_HANDLERS: ClassVar[dict[_StateType, _StateHandlers]] = {

1129 + _StateType.CONTENT: _StateHandlers(

1130 + emit_simple_content_open,

1131 + emit_simple_content_delta,

1132 + emit_simple_content_done,

1133 + ),

1134 + _StateType.REASONING: _StateHandlers(

1135 + emit_simple_reasoning_open,

1136 + emit_simple_reasoning_delta,

1137 + emit_simple_reasoning_done,

1138 + ),

1139 + _StateType.TOOL_CALL: _StateHandlers(

1140 + emit_simple_tool_call_open,

1141 + emit_simple_tool_call_delta,

1142 + emit_simple_tool_call_done,

1143 + ),

1144 + }

1145 +

1146 + def __init__(self, state: SimpleStreamingState | None = None) -> None:

1147 + self.state = state or SimpleStreamingState()

1148 +

1149 + def resolve_target_state(

1150 + self, delta_message: DeltaMessage

1151 + ) -> tuple[_StateType, Any]:

1152 + """

1153 + Decide which state the next delta belongs to.

1154 +

1155 + Priority: TOOL_CALL > REASONING > CONTENT, fallback to NONE.

1156 + For TOOL_CALL the first tool_call object is also returned so

1157 + callers can detect a switch between consecutive tools.

1158 + """

1159 + if (

1160 + delta_message.tool_calls

1161 + and delta_message.tool_calls[0].function is not None

1162 + ):

1163 + return _StateType.TOOL_CALL, delta_message.tool_calls[0]

1164 + if delta_message.reasoning is not None:

1165 + return _StateType.REASONING, None

1166 + if delta_message.content:

1167 + return _StateType.CONTENT, None

1168 + return _StateType.NONE, None

1169 +

1170 + def needs_transition(self, target_state: _StateType, tool_call: Any) -> bool:

1171 + """

1172 + Return True when we must close the current state and open a new one.

1173 +

1174 + Two cases trigger a transition:

1175 + 1. The target state differs from the current state

1176 + (e.g. CONTENT -> TOOL_CALL).

1177 + 2. We are already in TOOL_CALL but the next tool_call has a

1178 + different index (multiple consecutive tool calls).

1179 + """

1180 + if self.state.current_state != target_state:

1181 + return True

1182 + return (

1183 + target_state == _StateType.TOOL_CALL

1184 + and tool_call is not None

1185 + and self.state.tool_call_index is not None

1186 + and tool_call.index is not None

1187 + and self.state.tool_call_index != tool_call.index

1188 + )

1189 +

1190 + def close_current(self) -> list[StreamingResponsesResponse]:

1191 + """Close the current state and emit its 'done' event sequence."""

1192 + handlers = self._STATE_HANDLERS.get(self.state.current_state)

1193 + if handlers is None:

1194 + return []

1195 + return handlers.done_fn(self.state)

1196 +

1197 + def open(

1198 + self, target_state: _StateType, tool_call: Any = None

1199 + ) -> list[StreamingResponsesResponse]:

1200 + """Open a new state and emit its 'added' / 'open' event sequence."""

1201 + handlers = self._STATE_HANDLERS[target_state]

1202 + if target_state == _StateType.TOOL_CALL:

1203 + assert tool_call is not None

1204 + return handlers.open_fn(

1205 + self.state, tool_call.function.name, tool_call.index

1206 + )

1207 + return handlers.open_fn(self.state)

1208 +

1209 + def emit_delta(

1210 + self,

1211 + delta_message: DeltaMessage,

1212 + output: CompletionOutput,

1213 + get_logprobs: Callable[

1214 + [CompletionOutput], list[response_text_delta_event.Logprob]

1215 + ]

1216 + | None = None,

1217 + ) -> list[StreamingResponsesResponse]:

1218 + """

1219 + Emit incremental events for the current state from the delta.

1220 +

1221 + Special case: when already in REASONING and the same delta also

1222 + carries content, we emit the reasoning delta, close reasoning,

1223 + open content, and then emit the content delta.

1224 + """

1225 + handlers = self._STATE_HANDLERS[self.state.current_state]

1226 + events: list[StreamingResponsesResponse] = []

1227 +

1228 + # Special case: reasoning -> content inside a single delta.

1229 + if (

1230 + self.state.current_state == _StateType.REASONING

1231 + and delta_message.reasoning is not None

1232 + and delta_message.content is not None

1233 + ):

1234 + events.extend(handlers.delta_fn(self.state, delta_message.reasoning))

1235 + events.extend(self.close_current())

1236 + events.extend(self.open(_StateType.CONTENT))

1237 + content_handlers = self._STATE_HANDLERS[_StateType.CONTENT]

1238 + logprobs = get_logprobs(output) if get_logprobs else []

1239 + events.extend(

1240 + content_handlers.delta_fn(self.state, delta_message.content, logprobs)

1241 + )

1242 + return events

1243 +

1244 + if self.state.current_state == _StateType.TOOL_CALL:

1245 + assert delta_message.tool_calls is not None

1246 + tool_call_function = delta_message.tool_calls[0].function

1247 + assert tool_call_function is not None

1248 + if tool_call_function.arguments:

1249 + return handlers.delta_fn(self.state, tool_call_function.arguments)

1250 + return []

1251 + elif self.state.current_state == _StateType.REASONING:

1252 + assert delta_message.reasoning is not None

1253 + return handlers.delta_fn(self.state, delta_message.reasoning)

1254 + elif self.state.current_state == _StateType.CONTENT:

1255 + assert delta_message.content is not None

1256 + logprobs = get_logprobs(output) if get_logprobs else []

1257 + return handlers.delta_fn(self.state, delta_message.content, logprobs)

1258 + return []

vllm/parser/abstract_parser.py

+61 -8

@@ -38,6 +38,7 @@ from vllm.logger import init_logger

38 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

39 from vllm.tokenizers import TokenizerLike

40 from vllm.tool_parsers.abstract_tool_parser import ToolParser

41 +from vllm.tool_parsers.streaming import extract_required_tool_call_streaming

42 from vllm.tool_parsers.utils import Tool

43 from vllm.utils import random_uuid

@@ -53,6 +54,11 @@ class StreamState:

54 prompt_reasoning_checked: bool = False

55 previous_text: str = ""

56 previous_token_ids: list[int] = field(default_factory=list)

57 + history_tool_call_cnt: int = 0

58 + tool_call_id_type: str = "random"

59 + # only used for "required" and "named tool" choices,

60 + # tracks whether function name has been fully returned in the stream yet

61 + function_name_returned: bool = False

64 class Parser:

@@ -558,6 +564,47 @@ class DelegatingParser(Parser):

564 request,

565 )

566

567 + def _extract_tool_calls_streaming(

568 + self,

569 + previous_text: str,

570 + current_text: str,

571 + delta_text: str,

572 + previous_token_ids: Sequence[int],

573 + current_token_ids: Sequence[int],

574 + delta_token_ids: Sequence[int],

575 + request: ChatCompletionRequest,

576 + # The following parameters are used for "required" tool choice parsing and are

577 + # tracked in StreamState for streaming parsing.

578 + tool_call_idx: int | None = None,

579 + tool_call_id_type: str = "random",

580 + function_name_returned: bool = False,

581 + ) -> tuple[DeltaMessage | None, bool]:

582 + if request.tool_choice and isinstance(

583 + request.tool_choice, ChatCompletionNamedToolChoiceParam

584 + ):

585 + return None, False

586 + if request.tool_choice == "required":

587 + delta_message, function_name_returned = (

588 + extract_required_tool_call_streaming(

589 + previous_text=previous_text,

590 + current_text=current_text,

591 + delta_text=delta_text,

592 + function_name_returned=function_name_returned,

593 + tool_call_idx=tool_call_idx,

594 + tool_call_id_type=tool_call_id_type,

595 + )

596 + )

597 + return delta_message, function_name_returned

598 + return self.extract_tool_calls_streaming(

599 + previous_text,

600 + current_text,

601 + delta_text,

602 + previous_token_ids,

603 + current_token_ids,

604 + delta_token_ids,

605 + request,

606 + ), False

607 +

608 def is_reasoning_end(self, input_ids: list[int]) -> bool:

609 if self._reasoning_parser is None:

610 return False

@@ -628,14 +675,20 @@ class DelegatingParser(Parser):

675 state.previous_token_ids = []

676 delta_text = current_text

677 delta_token_ids = current_token_ids

631 - delta_message = self.extract_tool_calls_streaming(

632 - previous_text=state.previous_text,

633 - current_text=current_text,

634 - delta_text=delta_text,

635 - previous_token_ids=state.previous_token_ids,

636 - current_token_ids=current_token_ids,

637 - delta_token_ids=delta_token_ids,

638 - request=request, # type: ignore[arg-type]

678 +

679 + delta_message, state.function_name_returned = (

680 + self._extract_tool_calls_streaming(

681 + previous_text=state.previous_text,

682 + current_text=current_text,

683 + delta_text=delta_text,

684 + previous_token_ids=state.previous_token_ids,

685 + current_token_ids=current_token_ids,

686 + delta_token_ids=delta_token_ids,

687 + request=request, # type: ignore[arg-type]

688 + tool_call_idx=state.history_tool_call_cnt,

689 + tool_call_id_type=state.tool_call_id_type,

690 + function_name_returned=state.function_name_returned,

691 + )

692 )

693

694 # No parsers: pass through as content

vllm/tool_parsers/streaming.py new

+199

@@ -0,0 +1,199 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +import json

5 +from typing import TYPE_CHECKING

6 +

7 +import partial_json_parser

8 +import regex as re

9 +from partial_json_parser.core.options import Allow

10 +

11 +from vllm.entrypoints.chat_utils import make_tool_call_id

12 +from vllm.entrypoints.openai.engine.protocol import (

13 + DeltaFunctionCall,

14 + DeltaMessage,

15 + DeltaToolCall,

16 +)

17 +from vllm.tool_parsers.mistral_tool_parser import MistralToolCall

18 +from vllm.tool_parsers.utils import partial_json_loads

19 +from vllm.utils.mistral import is_mistral_tokenizer

20 +

21 +if TYPE_CHECKING:

22 + from vllm.tokenizers import TokenizerLike

23 +else:

24 + TokenizerLike = object

25 +

26 +

27 +def _bracket_level(s: str, opening: str = "{", closing: str = "}") -> int:

28 + """Calculate the current level of nested brackets in a string."""

29 + level = 0

30 + for char in s:

31 + if char == opening:

32 + level += 1

33 + elif char == closing:

34 + level -= 1

35 + return level

36 +

37 +

38 +def filter_delta_text(

39 + delta_text: str,

40 + previous_text: str,

41 +) -> tuple[str, bool]:

42 + """Trim trailing tool-list delimiters from required-tool streaming text."""

43 + bracket_level = _bracket_level(previous_text)

44 + updated_delta = ""

45 + passed_zero = False

46 + for char in delta_text:

47 + if char == "{":

48 + bracket_level += 1

49 + passed_zero = bracket_level == 0

50 + elif char == "}":

51 + bracket_level -= 1

52 + passed_zero = bracket_level == 0

53 +

54 + if bracket_level != 0:

55 + updated_delta += char

56 + else:

57 + if char == ",":

58 + break

59 + return updated_delta, passed_zero

60 +

61 +

62 +def extract_named_tool_call_streaming(

63 + *,

64 + delta_text: str,

65 + function_name: str,

66 + function_name_returned: bool,

67 + tool_call_idx: int | None,

68 + tool_call_id_type: str,

69 + tokenizer: "TokenizerLike",

70 + tool_call_array_index: int,

71 +) -> tuple[DeltaMessage, bool, bool]:

72 + """Build a streaming tool-call delta for forced named tool choice."""

73 + created_new_tool_call = False

74 + if function_name_returned:

75 + delta_tool_call = DeltaToolCall(

76 + function=DeltaFunctionCall(arguments=delta_text),

77 + index=tool_call_array_index,

78 + )

79 + else:

80 + if is_mistral_tokenizer(tokenizer):

81 + tool_call_id = MistralToolCall.generate_random_id()

82 + else:

83 + tool_call_id = make_tool_call_id(

84 + id_type=tool_call_id_type,

85 + func_name=function_name,

86 + idx=tool_call_idx,

87 + )

88 + delta_tool_call = DeltaToolCall(

89 + id=tool_call_id,

90 + type="function",

91 + function=DeltaFunctionCall(

92 + name=function_name,

93 + arguments=delta_text,

94 + ),

95 + index=tool_call_array_index,

96 + )

97 + function_name_returned = True

98 + created_new_tool_call = True

99 +

100 + return (

101 + DeltaMessage(tool_calls=[delta_tool_call]),

102 + function_name_returned,

103 + created_new_tool_call,

104 + )

105 +

106 +

107 +def extract_required_tool_call_streaming(

108 + *,

109 + previous_text: str,

110 + current_text: str | None,

111 + delta_text: str,

112 + function_name_returned: bool,

113 + tool_call_idx: int | None,

114 + tool_call_id_type: str,

115 +) -> tuple[DeltaMessage | None, bool]:

116 + if current_text is None or current_text == "":

117 + # if the current text is empty, we cannot parse it

118 + return None, function_name_returned

119 + try:

120 + flags = Allow.ALL

121 + obj, _ = partial_json_loads(current_text, flags)

122 + except (

123 + partial_json_parser.core.exceptions.MalformedJSON,

124 + json.JSONDecodeError,

125 + ):

126 + obj = None

127 +

128 + # check if the current text is a valid array

129 + # containing a partial tool calling object

130 + # if not repeat

131 + if obj is None or not isinstance(obj, list) or not len(obj) > 0:

132 + function_name_returned = False

133 + delta_message = None

134 + else:

135 + _, finishes_previous_tool = filter_delta_text(delta_text, previous_text)

136 + # take the last tool call from the generated list

137 + current_tool_call = obj[-1]

138 +

139 + # once parameters have been generated the name is complete as well

140 + if not finishes_previous_tool and (

141 + "name" not in current_tool_call or "parameters" not in current_tool_call

142 + ):

143 + function_name_returned = False

144 + delta_message = None

145 + else:

146 + if not function_name_returned:

147 + # get partly generated arguments from the latest tool call

148 + param_match = re.search(

149 + r'.*"parameters":\s*(.*)', current_text, re.DOTALL

150 + )

151 + arguments = param_match.group(1) if param_match else ""

152 + arguments, _ = filter_delta_text(arguments, previous_text)

153 +

154 + # if this iteration finishes a previous tool call but a

155 + # new incomplete tool is already generated, take the

156 + # previous from the list

157 + if finishes_previous_tool and "parameters" not in current_tool_call:

158 + current_tool_call = obj[-2]

159 +

160 + function_name_returned = True

161 + tool_call_id = make_tool_call_id(

162 + id_type=tool_call_id_type,

163 + func_name=current_tool_call["name"],

164 + idx=tool_call_idx,

165 + )

166 + delta_message = DeltaMessage(

167 + tool_calls=[

168 + DeltaToolCall(

169 + id=tool_call_id,

170 + function=DeltaFunctionCall(

171 + name=current_tool_call["name"], arguments=arguments

172 + ),

173 + index=len(obj) - 1,

174 + type="function",

175 + )

176 + ]

177 + )

178 +

179 + else:

180 + delta_text, _ = filter_delta_text(delta_text, previous_text)

181 +

182 + if delta_text != "":

183 + delta_message = DeltaMessage(

184 + tool_calls=[

185 + DeltaToolCall(

186 + function=DeltaFunctionCall(

187 + # OpenAI API returns None

188 + # instead of name every time

189 + name=None,

190 + arguments=delta_text,

191 + ),

192 + index=len(obj) - 1,

193 + )

194 + ]

195 + )

196 + else:

197 + delta_message = None

198 +

199 + return delta_message, function_name_returned