A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 19:30 UTC
[Refactor] to simplify and extract the shared logic between chat completion and responses (#27961)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Nov 5, 2025 at 15:46 UTC
0976711f3b569aae4a8c9ac148f0771624293120
2 files changed
+112
-62
vllm/entrypoints/openai/serving_chat.py
+35
-61
@@ -13,7 +13,6 @@ import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
-from pydantic import TypeAdapter
16
17
from vllm.engine.protocol import EngineClient
18
from vllm.entrypoints.chat_utils import (
@@ -47,8 +46,6 @@ from vllm.entrypoints.openai.protocol import (
46
DeltaMessage,
47
DeltaToolCall,
48
ErrorResponse,
50
- FunctionCall,
51
- FunctionDefinition,
49
PromptTokenUsageInfo,
50
RequestResponseMetadata,
51
ToolCall,
@@ -1394,6 +1391,16 @@ class OpenAIServingChat(OpenAIServing):
1391
auto_tools_called = False
1392
# if auto tools are not enabled, and a named tool choice using
1393
# outlines is not being used
1394
+ tool_calls, content = self._parse_tool_calls_from_content(
1395
+ request=request,
1396
+ tokenizer=tokenizer,
1397
+ content=content,
1398
+ enable_auto_tools=self.enable_auto_tools,
1399
+ tool_parser_cls=self.tool_parser,
1400
+ )
1401
+ tool_call_class = (
1402
+ MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
1403
+ )
1404
if (not self.enable_auto_tools or not self.tool_parser) and (
1405
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
1406
and request.tool_choice != "required"
@@ -1407,63 +1414,33 @@ class OpenAIServingChat(OpenAIServing):
1414
request.tool_choice
1415
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
1416
):
1410
- tool_call_class = (
1411
- MistralToolCall
1412
- if isinstance(tokenizer, MistralTokenizer)
1413
- else ToolCall
1414
- )
1417
+ assert tool_calls is not None and len(tool_calls) > 0
1418
message = ChatMessage(
1419
role=role,
1420
reasoning_content=reasoning_content,
1421
content="",
1419
- tool_calls=[
1420
- tool_call_class(
1421
- function=FunctionCall(
1422
- name=request.tool_choice.function.name,
1423
- arguments=content,
1424
- )
1425
- )
1426
- ],
1422
+ tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1423
)
1424
1425
elif request.tool_choice and request.tool_choice == "required":
1430
- tool_call_class = (
1431
- MistralToolCall
1432
- if isinstance(tokenizer, MistralTokenizer)
1433
- else ToolCall
1434
- )
1435
-
1436
- # the fields of FunctionDefinition are a superset of the
1437
- # tool call outputs and can be used for parsing
1438
- assert content is not None
1439
- tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
1440
- content
1441
- )
1442
- tool_call_ids = []
1426
+ tool_call_class_items = []
1427
+ assert tool_calls is not None and len(tool_calls) > 0
1428
for tool_call in tool_calls:
1444
- tool_call_ids.append(
1445
- make_tool_call_id(
1446
- id_type=self.tool_call_id_type,
1447
- func_name=tool_call.name,
1448
- idx=history_tool_call_cnt,
1429
+ tool_call_class_items.append(
1430
+ tool_call_class(
1431
+ id=make_tool_call_id(
1432
+ id_type=self.tool_call_id_type,
1433
+ func_name=tool_call.name,
1434
+ idx=history_tool_call_cnt,
1435
+ ),
1436
+ function=tool_call,
1437
)
1438
)
1439
history_tool_call_cnt += 1
1440
message = ChatMessage(
1441
role=role,
1442
content="",
1455
- tool_calls=[
1456
- tool_call_class(
1457
- id=tool_call_ids[i],
1458
- function=FunctionCall(
1459
- name=tool_call.name,
1460
- arguments=json.dumps(
1461
- tool_call.parameters, ensure_ascii=False
1462
- ),
1463
- ),
1464
- )
1465
- for i, tool_call in enumerate(tool_calls)
1466
- ],
1443
+ tool_calls=tool_call_class_items,
1444
reasoning_content=reasoning_content,
1445
)
1446
@@ -1481,25 +1458,22 @@ class OpenAIServingChat(OpenAIServing):
1458
and self.enable_auto_tools
1459
and self.tool_parser
1460
):
1484
- try:
1485
- tool_parser = self.tool_parser(tokenizer)
1486
- except RuntimeError as e:
1487
- logger.exception("Error in tool parser creation.")
1488
- return self.create_error_response(str(e))
1489
-
1490
- tool_call_info = tool_parser.extract_tool_calls(
1491
- content if content is not None else "", request=request
1492
- )
1461
# In the OpenAI API the finish_reason is "tools_called"
1462
# if the tool choice is auto and the model produced a tool
1463
# call. The same is not true for named function calls
1496
- auto_tools_called = tool_call_info.tools_called
1497
- if tool_call_info.tools_called:
1464
+ auto_tools_called = tool_calls is not None and len(tool_calls) > 0
1465
+ if tool_calls:
1466
message = ChatMessage(
1467
role=role,
1468
reasoning_content=reasoning_content,
1501
- content=tool_call_info.content,
1502
- tool_calls=tool_call_info.tool_calls,
1469
+ content=content,
1470
+ tool_calls=[
1471
+ ToolCall(
1472
+ function=tc,
1473
+ type="function",
1474
+ )
1475
+ for tc in tool_calls
1476
+ ],
1477
)
1478
1479
else:
@@ -1509,8 +1483,8 @@ class OpenAIServingChat(OpenAIServing):
1483
1484
# try to use content return from tool parser first,
1485
# tool parser may do some modify for the content.
1512
- if tool_call_info.content and len(tool_call_info.content) > 0:
1513
- ret_content = tool_call_info.content
1486
+ if content and len(content) > 0:
1487
+ ret_content = content
1488
message = ChatMessage(
1489
role=role,
1490
reasoning_content=reasoning_content,
vllm/entrypoints/openai/serving_engine.py
+77
-1
@@ -12,7 +12,7 @@ from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
12
13
import torch
14
from fastapi import Request
15
-from pydantic import BaseModel, ConfigDict, Field
15
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
16
from starlette.datastructures import Headers
17
from typing_extensions import TypeIs
18
@@ -21,6 +21,10 @@ if sys.version_info >= (3, 12):
21
else:
22
from typing_extensions import TypedDict
23
24
+from openai.types.responses import (
25
+ ToolChoiceFunction,
26
+)
27
+
28
import vllm.envs as envs
29
from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
30
from vllm.engine.protocol import EngineClient
@@ -36,6 +40,7 @@ from vllm.entrypoints.chat_utils import (
40
from vllm.entrypoints.context import ConversationContext
41
from vllm.entrypoints.logger import RequestLogger
42
from vllm.entrypoints.openai.protocol import (
43
+ ChatCompletionNamedToolChoiceParam,
44
ChatCompletionRequest,
45
ChatCompletionResponse,
46
ClassificationRequest,
@@ -49,6 +54,8 @@ from vllm.entrypoints.openai.protocol import (
54
EmbeddingResponse,
55
ErrorInfo,
56
ErrorResponse,
57
+ FunctionCall,
58
+ FunctionDefinition,
59
IOProcessorRequest,
60
PoolingResponse,
61
RerankRequest,
@@ -1305,6 +1312,75 @@ class OpenAIServing:
1312
except ValueError:
1313
return None
1314
1315
+ @staticmethod
1316
+ def _parse_tool_calls_from_content(
1317
+ request: ResponsesRequest | ChatCompletionRequest,
1318
+ tokenizer: AnyTokenizer,
1319
+ enable_auto_tools: bool,
1320
+ tool_parser_cls: Callable[[AnyTokenizer], ToolParser] | None,
1321
+ content: str | None = None,
1322
+ ) -> tuple[list[FunctionCall] | None, str | None]:
1323
+ function_calls = list[FunctionCall]()
1324
+ if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
1325
+ assert content is not None
1326
+ # Forced Function Call
1327
+ function_calls.append(
1328
+ FunctionCall(name=request.tool_choice.name, arguments=content)
1329
+ )
1330
+ content = None # Clear content since tool is called.
1331
+ elif request.tool_choice and isinstance(
1332
+ request.tool_choice, ChatCompletionNamedToolChoiceParam
1333
+ ):
1334
+ assert content is not None
1335
+ # Forced Function Call
1336
+ function_calls.append(
1337
+ FunctionCall(name=request.tool_choice.function.name, arguments=content)
1338
+ )
1339
+ content = None # Clear content since tool is called.
1340
+ elif request.tool_choice == "required":
1341
+ assert content is not None
1342
+ tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(content)
1343
+ function_calls.extend(
1344
+ [
1345
+ FunctionCall(
1346
+ name=tool_call.name,
1347
+ arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
1348
+ )
1349
+ for tool_call in tool_calls
1350
+ ]
1351
+ )
1352
+ content = None # Clear content since tool is called.
1353
+ elif (
1354
+ tool_parser_cls
1355
+ and enable_auto_tools
1356
+ and (request.tool_choice == "auto" or request.tool_choice is None)
1357
+ ):
1358
+ # Automatic Tool Call Parsing
1359
+ try:
1360
+ tool_parser = tool_parser_cls(tokenizer)
1361
+ except RuntimeError as e:
1362
+ logger.exception("Error in tool parser creation.")
1363
+ raise e
1364
+ tool_call_info = tool_parser.extract_tool_calls(
1365
+ content if content is not None else "",
1366
+ request=request, # type: ignore
1367
+ )
1368
+ if tool_call_info is not None and tool_call_info.tools_called:
1369
+ # extract_tool_calls() returns a list of tool calls.
1370
+ function_calls.extend(
1371
+ FunctionCall(
1372
+ name=tool_call.function.name,
1373
+ arguments=tool_call.function.arguments,
1374
+ )
1375
+ for tool_call in tool_call_info.tool_calls
1376
+ )
1377
+ content = tool_call_info.content
1378
+ else:
1379
+ # No tool calls.
1380
+ return None, content
1381
+
1382
+ return function_calls, content
1383
+
1384
@staticmethod
1385
def _get_decoded_token(
1386
logprob: Logprob,