A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 03:15 UTC
Fix tool_parser_cls type annotation from Callable to type[ToolParser] (#37957)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Flora Feng committed
Mar 24, 2026 at 01:58 UTC
2e67fa756d05518823196815a30bc633031ddb60
6 files changed
+13
-17
vllm/entrypoints/openai/engine/serving.py
+2
-2
@@ -4,7 +4,7 @@ import asyncio
4
import contextlib
5
import json
6
import time
7
-from collections.abc import AsyncGenerator, Callable, Mapping
7
+from collections.abc import AsyncGenerator, Mapping
8
from dataclasses import dataclass, field
9
from http import HTTPStatus
10
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
@@ -882,7 +882,7 @@ class OpenAIServing:
882
request: ResponsesRequest | ChatCompletionRequest,
883
tokenizer: TokenizerLike | None,
884
enable_auto_tools: bool,
885
- tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
885
+ tool_parser_cls: type[ToolParser] | None,
886
content: str | None = None,
887
) -> tuple[list[FunctionCall] | None, str | None]:
888
function_calls = list[FunctionCall]()
vllm/entrypoints/openai/parser/responses_parser.py
+1
-1
@@ -39,7 +39,7 @@ class ResponsesParser:
39
reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser],
40
response_messages: list[ResponseInputOutputItem],
41
request: ResponsesRequest,
42
- tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
42
+ tool_parser_cls: type[ToolParser] | None,
43
):
44
self.response_messages: list[ResponseInputOutputItem] = (
45
# TODO: initial messages may not be properly typed
vllm/entrypoints/openai/responses/context.py
+1
-1
@@ -276,7 +276,7 @@ class ParsableContext(ConversationContext):
276
reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None,
277
request: ResponsesRequest,
278
available_tools: list[str] | None,
279
- tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
279
+ tool_parser_cls: type[ToolParser] | None,
280
chat_template: str | None,
281
chat_template_content_format: ChatTemplateContentFormatOption,
282
):
vllm/entrypoints/openai/responses/serving.py
+1
-1
@@ -602,7 +602,7 @@ class OpenAIServingResponses(OpenAIServing):
602
request: ResponsesRequest,
603
messages: list[ResponseInputOutputItem],
604
tool_dicts: list[dict[str, Any]] | None,
605
- tool_parser: Callable[[TokenizerLike], ToolParser] | None,
605
+ tool_parser: type[ToolParser] | None,
606
chat_template: str | None,
607
chat_template_content_format: ChatTemplateContentFormatOption,
608
):
vllm/entrypoints/pooling/base/io_processor.py
+2
-3
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Callable, Sequence
4
+from collections.abc import Sequence
5
from typing import Any, Final
6
7
from vllm import PoolingRequestOutput, PromptType
@@ -21,7 +21,6 @@ from vllm.entrypoints.pooling.typing import (
21
from vllm.inputs.data import ProcessorInputs, SingletonPrompt
22
from vllm.renderers import BaseRenderer, merge_kwargs
23
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
24
-from vllm.tokenizers import TokenizerLike
24
from vllm.tool_parsers import ToolParser
25
from vllm.utils.mistral import is_mistral_tokenizer
26
@@ -167,7 +166,7 @@ class PoolingIOProcessor:
166
default_template_content_format: ChatTemplateContentFormatOption,
167
default_template_kwargs: dict[str, Any] | None,
168
tool_dicts: list[dict[str, Any]] | None = None,
170
- tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
169
+ tool_parser: type[ToolParser] | None = None,
170
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
171
renderer = self.renderer
172
vllm/entrypoints/serve/render/serving.py
+6
-9
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from collections.abc import Callable, Sequence
3
+from collections.abc import Sequence
4
from http import HTTPStatus
5
from typing import Any
6
@@ -45,7 +45,6 @@ from vllm.renderers.inputs.preprocess import (
45
parse_model_prompt,
46
prompt_to_seq,
47
)
48
-from vllm.tokenizers import TokenizerLike
48
from vllm.tool_parsers import ToolParser
49
from vllm.utils import random_uuid
50
from vllm.utils.mistral import is_mistral_tokenizer
@@ -84,12 +83,10 @@ class OpenAIServingRender:
83
self.trust_request_chat_template = trust_request_chat_template
84
self.enable_auto_tools = enable_auto_tools
85
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
87
- self.tool_parser: Callable[[TokenizerLike], ToolParser] | None = (
88
- ParserManager.get_tool_parser(
89
- tool_parser_name=tool_parser,
90
- enable_auto_tools=enable_auto_tools,
91
- model_name=model_config.model,
92
- )
86
+ self.tool_parser: type[ToolParser] | None = ParserManager.get_tool_parser(
87
+ tool_parser_name=tool_parser,
88
+ enable_auto_tools=enable_auto_tools,
89
+ model_name=model_config.model,
90
)
91
self.default_chat_template_kwargs: dict[str, Any] = (
92
default_chat_template_kwargs or {}
@@ -499,7 +496,7 @@ class OpenAIServingRender:
496
default_template_content_format: ChatTemplateContentFormatOption,
497
default_template_kwargs: dict[str, Any] | None,
498
tool_dicts: list[dict[str, Any]] | None = None,
502
- tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
499
+ tool_parser: type[ToolParser] | None = None,
500
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
501
"""Copied from OpenAIServing._preprocess_chat."""
502
renderer = self.renderer