A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix]: do not shutdown server if `skip_special_use=False` for MistralTokenizer (#14094)
Signed-off-by: Guillaume Calmettes <gcalmettes@scaleway.com>
Guillaume Calmettes committed
Apr 10, 2025 at 04:43 UTC
1da6a0927418aceaa677d3903c93ca4684ee51d8
3 files changed
+13
-3
vllm/entrypoints/openai/serving_chat.py
+3
-1
@@ -39,7 +39,8 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
39
from vllm.sequence import Logprob
40
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
41
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
42
- truncate_tool_call_ids)
42
+ truncate_tool_call_ids,
43
+ validate_request_params)
44
45
logger = init_logger(__name__)
46
@@ -159,6 +160,7 @@ class OpenAIServingChat(OpenAIServing):
160
# for more info: see comment in `maybe_serialize_tool_calls`
161
maybe_serialize_tool_calls(request)
162
truncate_tool_call_ids(request)
163
+ validate_request_params(request)
164
165
if (request.tool_choice == "auto" and
166
not (self.enable_auto_tools and tool_parser is not None)
vllm/transformers_utils/tokenizers/__init__.py
+3
-2
@@ -1,8 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from .mistral import (MistralTokenizer, maybe_serialize_tool_calls,
4
- truncate_tool_call_ids)
4
+ truncate_tool_call_ids, validate_request_params)
5
6
__all__ = [
7
- "MistralTokenizer", "maybe_serialize_tool_calls", "truncate_tool_call_ids"
7
+ "MistralTokenizer", "maybe_serialize_tool_calls", "truncate_tool_call_ids",
8
+ "validate_request_params"
9
]
vllm/transformers_utils/tokenizers/mistral.py
+7
@@ -98,6 +98,13 @@ def truncate_tool_call_ids(request: "ChatCompletionRequest"):
98
request.messages[i]["tool_call_id"] = tool_call_id
99
100
101
+def validate_request_params(request: "ChatCompletionRequest"):
102
+ if (request.skip_special_tokens is not None
103
+ and not request.skip_special_tokens):
104
+ raise ValueError("skip_special_tokens=False is not supported "
105
+ "for Mistral tokenizers.")
106
+
107
+
108
def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
109
repo_cache = os.path.join(
110
huggingface_hub.constants.HF_HUB_CACHE,