A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[Misc] Implement `TokenizerLike.convert_tokens_to_ids` (#31796)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 6, 2026 at 20:08 UTC
6444824873711eeeb68ded97cad79394bd051ed1
3 files changed
+31
-3
vllm/tokenizers/deepseek_v32.py
+11
-1
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from pathlib import Path
5
-from typing import Any
5
+from typing import Any, overload
6
7
from transformers import BatchEncoding
8
@@ -65,6 +65,7 @@ class DeepseekV32Tokenizer(CachedHfTokenizer):
65
drop_thinking = messages[-1]["role"] == "user"
66
67
encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
68
+
69
prompt_str = encode_messages(messages, **encode_config) # type: ignore
70
71
if kwargs.get("tokenize", True):
@@ -161,6 +162,15 @@ class DeepseekV32Tokenizer(CachedHfTokenizer):
162
add_special_tokens=add_special_tokens,
163
)
164
165
+ @overload
166
+ def convert_tokens_to_ids(self, tokens: str) -> int: ...
167
+
168
+ @overload
169
+ def convert_tokens_to_ids(self, tokens: list[str]) -> list[int]: ...
170
+
171
+ def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
172
+ return self.tokenizer.convert_tokens_to_ids(tokens)
173
+
174
def convert_tokens_to_string(self, tokens: list[str]) -> str:
175
return self.tokenizer.convert_tokens_to_string(tokens)
176
vllm/tokenizers/mistral.py
+10
-1
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from pathlib import Path
4
-from typing import TYPE_CHECKING, Any, cast
4
+from typing import TYPE_CHECKING, Any, cast, overload
5
6
from mistral_common.protocol.instruct.request import (
7
ChatCompletionRequest as MistralChatCompletionRequest,
@@ -441,6 +441,15 @@ class MistralTokenizer(TokenizerLike):
441
ids, skip_special_tokens=skip_special_tokens
442
)
443
444
+ @overload
445
+ def convert_tokens_to_ids(self, tokens: str) -> int: ...
446
+
447
+ @overload
448
+ def convert_tokens_to_ids(self, tokens: list[str]) -> list[int]: ...
449
+
450
+ def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
451
+ return self.transformers_tokenizer.convert_tokens_to_ids(tokens)
452
+
453
def convert_tokens_to_string(self, tokens: list[str]) -> str:
454
to_decode_special_tokens = {SpecialTokens.tool_calls}
455
if self.is_tekken:
vllm/tokenizers/protocol.py
+10
-1
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from pathlib import Path
4
-from typing import TYPE_CHECKING, Any, Protocol
4
+from typing import TYPE_CHECKING, Any, Protocol, overload
5
6
if TYPE_CHECKING:
7
from transformers import BatchEncoding
@@ -100,6 +100,15 @@ class TokenizerLike(Protocol):
100
) -> str | list[int]:
101
raise NotImplementedError
102
103
+ @overload
104
+ def convert_tokens_to_ids(self, tokens: str) -> int: ...
105
+
106
+ @overload
107
+ def convert_tokens_to_ids(self, tokens: list[str]) -> list[int]: ...
108
+
109
+ def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
110
+ raise NotImplementedError
111
+
112
def convert_tokens_to_string(self, tokens: list[str]) -> str:
113
raise NotImplementedError
114