A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 09:45 UTC
Remove `all_special_tokens_extended` from tokenizer code (#29686)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Nov 28, 2025 at 20:26 UTC
fecae12cd7deb969dcbba37fda9d2d234697a944
6 files changed
+40
-65
tests/tokenization/test_cached_tokenizer.py
-1
@@ -31,7 +31,6 @@ def _check_consistency(target: AnyTokenizer, expected: AnyTokenizer):
31
# Cached attributes
32
assert target.all_special_ids == expected.all_special_ids
33
assert target.all_special_tokens == expected.all_special_tokens
34
- assert target.all_special_tokens_extended == expected.all_special_tokens_extended
34
assert target.get_vocab() == expected.get_vocab()
35
assert len(target) == len(expected)
36
tests/tokenization/test_mistral_tokenizer.py
+40
-46
@@ -258,52 +258,46 @@ def mistral_tokenizer(request) -> MistralTokenizer:
258
)
259
class TestMistralTokenizer:
260
def test_all_special_tokens(self, mistral_tokenizer: MistralTokenizer):
261
- attributes = [
262
- mistral_tokenizer.all_special_tokens,
263
- mistral_tokenizer.all_special_tokens_extended,
264
- ]
265
-
266
- for attribute in attributes:
267
- if mistral_tokenizer.is_tekken:
268
- assert attribute == [
269
- "<unk>",
270
- "<s>",
271
- "</s>",
272
- "[INST]",
273
- "[/INST]",
274
- "[AVAILABLE_TOOLS]",
275
- "[/AVAILABLE_TOOLS]",
276
- "[TOOL_RESULTS]",
277
- "[/TOOL_RESULTS]",
278
- "[TOOL_CALLS]",
279
- "[IMG]",
280
- "<pad>",
281
- "[IMG_BREAK]",
282
- "[IMG_END]",
283
- "[PREFIX]",
284
- "[MIDDLE]",
285
- "[SUFFIX]",
286
- "[SYSTEM_PROMPT]",
287
- "[/SYSTEM_PROMPT]",
288
- "[TOOL_CONTENT]",
289
- ] + [f"<SPECIAL_{i}>" for i in range(20, 32)] + [
290
- "[ARGS]",
291
- "[CALL_ID]",
292
- "[THINK]",
293
- "[/THINK]",
294
- ] + [f"<SPECIAL_{i}>" for i in range(36, 1000)]
295
- else:
296
- assert attribute == [
297
- "<s>",
298
- "</s>",
299
- "[INST]",
300
- "[/INST]",
301
- "[TOOL_CALLS]",
302
- "[AVAILABLE_TOOLS]",
303
- "[/AVAILABLE_TOOLS]",
304
- "[TOOL_RESULTS]",
305
- "[/TOOL_RESULTS]",
306
- ] + [f"[control_{i}]" for i in range(8, 769)]
261
+ if mistral_tokenizer.is_tekken:
262
+ assert mistral_tokenizer.all_special_tokens == [
263
+ "<unk>",
264
+ "<s>",
265
+ "</s>",
266
+ "[INST]",
267
+ "[/INST]",
268
+ "[AVAILABLE_TOOLS]",
269
+ "[/AVAILABLE_TOOLS]",
270
+ "[TOOL_RESULTS]",
271
+ "[/TOOL_RESULTS]",
272
+ "[TOOL_CALLS]",
273
+ "[IMG]",
274
+ "<pad>",
275
+ "[IMG_BREAK]",
276
+ "[IMG_END]",
277
+ "[PREFIX]",
278
+ "[MIDDLE]",
279
+ "[SUFFIX]",
280
+ "[SYSTEM_PROMPT]",
281
+ "[/SYSTEM_PROMPT]",
282
+ "[TOOL_CONTENT]",
283
+ ] + [f"<SPECIAL_{i}>" for i in range(20, 32)] + [
284
+ "[ARGS]",
285
+ "[CALL_ID]",
286
+ "[THINK]",
287
+ "[/THINK]",
288
+ ] + [f"<SPECIAL_{i}>" for i in range(36, 1000)]
289
+ else:
290
+ assert mistral_tokenizer.all_special_tokens == [
291
+ "<s>",
292
+ "</s>",
293
+ "[INST]",
294
+ "[/INST]",
295
+ "[TOOL_CALLS]",
296
+ "[AVAILABLE_TOOLS]",
297
+ "[/AVAILABLE_TOOLS]",
298
+ "[TOOL_RESULTS]",
299
+ "[/TOOL_RESULTS]",
300
+ ] + [f"[control_{i}]" for i in range(8, 769)]
301
302
def get_vocab(self, mistral_tokenizer: MistralTokenizer):
303
assert (
tests/tokenization/test_tokenizer_registry.py
-4
@@ -15,10 +15,6 @@ class TestTokenizer(TokenizerBase):
15
def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
16
return TestTokenizer()
17
18
- @property
19
- def all_special_tokens_extended(self) -> list[str]:
20
- raise NotImplementedError()
21
-
18
@property
19
def all_special_tokens(self) -> list[str]:
20
raise NotImplementedError()
vllm/transformers_utils/tokenizer.py
-5
@@ -96,7 +96,6 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
96
97
tokenizer_all_special_ids = tokenizer.all_special_ids
98
tokenizer_all_special_tokens = tokenizer.all_special_tokens
99
- tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended
99
tokenizer_vocab = tokenizer.get_vocab()
100
tokenizer_len = len(tokenizer)
101
@@ -118,10 +117,6 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
117
def all_special_tokens(self) -> list[str]:
118
return tokenizer_all_special_tokens
119
121
- @property
122
- def all_special_tokens_extended(self) -> list[str]:
123
- return tokenizer_all_special_tokens_extended
124
-
120
@property
121
def max_token_id(self) -> int:
122
return max_token_id
vllm/transformers_utils/tokenizer_base.py
-5
@@ -10,11 +10,6 @@ if TYPE_CHECKING:
10
11
12
class TokenizerBase(ABC):
13
- @property
14
- @abstractmethod
15
- def all_special_tokens_extended(self) -> list[str]:
16
- raise NotImplementedError()
17
-
13
@property
14
@abstractmethod
15
def all_special_tokens(self) -> list[str]:
vllm/transformers_utils/tokenizers/mistral.py
-4
@@ -254,10 +254,6 @@ class MistralTokenizer(TokenizerBase):
254
255
# the following attributes are set to fit vLLM's design and are used
256
# by the structured output backends.
257
- @property
258
- def all_special_tokens_extended(self) -> list[str]:
259
- return self.all_special_tokens
260
-
257
@property
258
def all_special_tokens(self) -> list[str]:
259
return self._special_tokens