A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 16:30 UTC
[Perf] Simplify DeepseekV32 tokenizer, ensure fast detokenization used (#33855)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
Feb 7, 2026 at 23:16 UTC
a96197f564cb848c5a89ae3438a8bc21f5f5d2c6
4 files changed
+88
-203
tests/tokenizers_/test_basic.py
+8
@@ -11,6 +11,7 @@ from transformers import (
11
12
from vllm.tokenizers import TokenizerLike, get_tokenizer
13
from vllm.tokenizers.grok2 import Grok2Tokenizer
14
+from vllm.tokenizers.hf import HfTokenizer
15
from vllm.tokenizers.mistral import MistralTokenizer
16
17
@@ -42,6 +43,13 @@ def test_tokenizer_like_protocol():
43
assert isinstance(tokenizer, Grok2Tokenizer)
44
_assert_tokenizer_like(tokenizer)
45
46
+ tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3", tokenizer_mode="deepseek_v32")
47
+ assert isinstance(tokenizer, HfTokenizer)
48
+ # Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)
49
+ assert isinstance(tokenizer, PreTrainedTokenizerFast)
50
+ assert "DSV32" in tokenizer.__class__.__name__
51
+ _assert_tokenizer_like(tokenizer)
52
+
53
54
@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])
55
def test_tokenizer_revision(tokenizer_name: str):
vllm/renderers/deepseek_v32.py
+3
-2
@@ -13,6 +13,7 @@ from vllm.logger import init_logger
13
from vllm.tokenizers import cached_get_tokenizer
14
from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
15
16
+from ..tokenizers.hf import HfTokenizer
17
from .inputs import DictPrompt
18
from .inputs.preprocess import parse_dec_only_prompt
19
from .params import ChatParams
@@ -48,10 +49,10 @@ class DeepseekV32Renderer(BaseRenderer):
49
self._tokenizer = tokenizer
50
51
@property
51
- def tokenizer(self) -> DeepseekV32Tokenizer | None:
52
+ def tokenizer(self) -> HfTokenizer | None:
53
return self._tokenizer
54
54
- def get_tokenizer(self) -> DeepseekV32Tokenizer:
55
+ def get_tokenizer(self) -> HfTokenizer:
56
tokenizer = self.tokenizer
57
if tokenizer is None:
58
raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`")
vllm/tokenizers/deepseek_v32.py
+77
-179
@@ -1,191 +1,89 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import copy
4
+from typing import Any
5
4
-from pathlib import Path
5
-from typing import Any, overload
6
-
7
-from transformers import BatchEncoding
6
+from transformers import AutoTokenizer
7
8
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
9
10
+from . import TokenizerLike
11
from .deepseek_v32_encoding import encode_messages
12
-from .hf import CachedHfTokenizer
13
-from .protocol import TokenizerLike
12
+from .hf import HfTokenizer, get_cached_tokenizer
13
14
16
-class DeepseekV32Tokenizer(CachedHfTokenizer):
17
- @classmethod
18
- def from_pretrained(
19
- cls,
20
- path_or_repo_id: str | Path,
21
- *args,
22
- trust_remote_code: bool = False,
23
- revision: str | None = None,
24
- download_dir: str | None = None,
25
- **kwargs,
26
- ) -> "TokenizerLike":
27
- tokenizer = super().from_pretrained(
28
- path_or_repo_id,
29
- *args,
30
- trust_remote_code=trust_remote_code,
31
- revision=revision,
32
- download_dir=download_dir,
15
+def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
16
+ """
17
+ Wraps a tokenizer to use the custom DeepSeek V3.2 chat template encoding.
18
+ """
19
+ dsv32_tokenizer = copy.copy(tokenizer)
20
+
21
+ added_vocab = tokenizer.get_added_vocab()
22
+ added_vocab_size = len(added_vocab)
23
+ tokenizer_vocab_size = tokenizer.vocab_size
24
+
25
+ class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore
26
+ def apply_chat_template(
27
+ self,
28
+ messages: list["ChatCompletionMessageParam"],
29
+ tools: list[dict[str, Any]] | None = None,
30
**kwargs,
34
- )
35
- return DeepseekV32Tokenizer(tokenizer)
36
-
37
- def __init__(self, tokenizer: TokenizerLike) -> None:
38
- super().__init__()
39
-
40
- self.tokenizer = tokenizer
41
- self.name_or_path = getattr(tokenizer, "name_or_path", "")
42
-
43
- self._added_vocab = self.tokenizer.get_added_vocab()
44
- self._added_vocab_size = len(self._added_vocab)
45
-
46
- def apply_chat_template(
47
- self,
48
- messages: list["ChatCompletionMessageParam"],
49
- tools: list[dict[str, Any]] | None = None,
50
- **kwargs,
51
- ) -> str | list[int]:
52
- thinking = kwargs.get("thinking", False)
53
- enable_thinking = kwargs.get("enable_thinking", False)
54
- thinking = thinking or enable_thinking
55
- thinking_mode = "thinking"
56
- if not thinking:
57
- thinking_mode = "chat"
58
- conversation = kwargs.get("conversation", messages)
59
- messages = conversation.copy()
60
- if tools is not None and len(tools) > 0:
61
- messages.insert(0, {"role": "system"})
62
- messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key]
63
-
64
- # Historical reasoning content is dropped when a new user message is introduced
65
- drop_thinking = messages[-1]["role"] == "user"
66
-
67
- encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
68
-
69
- prompt_str = encode_messages(messages, **encode_config) # type: ignore
70
-
71
- if kwargs.get("tokenize", True):
72
- tokenizer_kwargs = {
73
- k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs
74
- }
75
- return self.encode(
76
- prompt_str,
77
- add_special_tokens=False,
78
- **tokenizer_kwargs,
31
+ ) -> str | list[int]:
32
+ thinking = kwargs.get("thinking", False)
33
+ enable_thinking = kwargs.get("enable_thinking", False)
34
+ thinking = thinking or enable_thinking
35
+ thinking_mode = "thinking"
36
+ if not thinking:
37
+ thinking_mode = "chat"
38
+ conversation = kwargs.get("conversation", messages)
39
+ messages = conversation.copy()
40
+ if tools is not None and len(tools) > 0:
41
+ messages.insert(0, {"role": "system"})
42
+ messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key]
43
+
44
+ # Historical reasoning content is dropped when a new user message
45
+ # is introduced
46
+ drop_thinking = messages[-1]["role"] == "user"
47
+
48
+ encode_config = dict(
49
+ thinking_mode=thinking_mode, drop_thinking=drop_thinking
50
)
51
81
- return prompt_str
82
-
83
- def num_special_tokens_to_add(self) -> int:
84
- return len(self.encode(""))
85
-
86
- @property
87
- def all_special_tokens(self) -> list[str]:
88
- return self.tokenizer.all_special_tokens
89
-
90
- @property
91
- def all_special_ids(self) -> list[int]:
92
- return self.tokenizer.all_special_ids
93
-
94
- @property
95
- def bos_token_id(self) -> int:
96
- return self.tokenizer.bos_token_id
97
-
98
- @property
99
- def eos_token_id(self) -> int:
100
- return self.tokenizer.eos_token_id
101
-
102
- @property
103
- def pad_token_id(self) -> int:
104
- return self.tokenizer.pad_token_id
105
-
106
- @property
107
- def is_fast(self) -> bool:
108
- return self.tokenizer.is_fast
109
-
110
- @property
111
- def vocab_size(self) -> int:
112
- return self.tokenizer.vocab_size
113
-
114
- @property
115
- def max_token_id(self) -> int:
116
- return self.tokenizer.max_token_id
117
-
118
- @property
119
- def max_chars_per_token(self) -> int:
120
- return self.tokenizer.max_chars_per_token
121
-
122
- @property
123
- def truncation_side(self) -> str:
124
- return self.tokenizer.truncation_side
125
-
126
- def __hash__(self) -> int:
127
- return hash(id(self))
128
-
129
- def __len__(self) -> int:
130
- # </think> is an added token in DeepseekV32 tokenizer
131
- return self.vocab_size + self._added_vocab_size
132
-
133
- def __call__(
134
- self,
135
- text: str | list[str],
136
- text_pair: str | None = None,
137
- add_special_tokens: bool = True,
138
- truncation: bool = False,
139
- max_length: int | None = None,
140
- ) -> "BatchEncoding":
141
- return self.tokenizer(
142
- text,
143
- text_pair=text_pair,
144
- add_special_tokens=add_special_tokens,
145
- truncation=truncation,
146
- max_length=max_length,
147
- )
148
-
149
- def get_vocab(self) -> dict[str, int]:
150
- return self.tokenizer.get_vocab()
151
-
152
- def get_added_vocab(self) -> dict[str, int]:
153
- return self._added_vocab.copy()
154
-
155
- def encode(
156
- self,
157
- text: str,
158
- truncation: bool | None = None,
159
- max_length: int | None = None,
160
- add_special_tokens: bool = True,
161
- ) -> list[int]:
162
- return self.tokenizer.encode(
163
- text,
164
- truncation=truncation,
165
- max_length=max_length,
166
- add_special_tokens=add_special_tokens,
167
- )
168
-
169
- @overload
170
- def convert_tokens_to_ids(self, tokens: str) -> int: ...
171
-
172
- @overload
173
- def convert_tokens_to_ids(self, tokens: list[str]) -> list[int]: ...
174
-
175
- def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
176
- return self.tokenizer.convert_tokens_to_ids(tokens)
177
-
178
- def convert_tokens_to_string(self, tokens: list[str]) -> str:
179
- return self.tokenizer.convert_tokens_to_string(tokens)
180
-
181
- def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
182
- return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
183
-
184
- def convert_ids_to_tokens(
185
- self,
186
- ids: list[int],
187
- skip_special_tokens: bool = False,
188
- ) -> list[str]:
189
- return self.tokenizer.convert_ids_to_tokens(
190
- ids, skip_special_tokens=skip_special_tokens
191
- )
52
+ prompt_str = encode_messages(messages, **encode_config) # type: ignore
53
+
54
+ if kwargs.get("tokenize", True):
55
+ tokenizer_kwargs = {
56
+ k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs
57
+ }
58
+ return self.encode(
59
+ prompt_str,
60
+ add_special_tokens=False,
61
+ **tokenizer_kwargs,
62
+ )
63
+
64
+ return prompt_str
65
+
66
+ def num_special_tokens_to_add(self) -> int:
67
+ return len(self.encode(""))
68
+
69
+ def __len__(self) -> int:
70
+ # </think> is an added token in DeepseekV32 tokenizer
71
+ return tokenizer_vocab_size + added_vocab_size
72
+
73
+ def get_added_vocab(self) -> dict[str, int]:
74
+ return added_vocab.copy()
75
+
76
+ def __reduce__(self):
77
+ return get_deepseek_v32_tokenizer, (tokenizer,)
78
+
79
+ _DeepseekV32Tokenizer.__name__ = f"DSV32{tokenizer.__class__.__name__}"
80
+
81
+ dsv32_tokenizer.__class__ = _DeepseekV32Tokenizer
82
+ return dsv32_tokenizer
83
+
84
+
85
+class DeepseekV32Tokenizer(TokenizerLike):
86
+ @classmethod
87
+ def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
88
+ tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
89
+ return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer))
vllm/v1/structured_output/backend_xgrammar.py
-22
@@ -10,7 +10,6 @@ import torch
10
import vllm.envs
11
from vllm.logger import init_logger
12
from vllm.sampling_params import SamplingParams
13
-from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
13
from vllm.tokenizers.mistral import MistralTokenizer
14
from vllm.utils.import_utils import LazyLoader
15
from vllm.v1.structured_output.backend_types import (
@@ -57,27 +56,6 @@ class XgrammarBackend(StructuredOutputBackend):
56
stop_token_ids=stop_token_ids,
57
add_prefix_space=True,
58
)
60
- elif isinstance(self.tokenizer, DeepseekV32Tokenizer):
61
- # copy from xgr.TokenizerInfo.from_huggingface()
62
- # because we are using a custom tokenizer wrapper here.
63
- vocab_dict = self.tokenizer.get_vocab()
64
- tokenizer_vocab_size = max(len(vocab_dict), self.tokenizer.max_token_id + 1)
65
- vocab_size = self.vocab_size or tokenizer_vocab_size
66
- # maintain tokenizer's indexing
67
- encoded_vocab = [""] * vocab_size
68
- for token, idx in vocab_dict.items():
69
- if idx < vocab_size:
70
- encoded_vocab[idx] = token
71
- stop_token_ids = [self.tokenizer.eos_token_id]
72
- backend_str = self.tokenizer.tokenizer.backend_tokenizer.to_str() # type: ignore[attr-defined]
73
- metadata = xgr.TokenizerInfo._detect_metadata_from_hf(backend_str)
74
- tokenizer_info = xgr.TokenizerInfo(
75
- encoded_vocab=encoded_vocab,
76
- vocab_type=metadata["vocab_type"],
77
- vocab_size=vocab_size,
78
- stop_token_ids=stop_token_ids,
79
- add_prefix_space=metadata["add_prefix_space"],
80
- )
59
else:
60
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
61
self.tokenizer,