[Bugfix] Fix DeepSeek V4/3.2 tokenizer vocab size overcount crashing guided decoding (#51727)
Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Flora Feng committed
Aug 10, 2026 at 19:00 UTC
8bcc916a98a90822882455ab30aba02c409da2a6
2 files changed
-11
vllm/tokenizers/deepseek_v32.py
-6
@@ -19,8 +19,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
19
dsv32_tokenizer = copy.copy(tokenizer)
20
21
added_vocab = tokenizer.get_added_vocab()
22
- added_vocab_size = len(added_vocab)
23
- tokenizer_vocab_size = tokenizer.vocab_size
22
23
class _DeepseekV32Tokenizer(tokenizer.__class__): # type: ignore
24
def apply_chat_template(
@@ -66,10 +64,6 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
64
def num_special_tokens_to_add(self) -> int:
65
return len(self.encode(""))
66
69
- def __len__(self) -> int:
70
- # </think> is an added token in DeepseekV32 tokenizer
71
- return tokenizer_vocab_size + added_vocab_size
72
-
67
def get_added_vocab(self) -> dict[str, int]:
68
return added_vocab.copy()
69
vllm/tokenizers/deepseek_v4.py
-5
@@ -19,8 +19,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
19
dsv4_tokenizer = copy.copy(tokenizer)
20
21
added_vocab = tokenizer.get_added_vocab()
22
- added_vocab_size = len(added_vocab)
23
- tokenizer_vocab_size = tokenizer.vocab_size
22
23
class _DeepseekV4Tokenizer(tokenizer.__class__): # type: ignore
24
def apply_chat_template(
@@ -78,9 +76,6 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
76
def num_special_tokens_to_add(self) -> int:
77
return len(self.encode(""))
78
81
- def __len__(self) -> int:
82
- return tokenizer_vocab_size + added_vocab_size
83
-
79
def get_added_vocab(self) -> dict[str, int]:
80
return added_vocab.copy()
81