[Tokenizer] Drop dead Mistral tokenizer shims for transformers#41962 (#56325)
Signed-off-by: Aditya Ganesh Kumar <adty910@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Aditya Ganesh Kumar committed
Sep 18, 2026 at 19:46 UTC
e562003adbb9c4eb177194d1318eb2e6862a8e15
1 file changed
+5
-17
vllm/tokenizers/mistral.py
+5
-17
@@ -354,22 +354,13 @@ class MistralTokenizer(TokenizerLike):
354
"`text_pair` is not supported by `MistralTokenizer.__call__`."
355
)
356
357
- encoded = self.transformers_tokenizer(
357
+ return self.transformers_tokenizer(
358
text=text,
359
text_pair=text_pair,
360
add_special_tokens=add_special_tokens,
361
truncation=truncation,
362
max_length=max_length,
363
)
364
- # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
365
- # is in, revert to only call self.transformers_tokenizer(...).
366
- # Hack to fix wrongly added eos token, when fix will be supported the condition
367
- # below will be False even before the revert is done.
368
- if encoded["input_ids"] and encoded["input_ids"][-1] == self.eos_token_id:
369
- encoded["input_ids"].pop(-1)
370
- if attention_mask := encoded.get("attention_mask"):
371
- attention_mask.pop(-1)
372
- return encoded
364
365
@property
366
def vocab(self) -> list[str]:
@@ -389,8 +380,10 @@ class MistralTokenizer(TokenizerLike):
380
max_length: int | None = None,
381
add_special_tokens: bool = True,
382
) -> list[int]:
392
- # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
393
- # is in, directly call self.transformers_tokenizer.encode(...).
383
+ # NOTE: transformers guards truncation with `if max_length and ...`, so
384
+ # `max_length=0` returns the full sequence instead of an empty one. vLLM
385
+ # treats `truncate_prompt_tokens=0` as an empty prompt, so keep slicing
386
+ # here until that is fixed upstream.
387
encoded = self.tokenizer.encode(text, bos=add_special_tokens, eos=False)
388
389
if truncation is not False and max_length is not None:
@@ -437,11 +430,6 @@ class MistralTokenizer(TokenizerLike):
430
def decode(
431
self, ids: Sequence[int] | int, skip_special_tokens: bool = False
432
) -> str:
440
- # TODO(juliendenize): once https://github.com/huggingface/transformers/pull/41962
441
- # is in, directly call self.transformers_tokenizer.decode(...).
442
- if isinstance(ids, int):
443
- ids = [ids]
444
-
433
return self.transformers_tokenizer.decode(
434
ids, skip_special_tokens=skip_special_tokens
435
)