A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 14:13 UTC
Stop testing for slow tokenizers as they will not exist soon (#34235)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Feb 10, 2026 at 13:08 UTC
61413973e83b9ca07f3c894a90ddecca0a39d2b6
1 file changed
-5
tests/tokenizers_/test_basic.py
-5
@@ -4,7 +4,6 @@ from typing import _get_protocol_attrs # type: ignore
4
5
import pytest
6
from transformers import (
7
- PreTrainedTokenizer,
7
PreTrainedTokenizerBase,
8
PreTrainedTokenizerFast,
9
)
@@ -25,10 +24,6 @@ def _assert_tokenizer_like(tokenizer: object):
24
25
26
def test_tokenizer_like_protocol():
28
- tokenizer = get_tokenizer("gpt2", use_fast=False)
29
- assert isinstance(tokenizer, PreTrainedTokenizer)
30
- _assert_tokenizer_like(tokenizer)
31
-
27
tokenizer = get_tokenizer("gpt2", use_fast=True)
28
assert isinstance(tokenizer, PreTrainedTokenizerFast)
29
_assert_tokenizer_like(tokenizer)