| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from typing import _get_protocol_attrs # type: ignore |
| 4 | |
| 5 | import pytest |
| 6 | from transformers import ( |
| 7 | PreTrainedTokenizerBase, |
| 8 | TokenizersBackend, |
| 9 | ) |
| 10 | |
| 11 | from vllm.tokenizers import TokenizerLike, get_tokenizer |
| 12 | from vllm.tokenizers.hf import HfTokenizer |
| 13 | from vllm.tokenizers.mistral import MistralTokenizer |
| 14 | |
| 15 | |
| 16 | def _get_missing_attrs(obj: object, target: type): |
| 17 | return [k for k in _get_protocol_attrs(target) if not hasattr(obj, k)] |
| 18 | |
| 19 | |
| 20 | def _assert_tokenizer_like(tokenizer: object): |
| 21 | missing_attrs = _get_missing_attrs(tokenizer, TokenizerLike) |
| 22 | assert not missing_attrs, f"Missing attrs: {missing_attrs}" |
| 23 | |
| 24 | |
| 25 | def test_tokenizer_like_protocol(): |
| 26 | tokenizer = get_tokenizer("openai-community/gpt2") |
| 27 | assert isinstance(tokenizer, TokenizersBackend) |
| 28 | _assert_tokenizer_like(tokenizer) |
| 29 | |
| 30 | tokenizer = get_tokenizer( |
| 31 | "mistralai/Mistral-7B-Instruct-v0.3", |
| 32 | tokenizer_mode="mistral", |
| 33 | ) |
| 34 | assert isinstance(tokenizer, MistralTokenizer) |
| 35 | _assert_tokenizer_like(tokenizer) |
| 36 | |
| 37 | tokenizer = get_tokenizer("deepseek-ai/DeepSeek-V3", tokenizer_mode="deepseek_v32") |
| 38 | assert isinstance(tokenizer, HfTokenizer) |
| 39 | |
| 40 | # Verify it's a fast tokenizer (required for FastIncrementalDetokenizer) |
| 41 | assert isinstance(tokenizer, TokenizersBackend) |
| 42 | assert "DSV32" in tokenizer.__class__.__name__ |
| 43 | _assert_tokenizer_like(tokenizer) |
| 44 | |
| 45 | |
| 46 | @pytest.mark.parametrize( |
| 47 | "tokenizer_name", ["facebook/opt-125m", "openai-community/gpt2"] |
| 48 | ) |
| 49 | def test_tokenizer_revision(tokenizer_name: str): |
| 50 | # Assume that "main" branch always exists |
| 51 | tokenizer = get_tokenizer(tokenizer_name, revision="main") |
| 52 | assert isinstance(tokenizer, PreTrainedTokenizerBase) |
| 53 | |
| 54 | # Assume that "never" branch always does not exist |
| 55 | with pytest.raises(OSError, match="not a valid git identifier"): |
| 56 | get_tokenizer(tokenizer_name, revision="never") |
| 57 | |
| 58 | |
| 59 | @pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"]) |
| 60 | @pytest.mark.parametrize("n_tokens", [510]) |
| 61 | def test_special_tokens(tokenizer_name: str, n_tokens: int): |
| 62 | tokenizer = get_tokenizer(tokenizer_name, revision="main") |
| 63 | |
| 64 | prompts = "[UNK]" * n_tokens |
| 65 | prompt_token_ids = tokenizer.encode(prompts) |
| 66 | assert len(prompt_token_ids) == n_tokens + 2 |