| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | from transformers import AutoTokenizer |
| 6 | |
| 7 | from vllm.tokenizers import TokenizerLike |
| 8 | |
| 9 | |
| 10 | @pytest.fixture(scope="module") |
| 11 | def default_tokenizer() -> TokenizerLike: |
| 12 | return AutoTokenizer.from_pretrained("openai-community/gpt2") |