34a984274e · tduncan/Nvidia-Optimization-Stack-VLLM

.buildkite/test-amd.yaml

+2 -2

@@ -316,7 +316,7 @@ steps:

316 source_file_dependencies:

317 - vllm/

318 - tests/engine

319 - - tests/tokenization

319 + - tests/tokenizers_

320 - tests/test_sequence

321 - tests/test_config

322 - tests/test_logger

@@ -324,7 +324,7 @@ steps:

324 commands:

325 - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py

326 # OOM in the CI unless we run this separately

327 - - pytest -v -s tokenization

327 + - pytest -v -s tokenizers_

328

329 - label: V1 Test e2e + engine # 30min

330 timeout_in_minutes: 45

.buildkite/test-pipeline.yaml

+2 -2

@@ -282,7 +282,7 @@ steps:

282 source_file_dependencies:

283 - vllm/

284 - tests/engine

285 - - tests/tokenization

285 + - tests/tokenizers_

286 - tests/test_sequence

287 - tests/test_config

288 - tests/test_logger

@@ -290,7 +290,7 @@ steps:

290 commands:

291 - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py

292 # OOM in the CI unless we run this separately

293 - - pytest -v -s tokenization

293 + - pytest -v -s tokenizers_

294

295 - label: V1 Test e2e + engine # 30min

296 timeout_in_minutes: 45

benchmarks/backend_request_func.py

+1 -1

@@ -620,7 +620,7 @@ def get_tokenizer(

620 kwargs["use_fast"] = False

621 if tokenizer_mode == "mistral":

622 try:

623 - from vllm.transformers_utils.tokenizer import MistralTokenizer

623 + from vllm.tokenizers import MistralTokenizer

624 except ImportError as e:

625 raise ImportError(

626 "MistralTokenizer requires vllm package.\n"

docs/features/reasoning_outputs.md

+2 -3

@@ -216,14 +216,13 @@ You can add a new `ReasoningParser` similar to [vllm/reasoning/deepseek_r1_reaso

216 # import the required packages

217

218 from vllm.reasoning import ReasoningParser, ReasoningParserManager

219 - from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,

220 - DeltaMessage)

219 + from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage

220

221 # define a reasoning parser and register it to vllm

222 # the name list in register_module can be used

223 # in --reasoning-parser.

224 class ExampleParser(ReasoningParser):

226 - def __init__(self, tokenizer: AnyTokenizer):

225 + def __init__(self, tokenizer: TokenizerLike):

226 super().__init__(tokenizer)

227

228 def extract_reasoning_streaming(

docs/features/tool_calling.md

+1 -1

@@ -422,7 +422,7 @@ Here is a summary of a plugin file:

422 # in --tool-call-parser. you can define as many

423 # tool parsers as you want here.

424 class ExampleToolParser(ToolParser):

425 - def __init__(self, tokenizer: AnyTokenizer):

425 + def __init__(self, tokenizer: TokenizerLike):

426 super().__init__(tokenizer)

427

428 # adjust request. e.g.: set skip special tokens

tests/entrypoints/openai/test_serving_engine.py

+1 -1

@@ -10,7 +10,7 @@ import pytest

10 from vllm.config import ModelConfig

11 from vllm.entrypoints.openai.serving_engine import OpenAIServing

12 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

13 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

13 +from vllm.tokenizers import MistralTokenizer

14

15

16 @pytest.fixture()

tests/entrypoints/openai/tool_parsers/conftest.py

+2 -2

@@ -4,9 +4,9 @@

4 import pytest

5 from transformers import AutoTokenizer

6

7 -from vllm.transformers_utils.tokenizer import AnyTokenizer

7 +from vllm.tokenizers import TokenizerLike

8

9

10 @pytest.fixture(scope="function")

11 -def default_tokenizer() -> AnyTokenizer:

11 +def default_tokenizer() -> TokenizerLike:

12 return AutoTokenizer.from_pretrained("gpt2")

tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py

+6 -6

@@ -7,7 +7,7 @@ import pytest

7

8 from vllm.entrypoints.openai.protocol import ChatCompletionRequest

9 from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser

10 -from vllm.transformers_utils.tokenizer import AnyTokenizer

10 +from vllm.tokenizers import TokenizerLike

11

12 from ....utils import RemoteOpenAIServer

13

@@ -270,14 +270,14 @@ async def test_streaming_product_tool_call():

270

271

272 @pytest.fixture

273 -def qwen_tokenizer() -> AnyTokenizer:

273 +def qwen_tokenizer() -> TokenizerLike:

274 from vllm.transformers_utils.tokenizer import get_tokenizer

275

276 return get_tokenizer("Qwen/Qwen3-32B")

277

278

279 @pytest.fixture

280 -def hermes_parser(qwen_tokenizer: AnyTokenizer) -> Hermes2ProToolParser:

280 +def hermes_parser(qwen_tokenizer: TokenizerLike) -> Hermes2ProToolParser:

281 return Hermes2ProToolParser(qwen_tokenizer)

282

283

@@ -291,7 +291,7 @@ def any_chat_request() -> ChatCompletionRequest:

291

292

293 def test_hermes_parser_streaming_just_forward_text(

294 - qwen_tokenizer: AnyTokenizer,

294 + qwen_tokenizer: TokenizerLike,

295 hermes_parser: Hermes2ProToolParser,

296 any_chat_request: ChatCompletionRequest,

297 ) -> None:

@@ -323,7 +323,7 @@ def test_hermes_parser_streaming_just_forward_text(

323

324

325 def test_hermes_parser_streaming_failure_case_bug_19056(

326 - qwen_tokenizer: AnyTokenizer,

326 + qwen_tokenizer: TokenizerLike,

327 hermes_parser: Hermes2ProToolParser,

328 any_chat_request: ChatCompletionRequest,

329 ) -> None:

@@ -357,7 +357,7 @@ def test_hermes_parser_streaming_failure_case_bug_19056(

357

358

359 def test_hermes_parser_streaming(

360 - qwen_tokenizer: AnyTokenizer,

360 + qwen_tokenizer: TokenizerLike,

361 hermes_parser: Hermes2ProToolParser,

362 any_chat_request: ChatCompletionRequest,

363 ) -> None:

tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py

+2 -2

@@ -7,11 +7,11 @@ import pytest

7

8 from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation

9 from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser

10 -from vllm.transformers_utils.tokenizer import AnyTokenizer

10 +from vllm.tokenizers import TokenizerLike

11

12

13 @pytest.fixture

14 -def parser(default_tokenizer: AnyTokenizer):

14 +def parser(default_tokenizer: TokenizerLike):

15 return Llama3JsonToolParser(default_tokenizer)

16

17

tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py

+5 -5

@@ -11,7 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (

11 )

12 from vllm.entrypoints.openai.protocol import FunctionCall

13 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer

14 +from vllm.tokenizers import TokenizerLike

15

16 # Test cases similar to pythonic parser but with Llama4 specific format

17 SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]"

@@ -64,7 +64,7 @@ PYTHON_TAG_FUNCTION_OUTPUT = (

64

65

66 @pytest.mark.parametrize("streaming", [True, False])

67 -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):

67 +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):

68 tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(

69 default_tokenizer

70 )

@@ -208,7 +208,7 @@ def test_tool_call(

208 streaming: bool,

209 model_output: str,

210 expected_tool_calls: list[FunctionCall],

211 - default_tokenizer: AnyTokenizer,

211 + default_tokenizer: TokenizerLike,

212 ):

213 tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(

214 default_tokenizer

@@ -224,7 +224,7 @@ def test_tool_call(

224 assert actual.function == expected

225

226

227 -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

227 +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):

228 tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(

229 default_tokenizer

230 )

@@ -246,7 +246,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

246

247

248 @pytest.mark.parametrize("streaming", [False])

249 -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):

249 +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):

250 """test regex timeout is handled gracefully"""

251 tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(

252 default_tokenizer

tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py

+5 -5

@@ -11,7 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (

11 )

12 from vllm.entrypoints.openai.protocol import FunctionCall

13 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer

14 +from vllm.tokenizers import TokenizerLike

15

16 # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1

17 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"

@@ -69,7 +69,7 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall(

69

70

71 @pytest.mark.parametrize("streaming", [True, False])

72 -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):

72 +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):

73 tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(

74 default_tokenizer

75 )

@@ -188,7 +188,7 @@ def test_tool_call(

188 streaming: bool,

189 model_output: str,

190 expected_tool_calls: list[FunctionCall],

191 - default_tokenizer: AnyTokenizer,

191 + default_tokenizer: TokenizerLike,

192 ):

193 tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(

194 default_tokenizer

@@ -205,7 +205,7 @@ def test_tool_call(

205 assert actual.function == expected

206

207

208 -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

208 +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):

209 tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(

210 default_tokenizer

211 )

@@ -228,7 +228,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

228

229

230 @pytest.mark.parametrize("streaming", [False])

231 -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):

231 +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):

232 """test regex timeout is handled gracefully"""

233 tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(

234 default_tokenizer

tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py

+5 -5

@@ -11,7 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (

11 )

12 from vllm.entrypoints.openai.protocol import FunctionCall

13 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer

14 +from vllm.tokenizers import TokenizerLike

15

16 # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1

17 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"

@@ -61,7 +61,7 @@ ESCAPED_STRING_FUNCTION_CALL = FunctionCall(

61

62

63 @pytest.mark.parametrize("streaming", [True, False])

64 -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer):

64 +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):

65 tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(

66 default_tokenizer

67 )

@@ -168,7 +168,7 @@ def test_tool_call(

168 streaming: bool,

169 model_output: str,

170 expected_tool_calls: list[FunctionCall],

171 - default_tokenizer: AnyTokenizer,

171 + default_tokenizer: TokenizerLike,

172 ):

173 tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(

174 default_tokenizer

@@ -185,7 +185,7 @@ def test_tool_call(

185 assert actual.function == expected

186

187

188 -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

188 +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):

189 tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(

190 default_tokenizer

191 )

@@ -208,7 +208,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer):

208

209

210 @pytest.mark.parametrize("streaming", [False])

211 -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer):

211 +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):

212 """test regex timeout is handled gracefully"""

213 tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(

214 default_tokenizer

tests/entrypoints/openai/tool_parsers/utils.py

+2 -2

@@ -11,7 +11,7 @@ from vllm.entrypoints.openai.protocol import (

11 ToolCall,

12 )

13 from vllm.entrypoints.openai.tool_parsers import ToolParser

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer

14 +from vllm.tokenizers import TokenizerLike

15

16

17 class StreamingToolReconstructor:

@@ -111,7 +111,7 @@ def run_tool_extraction_nonstreaming(

111 return tool_parser.extract_tool_calls(model_output, request)

112

113

114 -def split_string_into_token_deltas(tokenizer: AnyTokenizer, text: str) -> list[str]:

114 +def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]:

115 # Split a string into a series of deltas using the provided tokenizer. Each

116 # delta will be the string equivalent of a single token.

117 token_ids = tokenizer.encode(text, add_special_tokens=False)

tests/entrypoints/test_chat_utils.py

+1 -1

@@ -28,8 +28,8 @@ from vllm.multimodal.utils import (

28 encode_image_base64,

29 encode_video_base64,

30 )

31 +from vllm.tokenizers import MistralTokenizer

32 from vllm.transformers_utils.tokenizer import get_tokenizer

32 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

33

34 from ..models.registry import HF_EXAMPLE_MODELS

35 from ..utils import VLLM_PATH

tests/models/language/generation/test_mistral.py

+1 -1

@@ -10,7 +10,7 @@ from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (

10 MistralToolParser,

11 )

12 from vllm.sampling_params import SamplingParams

13 -from vllm.transformers_utils.tokenizer import MistralTokenizer

13 +from vllm.tokenizers import MistralTokenizer

14

15 from ...utils import check_logprobs_close

16

tests/models/multimodal/generation/test_voxtral.py

+1 -1

@@ -9,7 +9,7 @@ from mistral_common.audio import Audio

9 from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk

10 from mistral_common.protocol.instruct.messages import UserMessage

11

12 -from vllm.transformers_utils.tokenizer import MistralTokenizer

12 +from vllm.tokenizers import MistralTokenizer

13

14 from ....conftest import AudioTestAssets

15 from ....utils import RemoteOpenAIServer

tests/models/multimodal/generation/vlm_utils/core.py

+2 -2

@@ -9,7 +9,7 @@ import torch

9 from transformers.models.auto.auto_factory import _BaseAutoModelClass

10

11 from vllm.config.model import RunnerOption

12 -from vllm.transformers_utils.tokenizer import AnyTokenizer

12 +from vllm.tokenizers import TokenizerLike

13

14 from .....conftest import HfRunner, VllmRunner

15 from ....registry import HF_EXAMPLE_MODELS

@@ -33,7 +33,7 @@ def run_test(

33 auto_cls: type[_BaseAutoModelClass],

34 use_tokenizer_eos: bool,

35 comparator: Callable[..., None],

36 - get_stop_token_ids: Callable[[AnyTokenizer], list[int]] | None,

36 + get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,

37 stop_str: list[str] | None,

38 limit_mm_per_prompt: dict[str, int],

39 vllm_runner_kwargs: dict[str, Any] | None,

tests/models/multimodal/generation/vlm_utils/types.py

+2 -2

@@ -14,7 +14,7 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClass

14

15 from vllm.config.model import RunnerOption

16 from vllm.logprobs import SampleLogprobs

17 -from vllm.transformers_utils.tokenizer import AnyTokenizer

17 +from vllm.tokenizers import TokenizerLike

18

19 from .....conftest import (

20 AUDIO_ASSETS,

@@ -126,7 +126,7 @@ class VLMTestInfo(NamedTuple):

126 vllm_runner_kwargs: dict[str, Any] | None = None

127

128 # Optional callable which gets a list of token IDs from the model tokenizer

129 - get_stop_token_ids: Callable[[AnyTokenizer], list[int]] | None = None

129 + get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None

130 # Optional list of strings to stop generation, useful when stop tokens are

131 # not special tokens in the tokenizer

132 stop_str: list[str] | None = None

tests/models/multimodal/processing/test_common.py

+1 -1

@@ -22,8 +22,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict

22 from vllm.multimodal.cache import MultiModalProcessorOnlyCache

23 from vllm.multimodal.inputs import MultiModalInputs

24 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext

25 +from vllm.tokenizers import MistralTokenizer

26 from vllm.transformers_utils.tokenizer import (

26 - MistralTokenizer,

27 cached_tokenizer_from_config,

28 encode_tokens,

29 )

tests/multimodal/test_processing.py

+10 -11

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 +import time

5 from contextlib import nullcontext

6 from typing import cast

7

@@ -23,7 +24,7 @@ from vllm.multimodal.processing import (

24 replace_token_matches,

25 )

26 from vllm.multimodal.profiling import MultiModalProfiler

26 -from vllm.transformers_utils.tokenizer import AnyTokenizer

27 +from vllm.tokenizers import TokenizerLike

28

29 from .utils import random_image

30

@@ -238,7 +239,7 @@ def test_find_token_matches(

239 update_type,

240 ):

241 # Should not be used since there is nothing to convert to token IDs

241 - mock_tokenizer = cast(AnyTokenizer, object())

242 + mock_tokenizer = cast(TokenizerLike, object())

243

244 prompt_updates = {

245 key: update_type(key, target, []).resolve(0)

@@ -385,7 +386,7 @@ def test_find_text_matches(

386 update_type,

387 ):

388 # Should not be used since there is nothing to convert to text

388 - mock_tokenizer = cast(AnyTokenizer, object())

389 + mock_tokenizer = cast(TokenizerLike, object())

390

391 prompt_updates = {

392 key: update_type(key, target, []).resolve(0)

@@ -545,7 +546,7 @@ def test_find_update_text(

546 expected_by_update_type_mm_count,

547 ):

548 # Should not be used since there is nothing to convert to text

548 - mock_tokenizer = cast(AnyTokenizer, object())

549 + mock_tokenizer = cast(TokenizerLike, object())

550

551 for (

552 update_type,

@@ -750,7 +751,7 @@ def test_find_update_tokens(

751 expected_by_update_type_mm_count,

752 ):

753 # Should not be used since there is nothing to convert to tokens

753 - mock_tokenizer = cast(AnyTokenizer, object())

754 + mock_tokenizer = cast(TokenizerLike, object())

755

756 for (

757 update_type,

@@ -900,7 +901,7 @@ def test_find_mm_placeholders(

901 update_type,

902 ):

903 # Should not be used since there is nothing to convert to tokens

903 - mock_tokenizer = cast(AnyTokenizer, object())

904 + mock_tokenizer = cast(TokenizerLike, object())

905

906 mm_prompt_updates = {

907 key: [[update_type(key, [], repl).resolve(i)] for i in range(3)]

@@ -1029,7 +1030,7 @@ def test_hf_processor_init_kwargs(

1030 expected_kwargs,

1031 ):

1032 # Should not be used since there is nothing to convert to tokens

1032 - mock_tokenizer = cast(AnyTokenizer, object())

1033 + mock_tokenizer = cast(TokenizerLike, object())

1034

1035 ctx = InputProcessingContext(

1036 model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

@@ -1065,7 +1066,7 @@ def test_hf_processor_call_kwargs(

1066 expected_kwargs,

1067 ):

1068 # Should not be used since there is nothing to convert to tokens

1068 - mock_tokenizer = cast(AnyTokenizer, object())

1069 + mock_tokenizer = cast(TokenizerLike, object())

1070

1071 ctx = InputProcessingContext(

1072 model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs),

@@ -1088,9 +1089,7 @@ def test_apply_matches_no_match_exits_quickly():

1089

1090 With the fix, it should exit immediately when no match is found.

1091 """

1091 - import time

1092 -

1093 - mock_tokenizer = cast(AnyTokenizer, object())

1092 + mock_tokenizer = cast(TokenizerLike, object())

1093

1094 # Create a long prompt with no placeholder

1095 long_prompt = "x" * 10000

tests/reasoning/test_mistral_reasoning_parser.py

+1 -1

@@ -5,7 +5,7 @@ import pytest

5

6 from tests.reasoning.utils import run_reasoning_extraction_mistral

7 from vllm.reasoning import ReasoningParser, ReasoningParserManager

8 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

8 +from vllm.tokenizers import MistralTokenizer

9

10 parser_name = "mistral"

11

tests/reasoning/utils.py

+1 -1

@@ -4,7 +4,7 @@

4

5 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage

6 from vllm.reasoning import ReasoningParser

7 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

7 +from vllm.tokenizers import MistralTokenizer

8

9

10 class StreamingReasoningReconstructor:

tests/tokenization/__init__.py

tests/tokenization/test_do_lower_case.py deleted

-18

@@ -1,18 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -

6 -from vllm.transformers_utils.tokenizer import get_tokenizer

7 -

8 -TOKENIZER_NAMES = ["BAAI/bge-base-en"]

9 -

10 -

11 -@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES)

12 -@pytest.mark.parametrize("n_tokens", [510])

13 -def test_special_tokens(tokenizer_name: str, n_tokens: int):

14 - tokenizer = get_tokenizer(tokenizer_name, revision="main")

15 -

16 - prompts = "[UNK]" * n_tokens

17 - prompt_token_ids = tokenizer.encode(prompts)

18 - assert len(prompt_token_ids) == n_tokens + 2

tests/tokenization/test_get_eos.py deleted

-32

@@ -1,32 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""

4 -This test file includes some cases where it is inappropriate to

5 -only get the `eos_token_id` from the tokenizer as defined by

6 -{meth}`vllm.LLMEngine._get_eos_token_id`.

7 -"""

8 -

9 -from vllm.transformers_utils.config import try_get_generation_config

10 -from vllm.transformers_utils.tokenizer import get_tokenizer

11 -

12 -

13 -def test_get_llama3_eos_token():

14 - model_name = "meta-llama/Llama-3.2-1B-Instruct"

15 -

16 - tokenizer = get_tokenizer(model_name)

17 - assert tokenizer.eos_token_id == 128009

18 -

19 - generation_config = try_get_generation_config(model_name, trust_remote_code=False)

20 - assert generation_config is not None

21 - assert generation_config.eos_token_id == [128001, 128008, 128009]

22 -

23 -

24 -def test_get_blip2_eos_token():

25 - model_name = "Salesforce/blip2-opt-2.7b"

26 -

27 - tokenizer = get_tokenizer(model_name)

28 - assert tokenizer.eos_token_id == 2

29 -

30 - generation_config = try_get_generation_config(model_name, trust_remote_code=False)

31 - assert generation_config is not None

32 - assert generation_config.eos_token_id == 50118

tests/tokenization/test_tokenizer.py deleted

-23

@@ -1,23 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -from transformers import PreTrainedTokenizerBase

6 -

7 -from vllm.transformers_utils.tokenizer import get_tokenizer

8 -

9 -TOKENIZER_NAMES = [

10 - "facebook/opt-125m",

11 - "gpt2",

12 -]

13 -

14 -

15 -@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES)

16 -def test_tokenizer_revision(tokenizer_name: str):

17 - # Assume that "main" branch always exists

18 - tokenizer = get_tokenizer(tokenizer_name, revision="main")

19 - assert isinstance(tokenizer, PreTrainedTokenizerBase)

20 -

21 - # Assume that "never" branch always does not exist

22 - with pytest.raises(OSError, match="not a valid git identifier"):

23 - get_tokenizer(tokenizer_name, revision="never")

tests/tokenization/test_tokenizer_registry.py deleted

-120

@@ -1,120 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from typing import TYPE_CHECKING, Any

5 -

6 -from vllm.transformers_utils.tokenizer import get_tokenizer

7 -from vllm.transformers_utils.tokenizer_base import TokenizerBase, TokenizerRegistry

8 -

9 -if TYPE_CHECKING:

10 - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

11 -

12 -

13 -class TestTokenizer(TokenizerBase):

14 - @classmethod

15 - def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":

16 - return TestTokenizer()

17 -

18 - @property

19 - def all_special_tokens(self) -> list[str]:

20 - raise NotImplementedError()

21 -

22 - @property

23 - def all_special_ids(self) -> list[int]:

24 - raise NotImplementedError()

25 -

26 - @property

27 - def bos_token_id(self) -> int:

28 - return 0

29 -

30 - @property

31 - def eos_token_id(self) -> int:

32 - return 1

33 -

34 - @property

35 - def sep_token(self) -> str:

36 - raise NotImplementedError()

37 -

38 - @property

39 - def pad_token(self) -> str:

40 - raise NotImplementedError()

41 -

42 - @property

43 - def is_fast(self) -> bool:

44 - raise NotImplementedError()

45 -

46 - @property

47 - def vocab_size(self) -> int:

48 - raise NotImplementedError()

49 -

50 - @property

51 - def max_token_id(self) -> int:

52 - raise NotImplementedError()

53 -

54 - @property

55 - def truncation_side(self) -> str:

56 - raise NotImplementedError()

57 -

58 - def __call__(

59 - self,

60 - text: str | list[str] | list[int],

61 - text_pair: str | None = None,

62 - add_special_tokens: bool = False,

63 - truncation: bool = False,

64 - max_length: int | None = None,

65 - ):

66 - raise NotImplementedError()

67 -

68 - def get_vocab(self) -> dict[str, int]:

69 - raise NotImplementedError()

70 -

71 - def get_added_vocab(self) -> dict[str, int]:

72 - raise NotImplementedError()

73 -

74 - def encode_one(

75 - self,

76 - text: str,

77 - truncation: bool = False,

78 - max_length: int | None = None,

79 - ) -> list[int]:

80 - raise NotImplementedError()

81 -

82 - def encode(self, text: str, add_special_tokens: bool | None = None) -> list[int]:

83 - raise NotImplementedError()

84 -

85 - def apply_chat_template(

86 - self,

87 - messages: list["ChatCompletionMessageParam"],

88 - tools: list[dict[str, Any]] | None = None,

89 - **kwargs,

90 - ) -> list[int]:

91 - raise NotImplementedError()

92 -

93 - def convert_tokens_to_string(self, tokens: list[str]) -> str:

94 - raise NotImplementedError()

95 -

96 - def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:

97 - raise NotImplementedError()

98 -

99 - def convert_ids_to_tokens(

100 - self,

101 - ids: list[int],

102 - skip_special_tokens: bool = True,

103 - ) -> list[str]:

104 - raise NotImplementedError()

105 -

106 -

107 -def test_customized_tokenizer():

108 - TokenizerRegistry.register(

109 - "test_tokenizer", "tests.tokenization.test_tokenizer_registry", "TestTokenizer"

110 - )

111 -

112 - tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")

113 - assert isinstance(tokenizer, TestTokenizer)

114 - assert tokenizer.bos_token_id == 0

115 - assert tokenizer.eos_token_id == 1

116 -

117 - tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")

118 - assert isinstance(tokenizer, TestTokenizer)

119 - assert tokenizer.bos_token_id == 0

120 - assert tokenizer.eos_token_id == 1

tests/tokenizers_/__init__.py new

+4

@@ -0,0 +1,4 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +# NOTE: Since CI runs the tests from the `tests` directory, it is necessary to rename

4 +# this module to avoid conflicting with HF's `tokenizers` package

tests/tokenizers_/test_basic.py new

+59

@@ -0,0 +1,59 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from typing import _get_protocol_attrs # type: ignore

4 +

5 +import pytest

6 +from transformers import PreTrainedTokenizerBase

7 +

8 +from vllm.tokenizers import TokenizerLike

9 +from vllm.transformers_utils.tokenizer import get_tokenizer

10 +

11 +

12 +def _get_missing_attrs(obj: object, target: type):

13 + return [k for k in _get_protocol_attrs(target) if not hasattr(obj, k)]

14 +

15 +

16 +def test_tokenizer_like_protocol():

17 + assert not (

18 + missing_attrs := _get_missing_attrs(

19 + get_tokenizer("gpt2", use_fast=False),

20 + TokenizerLike,

21 + )

22 + ), f"Missing attrs: {missing_attrs}"

23 +

24 + assert not (

25 + missing_attrs := _get_missing_attrs(

26 + get_tokenizer("gpt2", use_fast=True),

27 + TokenizerLike,

28 + )

29 + ), f"Missing attrs: {missing_attrs}"

30 +

31 + assert not (

32 + missing_attrs := _get_missing_attrs(

33 + get_tokenizer(

34 + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral"

35 + ),

36 + TokenizerLike,

37 + )

38 + ), f"Missing attrs: {missing_attrs}"

39 +

40 +

41 +@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"])

42 +def test_tokenizer_revision(tokenizer_name: str):

43 + # Assume that "main" branch always exists

44 + tokenizer = get_tokenizer(tokenizer_name, revision="main")

45 + assert isinstance(tokenizer, PreTrainedTokenizerBase)

46 +

47 + # Assume that "never" branch always does not exist

48 + with pytest.raises(OSError, match="not a valid git identifier"):

49 + get_tokenizer(tokenizer_name, revision="never")

50 +

51 +

52 +@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"])

53 +@pytest.mark.parametrize("n_tokens", [510])

54 +def test_special_tokens(tokenizer_name: str, n_tokens: int):

55 + tokenizer = get_tokenizer(tokenizer_name, revision="main")

56 +

57 + prompts = "[UNK]" * n_tokens

58 + prompt_token_ids = tokenizer.encode(prompts)

59 + assert len(prompt_token_ids) == n_tokens + 2

tests/tokenizers_/test_cached_tokenizer.py renamed

+3 -2

@@ -6,7 +6,8 @@ from copy import deepcopy

6 import pytest

7 from transformers import AutoTokenizer

8

9 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_cached_tokenizer

9 +from vllm.tokenizers import TokenizerLike

10 +from vllm.transformers_utils.tokenizer import get_cached_tokenizer

11

12

13 @pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"])

@@ -25,7 +26,7 @@ def test_cached_tokenizer(model_id: str):

26 _check_consistency(unpickled_tokenizer, reference_tokenizer)

27

28

28 -def _check_consistency(target: AnyTokenizer, expected: AnyTokenizer):

29 +def _check_consistency(target: TokenizerLike, expected: TokenizerLike):

30 assert isinstance(target, type(expected))

31

32 # Cached attributes

tests/tokenizers_/test_detokenize.py renamed

+1 -1

@@ -8,7 +8,7 @@ import pytest

8 from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast

9

10 from vllm.sampling_params import SamplingParams

11 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

11 +from vllm.tokenizers import MistralTokenizer

12 from vllm.v1.engine import EngineCoreRequest

13 from vllm.v1.engine.detokenizer import (

14 FastIncrementalDetokenizer,

tests/tokenizers_/test_mistral.py renamed

+1 -20

@@ -7,7 +7,7 @@ import pytest

7 from mistral_common.exceptions import InvalidMessageStructureException

8 from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy

9

10 -from vllm.transformers_utils.tokenizers.mistral import (

10 +from vllm.tokenizers.mistral import (

11 MistralTokenizer,

12 _prepare_apply_chat_template_tools_and_messages,

13 )

@@ -308,25 +308,6 @@ class TestMistralTokenizer:

308 def test_get_added_vocab(self, mistral_tokenizer: MistralTokenizer):

309 assert mistral_tokenizer.get_added_vocab() == {}

310

311 - def test_encode_one(self, mistral_tokenizer: MistralTokenizer):

312 - token_ids = (

313 - [22177, 4304, 2662] if mistral_tokenizer.is_tekken else [23325, 2294, 1686]

314 - )

315 -

316 - assert mistral_tokenizer.encode_one("Hello world !") == token_ids

317 - assert mistral_tokenizer.encode_one("Hello world !", max_length=1) == token_ids

318 - assert (

319 - mistral_tokenizer.encode_one("Hello world !", truncation=True, max_length=1)

320 - == token_ids[:-2]

321 - )

322 - assert (

323 - mistral_tokenizer.encode_one(

324 - "Hello world !", truncation=False, max_length=1

325 - )

326 - == token_ids

327 - )

328 - assert mistral_tokenizer.encode_one("") == []

329 -

311 def test_encode(self, mistral_tokenizer: MistralTokenizer):

312 token_ids = (

313 [1, 22177, 4304, 2662]

tests/tokenizers_/test_registry.py new

+36

@@ -0,0 +1,36 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +from vllm.tokenizers import TokenizerLike, TokenizerRegistry

4 +from vllm.transformers_utils.tokenizer import get_tokenizer

5 +

6 +

7 +class TestTokenizer(TokenizerLike):

8 + @classmethod

9 + def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":

10 + return TestTokenizer() # type: ignore

11 +

12 + @property

13 + def bos_token_id(self) -> int:

14 + return 0

15 +

16 + @property

17 + def eos_token_id(self) -> int:

18 + return 1

19 +

20 +

21 +def test_customized_tokenizer():

22 + TokenizerRegistry.register(

23 + "test_tokenizer",

24 + __name__,

25 + TestTokenizer.__name__,

26 + )

27 +

28 + tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")

29 + assert isinstance(tokenizer, TestTokenizer)

30 + assert tokenizer.bos_token_id == 0

31 + assert tokenizer.eos_token_id == 1

32 +

33 + tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom")

34 + assert isinstance(tokenizer, TestTokenizer)

35 + assert tokenizer.bos_token_id == 0

36 + assert tokenizer.eos_token_id == 1

tests/tool_use/test_ernie45_moe_tool_parser.py

+3 -2

@@ -14,8 +14,9 @@ from vllm.entrypoints.openai.protocol import (

14 ToolCall,

15 )

16 from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser

17 +from vllm.tokenizers import TokenizerLike

18 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

18 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

19 +from vllm.transformers_utils.tokenizer import get_tokenizer

20

21 # Use a common model that is likely to be available

22 MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking"

@@ -173,7 +174,7 @@ def test_extract_tool_calls(

174

175 def stream_delta_message_generator(

176 ernie45_tool_parser: Ernie45ToolParser,

176 - ernie45_tokenizer: AnyTokenizer,

177 + ernie45_tokenizer: TokenizerLike,

178 model_output: str,

179 request: ChatCompletionRequest | None = None,

180 ) -> Generator[DeltaMessage, None, None]:

tests/tool_use/test_jamba_tool_parser.py

+5 -2

@@ -10,8 +10,9 @@ from partial_json_parser.core.options import Allow

10

11 from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall

12 from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser

13 +from vllm.tokenizers import TokenizerLike

14 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

15 +from vllm.transformers_utils.tokenizer import get_tokenizer

16

17 pytestmark = pytest.mark.cpu_test

18

@@ -44,7 +45,9 @@ def assert_tool_calls(

45

46

47 def stream_delta_message_generator(

47 - jamba_tool_parser: JambaToolParser, jamba_tokenizer: AnyTokenizer, model_output: str

48 + jamba_tool_parser: JambaToolParser,

49 + jamba_tokenizer: TokenizerLike,

50 + model_output: str,

51 ) -> Generator[DeltaMessage, None, None]:

52 all_token_ids = jamba_tokenizer.encode(model_output, add_special_tokens=False)

53

tests/tool_use/test_qwen3coder_tool_parser.py

+3 -2

@@ -17,8 +17,9 @@ from vllm.entrypoints.openai.tool_parsers.qwen3coder_tool_parser import (

17 Qwen3CoderToolParser,

18 )

19 from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser

20 +from vllm.tokenizers import TokenizerLike

21 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

21 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

22 +from vllm.transformers_utils.tokenizer import get_tokenizer

23

24 pytestmark = pytest.mark.cpu_test

25

@@ -104,7 +105,7 @@ def assert_tool_calls(

105

106 def stream_delta_message_generator(

107 qwen3_tool_parser,

107 - qwen3_tokenizer: AnyTokenizer,

108 + qwen3_tokenizer: TokenizerLike,

109 model_output: str,

110 request: ChatCompletionRequest | None = None,

111 ) -> Generator[DeltaMessage, None, None]:

tests/tool_use/test_seed_oss_tool_parser.py

+3 -2

@@ -15,8 +15,9 @@ from vllm.entrypoints.openai.protocol import (

15 ToolCall,

16 )

17 from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser

18 +from vllm.tokenizers import TokenizerLike

19 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

19 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

20 +from vllm.transformers_utils.tokenizer import get_tokenizer

21

22 pytestmark = pytest.mark.cpu_test

23

@@ -256,7 +257,7 @@ def test_streaming_tool_calls_no_tools(seed_oss_tool_parser):

257

258 def stream_delta_message_generator(

259 seed_oss_tool_parser: SeedOssToolParser,

259 - seed_oss_tokenizer: AnyTokenizer,

260 + seed_oss_tokenizer: TokenizerLike,

261 model_output: str,

262 request: ChatCompletionRequest | None = None,

263 ) -> Generator[DeltaMessage, None, None]:

tests/tool_use/test_xlam_tool_parser.py

+3 -2

@@ -13,8 +13,9 @@ from vllm.entrypoints.openai.protocol import (

13 ToolCall,

14 )

15 from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser

16 +from vllm.tokenizers import TokenizerLike

17 from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally

17 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer

18 +from vllm.transformers_utils.tokenizer import get_tokenizer

19

20 pytestmark = pytest.mark.cpu_test

21

@@ -49,7 +50,7 @@ def assert_tool_calls(

50

51 def stream_delta_message_generator(

52 xlam_tool_parser: xLAMToolParser,

52 - xlam_tokenizer: AnyTokenizer,

53 + xlam_tokenizer: TokenizerLike,

54 model_output: str,

55 request: ChatCompletionRequest | None = None,

56 ) -> Generator[DeltaMessage, None, None]:

tests/transformers_utils/test_config.py

+28 -58

@@ -1,62 +1,32 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +This test file includes some cases where it is inappropriate to

5 +only get the `eos_token_id` from the tokenizer as defined by

6 +`vllm.LLMEngine._get_eos_token_id`.

7 +"""

8

9 +from vllm.transformers_utils.config import try_get_generation_config

10 +from vllm.transformers_utils.tokenizer import get_tokenizer

11

5 -import tempfile

6 -from pathlib import Path

7 -from unittest.mock import MagicMock, call, patch

8 -

9 -import pytest

10 -

11 -from vllm.transformers_utils.repo_utils import list_filtered_repo_files

12 -

13 -

14 -@pytest.mark.parametrize(

15 - "allow_patterns,expected_relative_files",

16 - [

17 - (

18 - ["*.json", "correct*.txt"],

19 - ["json_file.json", "subfolder/correct.txt", "correct_2.txt"],

20 - ),

21 - ],

22 -)

23 -def test_list_filtered_repo_files(

24 - allow_patterns: list[str], expected_relative_files: list[str]

25 -):

26 - with tempfile.TemporaryDirectory() as tmp_dir:

27 - # Prep folder and files

28 - path_tmp_dir = Path(tmp_dir)

29 - subfolder = path_tmp_dir / "subfolder"

30 - subfolder.mkdir()

31 - (path_tmp_dir / "json_file.json").touch()

32 - (path_tmp_dir / "correct_2.txt").touch()

33 - (path_tmp_dir / "uncorrect.txt").touch()

34 - (path_tmp_dir / "uncorrect.jpeg").touch()

35 - (subfolder / "correct.txt").touch()

36 - (subfolder / "uncorrect_sub.txt").touch()

37 -

38 - def _glob_path() -> list[str]:

39 - return [

40 - str(file.relative_to(path_tmp_dir))

41 - for file in path_tmp_dir.glob("**/*")

42 - if file.is_file()

43 - ]

44 -

45 - # Patch list_repo_files called by fn

46 - with patch(

47 - "vllm.transformers_utils.repo_utils.list_repo_files",

48 - MagicMock(return_value=_glob_path()),

49 - ) as mock_list_repo_files:

50 - out_files = sorted(

51 - list_filtered_repo_files(

52 - tmp_dir, allow_patterns, "revision", "model", "token"

53 - )

54 - )

55 - assert out_files == sorted(expected_relative_files)

56 - assert mock_list_repo_files.call_count == 1

57 - assert mock_list_repo_files.call_args_list[0] == call(

58 - repo_id=tmp_dir,

59 - revision="revision",

60 - repo_type="model",

61 - token="token",

62 - )

12 +

13 +def test_get_llama3_eos_token():

14 + model_name = "meta-llama/Llama-3.2-1B-Instruct"

15 +

16 + tokenizer = get_tokenizer(model_name)

17 + assert tokenizer.eos_token_id == 128009

18 +

19 + generation_config = try_get_generation_config(model_name, trust_remote_code=False)

20 + assert generation_config is not None

21 + assert generation_config.eos_token_id == [128001, 128008, 128009]

22 +

23 +

24 +def test_get_blip2_eos_token():

25 + model_name = "Salesforce/blip2-opt-2.7b"

26 +

27 + tokenizer = get_tokenizer(model_name)

28 + assert tokenizer.eos_token_id == 2

29 +

30 + generation_config = try_get_generation_config(model_name, trust_remote_code=False)

31 + assert generation_config is not None

32 + assert generation_config.eos_token_id == 50118

tests/transformers_utils/test_processor.py renamed

tests/transformers_utils/test_repo_utils.py new

+62

@@ -0,0 +1,62 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +

5 +import tempfile

6 +from pathlib import Path

7 +from unittest.mock import MagicMock, call, patch

8 +

9 +import pytest

10 +

11 +from vllm.transformers_utils.repo_utils import list_filtered_repo_files

12 +

13 +

14 +@pytest.mark.parametrize(

15 + "allow_patterns,expected_relative_files",

16 + [

17 + (

18 + ["*.json", "correct*.txt"],

19 + ["json_file.json", "subfolder/correct.txt", "correct_2.txt"],

20 + ),

21 + ],

22 +)

23 +def test_list_filtered_repo_files(

24 + allow_patterns: list[str], expected_relative_files: list[str]

25 +):

26 + with tempfile.TemporaryDirectory() as tmp_dir:

27 + # Prep folder and files

28 + path_tmp_dir = Path(tmp_dir)

29 + subfolder = path_tmp_dir / "subfolder"

30 + subfolder.mkdir()

31 + (path_tmp_dir / "json_file.json").touch()

32 + (path_tmp_dir / "correct_2.txt").touch()

33 + (path_tmp_dir / "uncorrect.txt").touch()

34 + (path_tmp_dir / "uncorrect.jpeg").touch()

35 + (subfolder / "correct.txt").touch()

36 + (subfolder / "uncorrect_sub.txt").touch()

37 +

38 + def _glob_path() -> list[str]:

39 + return [

40 + str(file.relative_to(path_tmp_dir))

41 + for file in path_tmp_dir.glob("**/*")

42 + if file.is_file()

43 + ]

44 +

45 + # Patch list_repo_files called by fn

46 + with patch(

47 + "vllm.transformers_utils.repo_utils.list_repo_files",

48 + MagicMock(return_value=_glob_path()),

49 + ) as mock_list_repo_files:

50 + out_files = sorted(

51 + list_filtered_repo_files(

52 + tmp_dir, allow_patterns, "revision", "model", "token"

53 + )

54 + )

55 + assert out_files == sorted(expected_relative_files)

56 + assert mock_list_repo_files.call_count == 1

57 + assert mock_list_repo_files.call_args_list[0] == call(

58 + repo_id=tmp_dir,

59 + revision="revision",

60 + repo_type="model",

61 + token="token",

62 + )

tests/v1/engine/test_output_processor.py

+2 -2

@@ -18,7 +18,7 @@ from vllm.logprobs import PromptLogprobs, SampleLogprobs

18 from vllm.lora.request import LoRARequest

19 from vllm.outputs import CompletionOutput, RequestOutput

20 from vllm.sampling_params import RequestOutputKind, SamplingParams

21 -from vllm.transformers_utils.tokenizer import AnyTokenizer

21 +from vllm.tokenizers import TokenizerLike

22 from vllm.v1.engine import (

23 EngineCoreEvent,

24 EngineCoreEventType,

@@ -31,7 +31,7 @@ from vllm.v1.metrics.stats import IterationStats, SchedulerStats

31

32

33 def _ref_convert_id_to_token(

34 - tokenizer: AnyTokenizer,

34 + tokenizer: TokenizerLike,

35 token_id: int,

36 ) -> str:

37 """Reference impl of logprobs detokenization.

tools/pre_commit/check_pickle_imports.py

+1 -1

@@ -27,8 +27,8 @@ ALLOWED_FILES = {

27 "vllm/distributed/device_communicators/shm_broadcast.py",

28 "vllm/distributed/device_communicators/shm_object_storage.py",

29 "vllm/utils/hashing.py",

30 + "tests/tokenizers_/test_cached_tokenizer.py",

31 "tests/utils_/test_hashing.py",

31 - "tests/tokenization/test_cached_tokenizer.py",

32 "benchmarks/kernels/graph_machete_bench.py",

33 "benchmarks/kernels/benchmark_lora.py",

34 "benchmarks/kernels/benchmark_machete.py",

tools/pre_commit/mypy.py

+1

@@ -35,6 +35,7 @@ FILES = [

35 "vllm/multimodal",

36 "vllm/platforms",

37 "vllm/plugins",

38 + "vllm/tokenizers",

39 "vllm/transformers_utils",

40 "vllm/triton_utils",

41 "vllm/usage",

vllm/benchmarks/datasets.py

+2 -2

@@ -39,7 +39,7 @@ from vllm.lora.request import LoRARequest

39 from vllm.lora.utils import get_adapter_absolute_path

40 from vllm.multimodal import MultiModalDataDict

41 from vllm.multimodal.image import convert_image_mode

42 -from vllm.transformers_utils.tokenizer import AnyTokenizer

42 +from vllm.tokenizers import TokenizerLike

43 from vllm.utils.import_utils import PlaceholderModule

44

45 try:

@@ -293,7 +293,7 @@ def lora_path_on_disk(lora_path: str) -> str:

293

294

295 # Global cache for LoRA tokenizers.

296 -lora_tokenizer_cache: dict[int, AnyTokenizer] = {}

296 +lora_tokenizer_cache: dict[int, TokenizerLike] = {}

297

298

299 def process_image(image: Any) -> Mapping[str, Any]:

vllm/engine/protocol.py

+2 -2

@@ -13,7 +13,7 @@ from vllm.plugins.io_processors import IOProcessor

13 from vllm.pooling_params import PoolingParams

14 from vllm.sampling_params import SamplingParams

15 from vllm.tasks import SupportedTask

16 -from vllm.transformers_utils.tokenizer import AnyTokenizer

16 +from vllm.tokenizers import TokenizerLike

17 from vllm.v1.engine import EngineCoreRequest

18 from vllm.v1.engine.input_processor import InputProcessor

19

@@ -85,7 +85,7 @@ class EngineClient(ABC):

85 ...

86

87 @abstractmethod

88 - async def get_tokenizer(self) -> AnyTokenizer:

88 + async def get_tokenizer(self) -> TokenizerLike:

89 """Get the tokenizer"""

90 ...

91

vllm/entrypoints/chat_utils.py

+6 -6

@@ -49,9 +49,9 @@ from vllm.logger import init_logger

49 from vllm.model_executor.models import SupportsMultiModal

50 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict

51 from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector

52 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

53 from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path

54 from vllm.transformers_utils.processor import cached_get_processor

54 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

55 from vllm.utils import random_uuid

56 from vllm.utils.func_utils import supports_kw

57

@@ -536,7 +536,7 @@ def resolve_hf_chat_template(

536 def _resolve_chat_template_content_format(

537 chat_template: str | None,

538 tools: list[dict[str, Any]] | None,

539 - tokenizer: AnyTokenizer,

539 + tokenizer: TokenizerLike,

540 *,

541 model_config: ModelConfig,

542 ) -> _ChatTemplateContentFormat:

@@ -593,7 +593,7 @@ def resolve_chat_template_content_format(

593 chat_template: str | None,

594 tools: list[dict[str, Any]] | None,

595 given_format: ChatTemplateContentFormatOption,

596 - tokenizer: AnyTokenizer,

596 + tokenizer: TokenizerLike,

597 *,

598 model_config: ModelConfig,

599 ) -> _ChatTemplateContentFormat:

@@ -627,7 +627,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

627 maximum per prompt.

628 """

629

630 - def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):

630 + def __init__(self, model_config: ModelConfig, tokenizer: TokenizerLike):

631 super().__init__()

632

633 self._model_config = model_config

@@ -1592,7 +1592,7 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None:

1592 def parse_chat_messages(

1593 messages: list[ChatCompletionMessageParam],

1594 model_config: ModelConfig,

1595 - tokenizer: AnyTokenizer,

1595 + tokenizer: TokenizerLike,

1596 content_format: _ChatTemplateContentFormat,

1597 ) -> tuple[

1598 list[ConversationMessage],

@@ -1624,7 +1624,7 @@ def parse_chat_messages(

1624 def parse_chat_messages_futures(

1625 messages: list[ChatCompletionMessageParam],

1626 model_config: ModelConfig,

1627 - tokenizer: AnyTokenizer,

1627 + tokenizer: TokenizerLike,

1628 content_format: _ChatTemplateContentFormat,

1629 ) -> tuple[

1630 list[ConversationMessage],

vllm/entrypoints/llm.py

+6 -9

@@ -71,11 +71,8 @@ from vllm.platforms import current_platform

71 from vllm.pooling_params import PoolingParams

72 from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams

73 from vllm.tasks import PoolingTask

74 -from vllm.transformers_utils.tokenizer import (

75 - AnyTokenizer,

76 - MistralTokenizer,

77 - get_cached_tokenizer,

78 -)

74 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

75 +from vllm.transformers_utils.tokenizer import get_cached_tokenizer

76 from vllm.usage.usage_lib import UsageContext

77 from vllm.utils.collection_utils import as_iter, is_list_of

78 from vllm.utils.counter import Counter

@@ -350,11 +347,11 @@ class LLM:

347 self.input_processor = self.llm_engine.input_processor

348 self.io_processor = self.llm_engine.io_processor

349

353 - def get_tokenizer(self) -> AnyTokenizer:

350 + def get_tokenizer(self) -> TokenizerLike:

351 return self.llm_engine.get_tokenizer()

352

353 @deprecated("`set_tokenizer` is deprecated and will be removed in v0.13.")

357 - def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:

354 + def set_tokenizer(self, tokenizer: TokenizerLike) -> None:

355 # While CachedTokenizer is dynamic, have no choice but

356 # compare class name. Misjudgment will arise from

357 # user-defined tokenizer started with 'Cached'

@@ -1244,7 +1241,7 @@ class LLM:

1241

1242 def _embedding_score(

1243 self,

1247 - tokenizer: AnyTokenizer,

1244 + tokenizer: TokenizerLike,

1245 text_1: list[str | TextPrompt | TokensPrompt],

1246 text_2: list[str | TextPrompt | TokensPrompt],

1247 truncate_prompt_tokens: int | None = None,

@@ -1276,7 +1273,7 @@ class LLM:

1273

1274 def _cross_encoding_score(

1275 self,

1279 - tokenizer: AnyTokenizer,

1276 + tokenizer: TokenizerLike,

1277 data_1: list[str] | list[ScoreContentPartParam],

1278 data_2: list[str] | list[ScoreContentPartParam],

1279 truncate_prompt_tokens: int | None = None,

vllm/entrypoints/openai/serving_chat.py

+7 -6

@@ -62,8 +62,9 @@ from vllm.logger import init_logger

62 from vllm.logprobs import Logprob

63 from vllm.outputs import CompletionOutput, RequestOutput

64 from vllm.sampling_params import BeamSearchParams, SamplingParams

65 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

66 -from vllm.transformers_utils.tokenizers import (

65 +from vllm.tokenizers import TokenizerLike

66 +from vllm.tokenizers.mistral import (

67 + MistralTokenizer,

68 maybe_serialize_tool_calls,

69 truncate_tool_call_ids,

70 validate_request_params,

@@ -530,7 +531,7 @@ class OpenAIServingChat(OpenAIServing):

531 request_id: str,

532 model_name: str,

533 conversation: list[ConversationMessage],

533 - tokenizer: AnyTokenizer,

534 + tokenizer: TokenizerLike,

535 request_metadata: RequestResponseMetadata,

536 ) -> AsyncGenerator[str, None]:

537 created_time = int(time.time())

@@ -1296,7 +1297,7 @@ class OpenAIServingChat(OpenAIServing):

1297 request_id: str,

1298 model_name: str,

1299 conversation: list[ConversationMessage],

1299 - tokenizer: AnyTokenizer,

1300 + tokenizer: TokenizerLike,

1301 request_metadata: RequestResponseMetadata,

1302 ) -> ErrorResponse | ChatCompletionResponse:

1303 created_time = int(time.time())

@@ -1624,7 +1625,7 @@ class OpenAIServingChat(OpenAIServing):

1625 self,

1626 logprobs: dict[int, Logprob],

1627 top_logprobs: int | None,

1627 - tokenizer: AnyTokenizer,

1628 + tokenizer: TokenizerLike,

1629 should_return_as_token_id: bool,

1630 ) -> list[ChatCompletionLogProb]:

1631 return [

@@ -1648,7 +1649,7 @@ class OpenAIServingChat(OpenAIServing):

1649 self,

1650 token_ids: GenericSequence[int],

1651 top_logprobs: GenericSequence[dict[int, Logprob] | None],

1651 - tokenizer: AnyTokenizer,

1652 + tokenizer: TokenizerLike,

1653 num_output_top_logprobs: int | None = None,

1654 return_as_token_id: bool | None = None,

1655 ) -> ChatCompletionLogProbs:

vllm/entrypoints/openai/serving_classification.py

+1 -1

@@ -221,7 +221,7 @@ class ServingClassification(ClassificationMixin):

221

222 def _create_pooling_params(

223 self,

224 - ctx: ClassificationServeContext,

224 + ctx: ServeContext[ClassificationRequest],

225 ) -> PoolingParams | ErrorResponse:

226 pooling_params = super()._create_pooling_params(ctx)

227 if isinstance(pooling_params, ErrorResponse):

vllm/entrypoints/openai/serving_completion.py

+11 -5

@@ -33,7 +33,7 @@ from vllm.logger import init_logger

33 from vllm.logprobs import Logprob

34 from vllm.outputs import RequestOutput

35 from vllm.sampling_params import BeamSearchParams, SamplingParams

36 -from vllm.transformers_utils.tokenizer import AnyTokenizer

36 +from vllm.tokenizers import TokenizerLike

37 from vllm.utils.async_utils import merge_async_iterators

38 from vllm.utils.collection_utils import as_list

39 from vllm.v1.sample.logits_processor import validate_logits_processors_parameters

@@ -326,7 +326,7 @@ class OpenAIServingCompletion(OpenAIServing):

326 created_time: int,

327 model_name: str,

328 num_prompts: int,

329 - tokenizer: AnyTokenizer,

329 + tokenizer: TokenizerLike | None,

330 request_metadata: RequestResponseMetadata,

331 ) -> AsyncGenerator[str, None]:

332 num_choices = 1 if request.n is None else request.n

@@ -511,7 +511,7 @@ class OpenAIServingCompletion(OpenAIServing):

511 request_id: str,

512 created_time: int,

513 model_name: str,

514 - tokenizer: AnyTokenizer,

514 + tokenizer: TokenizerLike | None,

515 request_metadata: RequestResponseMetadata,

516 ) -> CompletionResponse:

517 choices: list[CompletionResponseChoice] = []

@@ -622,7 +622,7 @@ class OpenAIServingCompletion(OpenAIServing):

622 token_ids: GenericSequence[int],

623 top_logprobs: GenericSequence[dict[int, Logprob] | None],

624 num_output_top_logprobs: int,

625 - tokenizer: AnyTokenizer,

625 + tokenizer: TokenizerLike | None,

626 initial_text_offset: int = 0,

627 return_as_token_id: bool | None = None,

628 ) -> CompletionLogProbs:

@@ -642,9 +642,15 @@ class OpenAIServingCompletion(OpenAIServing):

642 for i, token_id in enumerate(token_ids):

643 step_top_logprobs = top_logprobs[i]

644 if step_top_logprobs is None:

645 - token = tokenizer.decode(token_id)

645 if should_return_as_token_id:

646 token = f"token_id:{token_id}"

647 + else:

648 + if tokenizer is None:

649 + raise ValueError(

650 + "Unable to get tokenizer because `skip_tokenizer_init=True`"

651 + )

652 +

653 + token = tokenizer.decode(token_id)

654

655 out_tokens.append(token)

656 out_token_logprobs.append(None)

vllm/entrypoints/openai/serving_engine.py

+42 -46

@@ -7,13 +7,14 @@ import time

7 import traceback

8 from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence

9 from concurrent.futures import ThreadPoolExecutor

10 +from dataclasses import dataclass, field

11 from http import HTTPStatus

12 from typing import Any, ClassVar, Generic, TypeAlias, TypeVar

13

14 import numpy as np

15 import torch

16 from fastapi import Request

16 -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter

17 +from pydantic import ConfigDict, TypeAdapter

18 from starlette.datastructures import Headers

19 from typing_extensions import TypeIs

20

@@ -96,12 +97,12 @@ from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput

97 from vllm.pooling_params import PoolingParams

98 from vllm.reasoning import ReasoningParser, ReasoningParserManager

99 from vllm.sampling_params import BeamSearchParams, SamplingParams

100 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

101 from vllm.tracing import (

102 contains_trace_headers,

103 extract_trace_headers,

104 log_tracing_disabled_warning,

105 )

104 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

106 from vllm.utils import random_uuid

107 from vllm.utils.async_utils import (

108 AsyncMicrobatchTokenizer,

@@ -184,19 +185,19 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:

185 RequestT = TypeVar("RequestT", bound=AnyRequest)

186

187

187 -class RequestProcessingMixin(BaseModel):

188 +@dataclass(kw_only=True)

189 +class RequestProcessingMixin:

190 """

191 Mixin for request processing,

192 handling prompt preparation and engine input.

193 """

194

193 - request_prompts: Sequence[RequestPrompt] | None = []

194 - engine_prompts: list[EngineTokensPrompt] | None = []

195 -

196 - model_config = ConfigDict(arbitrary_types_allowed=True)

195 + request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list)

196 + engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list)

197

198

199 -class ResponseGenerationMixin(BaseModel):

199 +@dataclass(kw_only=True)

200 +class ResponseGenerationMixin:

201 """

202 Mixin for response generation,

203 managing result generators and final batch results.

@@ -205,54 +206,38 @@ class ResponseGenerationMixin(BaseModel):

206 result_generator: (

207 AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None

208 ) = None

208 - final_res_batch: list[RequestOutput | PoolingRequestOutput] = Field(

209 + final_res_batch: list[RequestOutput | PoolingRequestOutput] = field(

210 default_factory=list

211 )

212

213 model_config = ConfigDict(arbitrary_types_allowed=True)

214

215

215 -class ServeContext(

216 - RequestProcessingMixin,

217 - ResponseGenerationMixin,

218 - BaseModel,

219 - Generic[RequestT],

220 -):

216 +@dataclass(kw_only=True)

217 +class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]):

218 # Shared across all requests

219 request: RequestT

220 raw_request: Request | None = None

221 model_name: str

222 request_id: str

226 - created_time: int = Field(default_factory=lambda: int(time.time()))

223 + created_time: int = field(default_factory=lambda: int(time.time()))

224 lora_request: LoRARequest | None = None

225

226 # Shared across most requests

230 - tokenizer: AnyTokenizer | None = None

231 -

232 - # `protected_namespaces` resolves Pydantic v2's warning

233 - # on conflict with protected namespace "model_"

234 - model_config = ConfigDict(

235 - protected_namespaces=(),

236 - arbitrary_types_allowed=True,

237 - )

227 + tokenizer: TokenizerLike | None = None

228

229

240 -ClassificationServeContext = ServeContext[ClassificationRequest]

230 +@dataclass(kw_only=True)

231 +class ClassificationServeContext(ServeContext[ClassificationRequest]):

232 + pass

233

234

235 +@dataclass(kw_only=True)

236 class EmbeddingServeContext(ServeContext[EmbeddingRequest]):

237 chat_template: str | None = None

238 chat_template_content_format: ChatTemplateContentFormatOption

239

240

248 -# Used to resolve the Pydantic error related to

249 -# forward reference of MultiModalDataDict in TokensPrompt

250 -RequestProcessingMixin.model_rebuild()

251 -ServeContext.model_rebuild()

252 -ClassificationServeContext.model_rebuild()

253 -EmbeddingServeContext.model_rebuild()

254 -

255 -

241 class OpenAIServing:

242 request_id_prefix: ClassVar[str] = """

243 A short string prepended to every request’s ID (e.g. "embd", "classify")

@@ -281,7 +266,7 @@ class OpenAIServing:

266 apply_mistral_chat_template, executor=self._tokenizer_executor

267 )

268

284 - self._async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer] = {}

269 + self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {}

270 self.log_error_stack = log_error_stack

271

272 self.input_processor = self.models.input_processor

@@ -291,7 +276,7 @@ class OpenAIServing:

276

277 def _get_tool_parser(

278 self, tool_parser_name: str | None = None, enable_auto_tools: bool = False

294 - ) -> Callable[[AnyTokenizer], ToolParser] | None:

279 + ) -> Callable[[TokenizerLike], ToolParser] | None:

280 """Get the tool parser based on the name."""

281 parser = None

282 if not enable_auto_tools or tool_parser_name is None:

@@ -317,7 +302,7 @@ class OpenAIServing:

302 def _get_reasoning_parser(

303 self,

304 reasoning_parser_name: str,

320 - ) -> Callable[[AnyTokenizer], ReasoningParser] | None:

305 + ) -> Callable[[TokenizerLike], ReasoningParser] | None:

306 """Get the reasoning parser based on the name."""

307 parser = None

308 if not reasoning_parser_name:

@@ -547,7 +532,7 @@ class OpenAIServing:

532 prompt_logprobs=None,

533 )

534

550 - def _get_renderer(self, tokenizer: AnyTokenizer | None) -> BaseRenderer:

535 + def _get_renderer(self, tokenizer: TokenizerLike | None) -> BaseRenderer:

536 """

537 Get a Renderer instance with the provided tokenizer.

538 Uses shared async tokenizer pool for efficiency.

@@ -877,7 +862,7 @@ class OpenAIServing:

862 self,

863 request: AnyRequest,

864 prompt: str,

880 - tokenizer: AnyTokenizer,

865 + tokenizer: TokenizerLike,

866 add_special_tokens: bool,

867 ) -> TextTokensPrompt:

868 async_tokenizer = self._get_async_tokenizer(tokenizer)

@@ -919,7 +904,7 @@ class OpenAIServing:

904 self,

905 request: AnyRequest,

906 prompt_ids: list[int],

922 - tokenizer: AnyTokenizer | None,

907 + tokenizer: TokenizerLike | None,

908 ) -> TextTokensPrompt:

909 truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None)

910

@@ -1015,7 +1000,7 @@ class OpenAIServing:

1000 async def _tokenize_prompt_input_async(

1001 self,

1002 request: AnyRequest,

1018 - tokenizer: AnyTokenizer,

1003 + tokenizer: TokenizerLike,

1004 prompt_input: str | list[int],

1005 add_special_tokens: bool = True,

1006 ) -> TextTokensPrompt:

@@ -1034,7 +1019,7 @@ class OpenAIServing:

1019 async def _tokenize_prompt_inputs_async(

1020 self,

1021 request: AnyRequest,

1037 - tokenizer: AnyTokenizer,

1022 + tokenizer: TokenizerLike,

1023 prompt_inputs: Iterable[str | list[int]],

1024 add_special_tokens: bool = True,

1025 ) -> AsyncGenerator[TextTokensPrompt, None]:

@@ -1079,7 +1064,7 @@ class OpenAIServing:

1064 async def _preprocess_chat(

1065 self,

1066 request: ChatLikeRequest | ResponsesRequest,

1082 - tokenizer: AnyTokenizer,

1067 + tokenizer: TokenizerLike | None,

1068 messages: list[ChatCompletionMessageParam],

1069 chat_template: str | None,

1070 chat_template_content_format: ChatTemplateContentFormatOption,

@@ -1088,13 +1073,18 @@ class OpenAIServing:

1073 tool_dicts: list[dict[str, Any]] | None = None,

1074 documents: list[dict[str, str]] | None = None,

1075 chat_template_kwargs: dict[str, Any] | None = None,

1091 - tool_parser: Callable[[AnyTokenizer], ToolParser] | None = None,

1076 + tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,

1077 add_special_tokens: bool = False,

1078 ) -> tuple[

1079 list[ConversationMessage],

1080 Sequence[RequestPrompt],

1081 list[EngineTokensPrompt],

1082 ]:

1083 + if tokenizer is None:

1084 + raise ValueError(

1085 + "Unable to get tokenizer because `skip_tokenizer_init=True`"

1086 + )

1087 +

1088 model_config = self.model_config

1089

1090 resolved_content_format = resolve_chat_template_content_format(

@@ -1370,9 +1360,9 @@ class OpenAIServing:

1360 @staticmethod

1361 def _parse_tool_calls_from_content(

1362 request: ResponsesRequest | ChatCompletionRequest,

1373 - tokenizer: AnyTokenizer,

1363 + tokenizer: TokenizerLike,

1364 enable_auto_tools: bool,

1375 - tool_parser_cls: Callable[[AnyTokenizer], ToolParser] | None,

1365 + tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,

1366 content: str | None = None,

1367 ) -> tuple[list[FunctionCall] | None, str | None]:

1368 function_calls = list[FunctionCall]()

@@ -1442,7 +1432,7 @@ class OpenAIServing:

1432 def _get_decoded_token(

1433 logprob: Logprob,

1434 token_id: int,

1445 - tokenizer: AnyTokenizer,

1435 + tokenizer: TokenizerLike | None,

1436 return_as_token_id: bool = False,

1437 ) -> str:

1438 if return_as_token_id:

@@ -1450,6 +1440,12 @@ class OpenAIServing:

1440

1441 if logprob.decoded_token is not None:

1442 return logprob.decoded_token

1443 +

1444 + if tokenizer is None:

1445 + raise ValueError(

1446 + "Unable to get tokenizer because `skip_tokenizer_init=True`"

1447 + )

1448 +

1449 return tokenizer.decode(token_id)

1450

1451 def _is_model_supported(self, model_name: str | None) -> bool:

vllm/entrypoints/openai/serving_responses.py

+10 -10

@@ -105,7 +105,7 @@ from vllm.logprobs import Logprob as SampleLogprob

105 from vllm.logprobs import SampleLogprobs

106 from vllm.outputs import CompletionOutput

107 from vllm.sampling_params import SamplingParams, StructuredOutputsParams

108 -from vllm.transformers_utils.tokenizer import AnyTokenizer

108 +from vllm.tokenizers import TokenizerLike

109 from vllm.utils import random_uuid

110

111 logger = init_logger(__name__)

@@ -492,7 +492,7 @@ class OpenAIServingResponses(OpenAIServing):

492 self,

493 request: ResponsesRequest,

494 prev_response: ResponsesResponse | None,

495 - tokenizer: AnyTokenizer,

495 + tokenizer: TokenizerLike,

496 ):

497 if request.tools is None or (

498 request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none

@@ -563,7 +563,7 @@ class OpenAIServingResponses(OpenAIServing):

563 result_generator: AsyncIterator[ConversationContext],

564 context: ConversationContext,

565 model_name: str,

566 - tokenizer: AnyTokenizer,

566 + tokenizer: TokenizerLike,

567 request_metadata: RequestResponseMetadata,

568 created_time: int | None = None,

569 ) -> ErrorResponse | ResponsesResponse:

@@ -675,7 +675,7 @@ class OpenAIServingResponses(OpenAIServing):

675 self,

676 logprobs: dict[int, SampleLogprob],

677 top_logprobs: int,

678 - tokenizer: AnyTokenizer,

678 + tokenizer: TokenizerLike,

679 ) -> list[LogprobTopLogprob]:

680 """Returns the top-k logprobs from the logprobs dictionary."""

681 out = []

@@ -700,7 +700,7 @@ class OpenAIServingResponses(OpenAIServing):

700 self,

701 token_ids: Sequence[int],

702 logprobs: SampleLogprobs | None,

703 - tokenizer: AnyTokenizer,

703 + tokenizer: TokenizerLike,

704 top_logprobs: int | None = None,

705 ) -> list[Logprob]:

706 assert logprobs is not None, "logprobs must be provided"

@@ -736,7 +736,7 @@ class OpenAIServingResponses(OpenAIServing):

736 self,

737 token_ids: Sequence[int],

738 logprobs: SampleLogprobs | None,

739 - tokenizer: AnyTokenizer,

739 + tokenizer: TokenizerLike,

740 top_logprobs: int | None = None,

741 ) -> list[response_text_delta_event.Logprob]:

742 lgs = self._create_response_logprobs(

@@ -763,7 +763,7 @@ class OpenAIServingResponses(OpenAIServing):

763 self,

764 request: ResponsesRequest,

765 final_output: CompletionOutput,

766 - tokenizer: AnyTokenizer,

766 + tokenizer: TokenizerLike,

767 ) -> list[ResponseOutputItem]:

768 if self.reasoning_parser:

769 try:

@@ -1135,7 +1135,7 @@ class OpenAIServingResponses(OpenAIServing):

1135 result_generator: AsyncIterator[ConversationContext | None],

1136 context: ConversationContext,

1137 model_name: str,

1138 - tokenizer: AnyTokenizer,

1138 + tokenizer: TokenizerLike,

1139 request_metadata: RequestResponseMetadata,

1140 created_time: int,

1141 _increment_sequence_number_and_return: Callable[

@@ -1438,7 +1438,7 @@ class OpenAIServingResponses(OpenAIServing):

1438 result_generator: AsyncIterator[ConversationContext | None],

1439 context: ConversationContext,

1440 model_name: str,

1441 - tokenizer: AnyTokenizer,

1441 + tokenizer: TokenizerLike,

1442 request_metadata: RequestResponseMetadata,

1443 created_time: int,

1444 _increment_sequence_number_and_return: Callable[

@@ -1891,7 +1891,7 @@ class OpenAIServingResponses(OpenAIServing):

1891 result_generator: AsyncIterator[ConversationContext | None],

1892 context: ConversationContext,

1893 model_name: str,

1894 - tokenizer: AnyTokenizer,

1894 + tokenizer: TokenizerLike,

1895 request_metadata: RequestResponseMetadata,

1896 created_time: int | None = None,

1897 ) -> AsyncGenerator[StreamingResponsesResponse, None]:

vllm/entrypoints/openai/serving_score.py

+4 -4

@@ -36,7 +36,7 @@ from vllm.inputs.data import TokensPrompt

36 from vllm.logger import init_logger

37 from vllm.lora.request import LoRARequest

38 from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput

39 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

39 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

40 from vllm.utils.async_utils import make_async, merge_async_iterators

41

42 logger = init_logger(__name__)

@@ -60,7 +60,7 @@ class ServingScores(OpenAIServing):

60

61 async def _embedding_score(

62 self,

63 - tokenizer: AnyTokenizer,

63 + tokenizer: TokenizerLike,

64 texts_1: list[str],

65 texts_2: list[str],

66 request: RerankRequest | ScoreRequest,

@@ -153,7 +153,7 @@ class ServingScores(OpenAIServing):

153 def _preprocess_score(

154 self,

155 request: RerankRequest | ScoreRequest,

156 - tokenizer: AnyTokenizer,

156 + tokenizer: TokenizerLike,

157 tokenization_kwargs: dict[str, Any],

158 data_1: str | ScoreContentPartParam,

159 data_2: str | ScoreContentPartParam,

@@ -175,7 +175,7 @@ class ServingScores(OpenAIServing):

175

176 async def _cross_encoding_score(

177 self,

178 - tokenizer: AnyTokenizer,

178 + tokenizer: TokenizerLike,

179 data_1: list[str] | list[ScoreContentPartParam],

180 data_2: list[str] | list[ScoreContentPartParam],

181 request: RerankRequest | ScoreRequest,

vllm/entrypoints/openai/serving_tokenization.py

+2 -2

@@ -22,7 +22,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing

22 from vllm.entrypoints.openai.serving_models import OpenAIServingModels

23 from vllm.entrypoints.renderer import RenderConfig

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26

27 logger = init_logger(__name__)

28

@@ -170,7 +170,7 @@ class OpenAIServingTokenization(OpenAIServing):

170

171 @dataclass

172 class TokenizerInfo:

173 - tokenizer: AnyTokenizer

173 + tokenizer: TokenizerLike

174 chat_template: str | None

175

176 def to_dict(self) -> dict[str, Any]:

vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py

+2 -2

@@ -22,7 +22,7 @@ from vllm.logger import init_logger

22 from vllm.sampling_params import (

23 StructuredOutputsParams,

24 )

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26 from vllm.utils.collection_utils import is_list_of

27 from vllm.utils.import_utils import import_from_path

28

@@ -36,7 +36,7 @@ class ToolParser:

36 derived classes.

37 """

38

39 - def __init__(self, tokenizer: AnyTokenizer):

39 + def __init__(self, tokenizer: TokenizerLike):

40 self.prev_tool_call_arr: list[dict] = []

41 # the index of the tool call that is currently being parsed

42 self.current_tool_id: int = -1

vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py

+2 -2

@@ -19,13 +19,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

19 ToolParser,

20 )

21 from vllm.logger import init_logger

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

22 +from vllm.tokenizers import TokenizerLike

23

24 logger = init_logger(__name__)

25

26

27 class DeepSeekV31ToolParser(ToolParser):

28 - def __init__(self, tokenizer: AnyTokenizer):

28 + def __init__(self, tokenizer: TokenizerLike):

29 super().__init__(tokenizer)

30

31 self.current_tool_name_sent: bool = False

vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py

+2 -2

@@ -19,13 +19,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

19 ToolParser,

20 )

21 from vllm.logger import init_logger

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

22 +from vllm.tokenizers import TokenizerLike

23

24 logger = init_logger(__name__)

25

26

27 class DeepSeekV3ToolParser(ToolParser):

28 - def __init__(self, tokenizer: AnyTokenizer):

28 + def __init__(self, tokenizer: TokenizerLike):

29 super().__init__(tokenizer)

30

31 self.current_tool_name_sent: bool = False

vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py

+2 -2

@@ -19,13 +19,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

19 ToolParser,

20 )

21 from vllm.logger import init_logger

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

22 +from vllm.tokenizers import TokenizerLike

23

24 logger = init_logger(__name__)

25

26

27 class Ernie45ToolParser(ToolParser):

28 - def __init__(self, tokenizer: AnyTokenizer):

28 + def __init__(self, tokenizer: TokenizerLike):

29 """

30 Ernie thinking model format:

31 abc\n</think>\n\n\n<tool_call>\ndef\n</tool_call>\n

vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py

+2 -2

@@ -22,13 +22,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 ToolParser,

23 )

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26

27 logger = init_logger(__name__)

28

29

30 class Glm4MoeModelToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33 self.current_tool_name_sent = False

34 self.prev_tool_call_arr: list[dict] = []

vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py

+2 -2

@@ -29,7 +29,7 @@ from vllm.entrypoints.openai.tool_parsers.utils import (

29 partial_json_loads,

30 )

31 from vllm.logger import init_logger

32 -from vllm.transformers_utils.tokenizer import AnyTokenizer

32 +from vllm.tokenizers import TokenizerLike

33

34 logger = init_logger(__name__)

35

@@ -44,7 +44,7 @@ class Granite20bFCToolParser(ToolParser):

44 are all set

45 """

46

47 - def __init__(self, tokenizer: AnyTokenizer):

47 + def __init__(self, tokenizer: TokenizerLike):

48 super().__init__(tokenizer)

49

50 self.bot_token = "<function_call>"

vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py

+2 -2

@@ -27,7 +27,7 @@ from vllm.entrypoints.openai.tool_parsers.utils import (

27 partial_json_loads,

28 )

29 from vllm.logger import init_logger

30 -from vllm.transformers_utils.tokenizer import AnyTokenizer

30 +from vllm.tokenizers import TokenizerLike

31

32 logger = init_logger(__name__)

33

@@ -42,7 +42,7 @@ class GraniteToolParser(ToolParser):

42 are all set

43 """

44

45 - def __init__(self, tokenizer: AnyTokenizer):

45 + def __init__(self, tokenizer: TokenizerLike):

46 super().__init__(tokenizer)

47 # for granite 3.0, the token `<|tool_call|>`

48 self.bot_token = "<|tool_call|>"

vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py

+4 -4

@@ -22,18 +22,18 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 ToolParser,

23 )

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

25 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

26

27 logger = init_logger(__name__)

28

29

30 class Hermes2ProToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33

34 - if isinstance(self.model_tokenizer, MistralTokenizer):

34 + if isinstance(tokenizer, MistralTokenizer):

35 logger.error("Detected Mistral tokenizer when using a Hermes model")

36 - self.model_tokenizer = self.model_tokenizer.tokenizer

36 + self.model_tokenizer = tokenizer.tokenizer

37

38 self.current_tool_name_sent: bool = False

39 self.prev_tool_call_arr: list[dict] = []

vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py

+2 -2

@@ -22,14 +22,14 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 )

23 from vllm.entrypoints.openai.tool_parsers.utils import consume_space

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26 from vllm.utils import random_uuid

27

28 logger = init_logger(__name__)

29

30

31 class HunyuanA13BToolParser(ToolParser):

32 - def __init__(self, tokenizer: AnyTokenizer):

32 + def __init__(self, tokenizer: TokenizerLike):

33 super().__init__(tokenizer)

34

35 # Initialize state for streaming mode

vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py

+2 -2

@@ -22,13 +22,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 )

23 from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26

27 logger = init_logger(__name__)

28

29

30 class Internlm2ToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33 self.position = 0

34

vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py

+2 -3

@@ -21,14 +21,13 @@ from vllm.entrypoints.openai.protocol import (

21 from vllm.entrypoints.openai.tool_parsers import ToolParser

22 from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff

23 from vllm.logger import init_logger

24 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 -from vllm.transformers_utils.tokenizers import MistralTokenizer

24 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

25

26 logger = init_logger(__name__)

27

28

29 class JambaToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

30 + def __init__(self, tokenizer: TokenizerLike):

31 super().__init__(tokenizer)

32

33 if isinstance(self.model_tokenizer, MistralTokenizer):

vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py

+2 -2

@@ -19,13 +19,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

19 ToolParser,

20 )

21 from vllm.logger import init_logger

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

22 +from vllm.tokenizers import TokenizerLike

23

24 logger = init_logger(__name__)

25

26

27 class KimiK2ToolParser(ToolParser):

28 - def __init__(self, tokenizer: AnyTokenizer):

28 + def __init__(self, tokenizer: TokenizerLike):

29 super().__init__(tokenizer)

30 self.current_tool_name_sent: bool = False

31 self.prev_tool_call_arr: list[dict] = []

vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py

+2 -2

@@ -4,11 +4,11 @@

4 import regex as re

5

6 from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser

7 -from vllm.transformers_utils.tokenizer import AnyTokenizer

7 +from vllm.tokenizers import TokenizerLike

8

9

10 class LongcatFlashToolParser(Hermes2ProToolParser):

11 - def __init__(self, tokenizer: AnyTokenizer):

11 + def __init__(self, tokenizer: TokenizerLike):

12 super().__init__(tokenizer)

13

14 self.tool_call_start_token: str = "<longcat_tool_call>"

vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py

+2 -2

@@ -21,13 +21,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

21 ToolParser,

22 )

23 from vllm.logger import init_logger

24 -from vllm.transformers_utils.tokenizer import AnyTokenizer

24 +from vllm.tokenizers import TokenizerLike

25

26 logger = init_logger(__name__)

27

28

29 class MinimaxM2ToolParser(ToolParser):

30 - def __init__(self, tokenizer: AnyTokenizer):

30 + def __init__(self, tokenizer: TokenizerLike):

31 super().__init__(tokenizer)

32

33 self.prev_tool_call_arr: list[dict] = []

vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py

+2 -2

@@ -22,13 +22,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 )

23 from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26

27 logger = init_logger(__name__)

28

29

30 class MinimaxToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33

34 # Initialize streaming state for tracking tool call progress

vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py

+3 -3

@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

25 )

26 from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff

27 from vllm.logger import init_logger

28 -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

28 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

29

30 logger = init_logger(__name__)

31

@@ -46,7 +46,7 @@ class MistralToolCall(ToolCall):

46 return id.isalnum() and len(id) == 9

47

48

49 -def _is_fn_name_regex_support(model_tokenizer: AnyTokenizer) -> bool:

49 +def _is_fn_name_regex_support(model_tokenizer: TokenizerLike) -> bool:

50 return (

51 isinstance(model_tokenizer, MistralTokenizer) and model_tokenizer.version >= 11

52 )

@@ -61,7 +61,7 @@ class MistralToolParser(ToolParser):

61 Used when --enable-auto-tool-choice --tool-call-parser mistral are all set

62 """

63

64 - def __init__(self, tokenizer: AnyTokenizer):

64 + def __init__(self, tokenizer: TokenizerLike):

65 super().__init__(tokenizer)

66

67 if not isinstance(self.model_tokenizer, MistralTokenizer):

vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py

+3 -3

@@ -18,15 +18,15 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

18 from vllm.logger import init_logger

19

20 if TYPE_CHECKING:

21 - from vllm.transformers_utils.tokenizer import AnyTokenizer

21 + from vllm.tokenizers import TokenizerLike

22 else:

23 - AnyTokenizer = object

23 + TokenizerLike = object

24

25 logger = init_logger(__name__)

26

27

28 class OpenAIToolParser(ToolParser):

29 - def __init__(self, tokenizer: "AnyTokenizer"):

29 + def __init__(self, tokenizer: "TokenizerLike"):

30 super().__init__(tokenizer)

31

32 def extract_tool_calls(

vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py

+2 -2

@@ -22,13 +22,13 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

22 ToolParser,

23 )

24 from vllm.logger import init_logger

25 -from vllm.transformers_utils.tokenizer import AnyTokenizer

25 +from vllm.tokenizers import TokenizerLike

26

27 logger = init_logger(__name__)

28

29

30 class Qwen3CoderToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33

34 self.current_tool_name_sent: bool = False

vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py

+2 -2

@@ -23,7 +23,7 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

23 ToolParser,

24 )

25 from vllm.logger import init_logger

26 -from vllm.transformers_utils.tokenizer import AnyTokenizer

26 +from vllm.tokenizers import TokenizerLike

27

28 logger = init_logger(__name__)

29

@@ -1165,7 +1165,7 @@ class StreamingXMLToolCallParser:

1165

1166

1167 class Qwen3XMLToolParser(ToolParser):

1168 - def __init__(self, tokenizer: AnyTokenizer):

1168 + def __init__(self, tokenizer: TokenizerLike):

1169 super().__init__(tokenizer)

1170 self.parser = StreamingXMLToolCallParser()

1171

vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py

+2 -2

@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

25 ToolParser,

26 )

27 from vllm.logger import init_logger

28 -from vllm.transformers_utils.tokenizer import AnyTokenizer

28 +from vllm.tokenizers import TokenizerLike

29

30 logger = init_logger(__name__)

31

@@ -34,7 +34,7 @@ class SeedOssToolParser(ToolParser):

34 TOOL_CALL_START = "<seed:tool_call>"

35 TOOL_CALL_END = "</seed:tool_call>"

36

37 - def __init__(self, tokenizer: AnyTokenizer):

37 + def __init__(self, tokenizer: TokenizerLike):

38 super().__init__(tokenizer)

39

40 # --- streaming state ---

vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py

+2 -2

@@ -21,7 +21,7 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

21 ToolParser,

22 )

23 from vllm.logger import init_logger

24 -from vllm.transformers_utils.tokenizer import AnyTokenizer

24 +from vllm.tokenizers import TokenizerLike

25 from vllm.utils import random_uuid

26

27 logger = init_logger(__name__)

@@ -41,7 +41,7 @@ class Step3ToolParser(ToolParser):

41 TOOL_SEP = "<｜tool_sep｜>"

42 SPECIAL_TOKENS = [TOOL_CALLS_BEGIN, TOOL_CALLS_END, TOOL_CALL_BEGIN, TOOL_CALL_END]

43

44 - def __init__(self, tokenizer: AnyTokenizer):

44 + def __init__(self, tokenizer: TokenizerLike):

45 super().__init__(tokenizer)

46 self.position = 0

47 # Explicit state flags for robust streaming

vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py

+2 -2

@@ -21,14 +21,14 @@ from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (

21 ToolParser,

22 )

23 from vllm.logger import init_logger

24 -from vllm.transformers_utils.tokenizer import AnyTokenizer

24 +from vllm.tokenizers import TokenizerLike

25 from vllm.utils import random_uuid

26

27 logger = init_logger(__name__)

28

29

30 class xLAMToolParser(ToolParser):

31 - def __init__(self, tokenizer: AnyTokenizer):

31 + def __init__(self, tokenizer: TokenizerLike):

32 super().__init__(tokenizer)

33

34 # Initialize state for streaming mode

vllm/entrypoints/renderer.py

+5 -5

@@ -16,7 +16,7 @@ from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt

16 from vllm.inputs.data import TextPrompt as EngineTextPrompt

17 from vllm.inputs.data import TokensPrompt as EngineTokensPrompt

18 from vllm.inputs.parse import get_prompt_components, parse_raw_prompts

19 -from vllm.transformers_utils.tokenizer import AnyTokenizer

19 +from vllm.tokenizers import TokenizerLike

20 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

21

22

@@ -85,7 +85,7 @@ class BaseRenderer(ABC):

85 def __init__(

86 self,

87 model_config: ModelConfig,

88 - tokenizer: AnyTokenizer | None = None,

88 + tokenizer: TokenizerLike | None = None,

89 ):

90 super().__init__()

91 self.model_config = model_config

@@ -200,8 +200,8 @@ class CompletionRenderer(BaseRenderer):

200 def __init__(

201 self,

202 model_config: ModelConfig,

203 - tokenizer: AnyTokenizer | None = None,

204 - async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer]

203 + tokenizer: TokenizerLike | None = None,

204 + async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer]

205 | None = None,

206 ):

207 super().__init__(model_config, tokenizer)

@@ -373,7 +373,7 @@ class CompletionRenderer(BaseRenderer):

373 return async_tokenizer

374

375 tokenizer = self.tokenizer

376 - if self.tokenizer is None:

376 + if tokenizer is None:

377 raise ValueError("No tokenizer available for text input processing")

378

379 if self.async_tokenizer_pool is None:

vllm/entrypoints/score_utils.py

+8 -10

@@ -19,11 +19,7 @@ from vllm.inputs import TokensPrompt

19 from vllm.model_executor.models.interfaces import supports_score_template

20 from vllm.multimodal.inputs import MultiModalDataDict

21 from vllm.outputs import PoolingRequestOutput

22 -from vllm.transformers_utils.tokenizer import (

23 - AnyTokenizer,

24 - PreTrainedTokenizer,

25 - PreTrainedTokenizerFast,

26 -)

22 +from vllm.transformers_utils.tokenizer import TokenizerLike

23

24 ScoreContentPartParam: TypeAlias = (

25 ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam

@@ -45,7 +41,7 @@ class ScoreMultiModalParam(TypedDict, total=False):

41

42

43 def _cosine_similarity(

48 - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,

44 + tokenizer: TokenizerLike,

45 embed_1: list[PoolingRequestOutput],

46 embed_2: list[PoolingRequestOutput],

47 ) -> list[PoolingRequestOutput]:

@@ -93,7 +89,7 @@ def parse_score_data(

89 data_1: str | ScoreContentPartParam,

90 data_2: str | ScoreContentPartParam,

91 model_config: ModelConfig,

96 - tokenizer: AnyTokenizer,

92 + tokenizer: TokenizerLike,

93 ) -> tuple[str, str, MultiModalDataDict | None]:

94 mm_tracker = MultiModalItemTracker(model_config, tokenizer)

95

@@ -118,12 +114,14 @@ def _parse_score_content(

114 mm_tracker: BaseMultiModalItemTracker,

115 ) -> _ContentPart | None:

116 if isinstance(data, str):

121 - data = ChatCompletionContentPartTextParam(type="text", text=data)

117 + part = ChatCompletionContentPartTextParam(type="text", text=data)

118 + else:

119 + part = data

120

121 mm_parser = mm_tracker.create_parser()

122

123 parse_res = _parse_chat_message_content_part(

126 - data,

124 + part,

125 mm_parser,

126 wrap_dicts=False,

127 interleave_strings=False,

@@ -181,7 +179,7 @@ def post_process_tokens(

179

180 def get_score_prompt(

181 model_config: ModelConfig,

184 - tokenizer: AnyTokenizer,

182 + tokenizer: TokenizerLike,

183 tokenization_kwargs: dict[str, Any],

184 data_1: str | ScoreContentPartParam,

185 data_2: str | ScoreContentPartParam,

vllm/entrypoints/utils.py

+1 -1

@@ -30,7 +30,7 @@ from vllm.entrypoints.openai.protocol import (

30 from vllm.entrypoints.openai.serving_models import LoRAModulePath

31 from vllm.logger import init_logger

32 from vllm.platforms import current_platform

33 -from vllm.transformers_utils.tokenizers import MistralTokenizer

33 +from vllm.tokenizers import MistralTokenizer

34 from vllm.utils.argparse_utils import FlexibleArgumentParser

35

36 logger = init_logger(__name__)

vllm/inputs/preprocess.py

+5 -5

@@ -17,7 +17,7 @@ from vllm.multimodal.inputs import (

17 MultiModalUUIDDict,

18 )

19 from vllm.multimodal.processing import BaseMultiModalProcessor

20 -from vllm.transformers_utils.tokenizer import AnyTokenizer

20 +from vllm.tokenizers import TokenizerLike

21 from vllm.utils.jsontree import json_iter_leaves

22 from vllm.v1.metrics.stats import MultiModalCacheStats

23

@@ -46,7 +46,7 @@ class InputPreprocessor:

46 def __init__(

47 self,

48 model_config: ModelConfig,

49 - tokenizer: AnyTokenizer | None,

49 + tokenizer: TokenizerLike | None,

50 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

51 mm_processor_cache: BaseMultiModalProcessorCache | None = None,

52 ) -> None:

@@ -59,7 +59,7 @@ class InputPreprocessor:

59

60 self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None

61

62 - def get_tokenizer(self) -> AnyTokenizer:

62 + def get_tokenizer(self) -> TokenizerLike:

63 if self.tokenizer is None:

64 raise ValueError(

65 "You cannot pass text prompts when `skip_tokenizer_init` is True"

@@ -228,11 +228,11 @@ class InputPreprocessor:

228

229 return tokenizer.encode(prompt, **tokenization_kwargs)

230

231 - def _get_mm_tokenizer(self) -> AnyTokenizer:

231 + def _get_mm_tokenizer(self) -> TokenizerLike:

232 # PrithviGeoSpatialMAE needs to be initialized without a tokenizer

233 # while using also multi-modal input

234 if not self.tokenizer:

235 - return cast(AnyTokenizer, object()) # Dummy

235 + return cast(TokenizerLike, object()) # Dummy

236

237 tokenizer = self.get_tokenizer()

238 return tokenizer

vllm/logits_process.py

+2 -2

@@ -5,7 +5,7 @@ from typing import TypeAlias

5

6 import torch

7

8 -from vllm.transformers_utils.tokenizer import AnyTokenizer

8 +from vllm.tokenizers import TokenizerLike

9

10 LogitsProcessor: TypeAlias = (

11 Callable[[list[int], torch.Tensor], torch.Tensor]

@@ -19,7 +19,7 @@ to sample from."""

19

20

21 def get_bad_words_logits_processors(

22 - bad_words: list[str], tokenizer: AnyTokenizer

22 + bad_words: list[str], tokenizer: TokenizerLike

23 ) -> list[LogitsProcessor]:

24 bad_words_ids: list[list[int]] = list()

25

vllm/model_executor/models/h2ovl.py

+2 -2

@@ -28,7 +28,7 @@ from vllm.multimodal.processing import (

28 PromptUpdate,

29 PromptUpdateDetails,

30 )

31 -from vllm.transformers_utils.tokenizer import AnyTokenizer

31 +from vllm.tokenizers import TokenizerLike

32

33 from .intern_vit import InternVisionModel

34 from .internvl import (

@@ -241,7 +241,7 @@ class H2OVLProcessor(BaseInternVLProcessor):

241 def __init__(

242 self,

243 config: PretrainedConfig,

244 - tokenizer: AnyTokenizer,

244 + tokenizer: TokenizerLike,

245 *,

246 min_dynamic_patch: int | None = None,

247 max_dynamic_patch: int | None = None,

vllm/model_executor/models/internvl.py

+3 -3

@@ -50,7 +50,7 @@ from vllm.multimodal.processing import (

50 )

51 from vllm.multimodal.profiling import BaseDummyInputsBuilder

52 from vllm.sequence import IntermediateTensors

53 -from vllm.transformers_utils.tokenizer import AnyTokenizer

53 +from vllm.tokenizers import TokenizerLike

54 from vllm.utils.tensor_schema import TensorSchema, TensorShape

55 from vllm.utils.torch_utils import set_default_torch_num_threads

56

@@ -347,7 +347,7 @@ class BaseInternVLProcessor(ABC):

347 def __init__(

348 self,

349 config: PretrainedConfig,

350 - tokenizer: AnyTokenizer,

350 + tokenizer: TokenizerLike,

351 *,

352 min_dynamic_patch: int | None = None,

353 max_dynamic_patch: int | None = None,

@@ -561,7 +561,7 @@ class InternVLProcessor(BaseInternVLProcessor):

561 def __init__(

562 self,

563 config: PretrainedConfig,

564 - tokenizer: AnyTokenizer,

564 + tokenizer: TokenizerLike,

565 *,

566 min_dynamic_patch: int | None = None,

567 max_dynamic_patch: int | None = None,

vllm/model_executor/models/nano_nemotron_vl.py

+5 -5

@@ -73,9 +73,9 @@ from vllm.multimodal.processing import (

73 )

74 from vllm.multimodal.profiling import BaseDummyInputsBuilder

75 from vllm.sequence import IntermediateTensors

76 +from vllm.tokenizers import TokenizerLike

77 from vllm.transformers_utils.configs.radio import RadioConfig

78 from vllm.transformers_utils.tokenizer import (

78 - AnyTokenizer,

79 cached_tokenizer_from_config,

80 encode_tokens,

81 )

@@ -284,7 +284,7 @@ class BaseNanoNemotronVLProcessor(ABC):

284 def __init__(

285 self,

286 config: PretrainedConfig,

287 - tokenizer: AnyTokenizer,

287 + tokenizer: TokenizerLike,

288 *args,

289 max_num_tiles: int | None = None,

290 **kwargs,

@@ -434,7 +434,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

434 def __init__(

435 self,

436 config: PretrainedConfig,

437 - tokenizer: AnyTokenizer,

437 + tokenizer: TokenizerLike,

438 *,

439 max_num_tiles: int | None = None,

440 min_dynamic_patch: int | None = None,

@@ -645,7 +645,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

645 tokens_per_frame: list[int],

646 frames_indices: list[int],

647 frame_duration_ms: int,

648 - tokenizer: AnyTokenizer,

648 + tokenizer: TokenizerLike,

649 img_start_token_ids: list[int],

650 img_end_token_ids: list[int],

651 img_context_token_ids: list[int],

@@ -670,7 +670,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

670 tokens_per_frame (list[int]): number of tokens per frame

671 frames_indices (list[int]): frame indices

672 frame_duration_ms (int): duration of each frame in milliseconds

673 - tokenizer (AnyTokenizer): tokenizer to use for tokenizing frame separators

673 + tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators

674 img_start_token_ids (list[int]): pre-tokenized IMG_START tokens

675 img_end_token_ids (list[int]): pre-tokenized IMG_END tokens

676 img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens

vllm/model_executor/models/nemotron_vl.py

+2 -2

@@ -34,8 +34,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY

34 from vllm.multimodal.image import convert_image_mode

35 from vllm.multimodal.processing import PromptUpdateDetails

36 from vllm.sequence import IntermediateTensors

37 +from vllm.tokenizers import TokenizerLike

38 from vllm.transformers_utils.processor import cached_image_processor_from_config

38 -from vllm.transformers_utils.tokenizer import AnyTokenizer

39

40 from .interfaces import (

41 MultiModalEmbeddings,

@@ -203,7 +203,7 @@ class NemotronVLProcessor(InternVLProcessor):

203 def __init__(

204 self,

205 config: PretrainedConfig,

206 - tokenizer: AnyTokenizer,

206 + tokenizer: TokenizerLike,

207 image_processor: BaseImageProcessorFast,

208 *,

209 min_dynamic_patch: int | None = None,

vllm/model_executor/models/opencua.py

+2 -2

@@ -31,7 +31,7 @@ from vllm.multimodal.processing import (

31 PromptReplacement,

32 PromptUpdate,

33 )

34 -from vllm.transformers_utils.tokenizer import AnyTokenizer

34 +from vllm.tokenizers import TokenizerLike

35

36 from .qwen2_5_vl import (

37 Qwen2_5_VisionTransformer as OpenCUAVisionTransformer,

@@ -79,7 +79,7 @@ class OpenCUAProcessor(Qwen2VLProcessor):

79 def __init__(

80 self,

81 vision_config: dict,

82 - tokenizer: AnyTokenizer,

82 + tokenizer: TokenizerLike,

83 **kwargs,

84 ):

85 image_processor = Qwen2VLImageProcessor(**vision_config)

vllm/model_executor/models/pixtral.py

+2 -4

@@ -59,10 +59,8 @@ from vllm.multimodal.processing import (

59 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

60 from vllm.platforms import current_platform

61 from vllm.sequence import IntermediateTensors

62 -from vllm.transformers_utils.tokenizer import (

63 - MistralTokenizer,

64 - cached_tokenizer_from_config,

65 -)

62 +from vllm.tokenizers import MistralTokenizer

63 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

64 from vllm.utils.tensor_schema import TensorSchema, TensorShape

65

66 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

vllm/model_executor/models/qwen2_vl.py

+2 -2

@@ -91,7 +91,7 @@ from vllm.multimodal.processing import (

91 )

92 from vllm.multimodal.profiling import BaseDummyInputsBuilder

93 from vllm.sequence import IntermediateTensors

94 -from vllm.transformers_utils.tokenizer import AnyTokenizer

94 +from vllm.tokenizers import TokenizerLike

95 from vllm.utils.tensor_schema import TensorSchema, TensorShape

96

97 from .interfaces import (

@@ -1533,7 +1533,7 @@ class Tarsier2Processor(Qwen2VLProcessor):

1533 def __init__(

1534 self,

1535 vision_config: dict,

1536 - tokenizer: AnyTokenizer,

1536 + tokenizer: TokenizerLike,

1537 **kwargs,

1538 ):

1539 self.image_processor = Tarsier2ImageProcessor(**vision_config)

vllm/model_executor/models/skyworkr1v.py

+2 -2

@@ -47,7 +47,7 @@ from vllm.multimodal.processing import (

47 )

48 from vllm.multimodal.profiling import BaseDummyInputsBuilder

49 from vllm.sequence import IntermediateTensors

50 -from vllm.transformers_utils.tokenizer import AnyTokenizer

50 +from vllm.tokenizers import TokenizerLike

51 from vllm.utils.tensor_schema import TensorSchema, TensorShape

52

53 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

@@ -282,7 +282,7 @@ class SkyworkR1VProcessor:

282 def __init__(

283 self,

284 config: PretrainedConfig,

285 - tokenizer: AnyTokenizer,

285 + tokenizer: TokenizerLike,

286 *,

287 min_dynamic_patch: int | None = None,

288 max_dynamic_patch: int | None = None,

vllm/model_executor/models/step3_vl.py

+2 -2

@@ -43,8 +43,8 @@ from vllm.multimodal.processing import (

43 )

44 from vllm.multimodal.profiling import BaseDummyInputsBuilder

45 from vllm.sequence import IntermediateTensors

46 +from vllm.tokenizers import TokenizerLike

47 from vllm.transformers_utils.configs import Step3VisionEncoderConfig

47 -from vllm.transformers_utils.tokenizer import AnyTokenizer

48 from vllm.utils.tensor_schema import TensorSchema, TensorShape

49

50 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

@@ -321,7 +321,7 @@ class Step3VLProcessor:

321 def __init__(

322 self,

323 config: PretrainedConfig,

324 - tokenizer: AnyTokenizer,

324 + tokenizer: TokenizerLike,

325 ) -> None:

326 super().__init__()

327

vllm/model_executor/models/voxtral.py

+2 -4

@@ -51,10 +51,8 @@ from vllm.multimodal.processing import (

51 )

52 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

53 from vllm.sequence import IntermediateTensors

54 -from vllm.transformers_utils.tokenizer import (

55 - MistralTokenizer,

56 - cached_tokenizer_from_config,

57 -)

54 +from vllm.tokenizers import MistralTokenizer

55 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

56

57 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription

58 from .utils import init_vllm_registered_model, maybe_prefix

vllm/multimodal/processing.py

+22 -21

@@ -23,8 +23,9 @@ import torch

23 from typing_extensions import TypeVar, assert_never

24

25 from vllm.logger import init_logger

26 +from vllm.tokenizers import TokenizerLike

27 from vllm.transformers_utils.processor import cached_processor_from_config

27 -from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens

28 +from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens

29 from vllm.utils.collection_utils import flatten_2d_lists, full_groupby

30 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

31 from vllm.utils.jsontree import JSONTree, json_map_leaves

@@ -76,7 +77,7 @@ PromptSeq: TypeAlias = str | list[int]

77

78 @lru_cache(maxsize=2048)

79 def _cached_encode(

79 - tokenizer: AnyTokenizer,

80 + tokenizer: TokenizerLike,

81 text: str,

82 *,

83 add_special_tokens: bool | None = None,

@@ -86,7 +87,7 @@ def _cached_encode(

87

88 @lru_cache(maxsize=2048)

89 def _cached_decode(

89 - tokenizer: AnyTokenizer,

90 + tokenizer: TokenizerLike,

91 token_ids: tuple[int, ...],

92 *,

93 skip_special_tokens: bool | None = None,

@@ -96,14 +97,14 @@ def _cached_decode(

97 )

98

99

99 -def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str:

100 +def _seq2text(tokenizer: TokenizerLike, seq: PromptSeq) -> str:

101 if isinstance(seq, str):

102 return seq

103

104 return _cached_decode(tokenizer, tuple(seq))

105

106

106 -def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:

107 +def _seq2tokens(tokenizer: TokenizerLike, seq: PromptSeq) -> list[int]:

108 if isinstance(seq, str):

109 return _cached_encode(tokenizer, seq, add_special_tokens=False)

110

@@ -113,7 +114,7 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]:

114 class _GetMatchIndex(Protocol):

115 def __call__(

116 self,

116 - tokenizer: AnyTokenizer,

117 + tokenizer: TokenizerLike,

118 prompt: PromptSeq,

119 start_idx: int = 0,

120 ) -> int | None: ...

@@ -143,7 +144,7 @@ class PromptIndexTargets:

144 """

145

146 def get_match_index(

146 - tokenizer: AnyTokenizer,

147 + tokenizer: TokenizerLike,

148 prompt: PromptSeq,

149 start_idx: int = 0,

150 ) -> int | None:

@@ -199,7 +200,7 @@ class PromptUpdateDetails(Generic[_S]):

200 full: _S

201 """The full content."""

202

202 - is_embed: Callable[[AnyTokenizer, PromptSeq], torch.Tensor] | None = None

203 + is_embed: Callable[[TokenizerLike, PromptSeq], torch.Tensor] | None = None

204 """

205 Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],

206 return a boolean mask of shape `(len(full),)` indicating which positions

@@ -220,7 +221,7 @@ class PromptUpdateDetails(Generic[_S]):

221 seq: _S,

222 embed_text: str,

223 ) -> "PromptUpdateDetails[_S]":

223 - def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:

224 + def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor:

225 embed_token_ids = encode_tokens(tokenizer, embed_text)

226 token_ids = _seq2tokens(tokenizer, full)

227

@@ -236,7 +237,7 @@ class PromptUpdateDetails(Generic[_S]):

237 seq: _S,

238 embed_token_id: int,

239 ) -> "PromptUpdateDetails[_S]":

239 - def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor:

240 + def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor:

241 token_ids = _seq2tokens(tokenizer, full)

242

243 return torch.tensor(token_ids) == embed_token_id

@@ -522,7 +523,7 @@ class ResolvedPromptUpdate:

523 def iter_token_matches(

524 self,

525 prompt: list[int],

525 - tokenizer: AnyTokenizer,

526 + tokenizer: TokenizerLike,

527 *,

528 start_idx: int = 0,

529 ) -> Generator[PromptTargetMatch]:

@@ -544,7 +545,7 @@ class ResolvedPromptUpdate:

545 def iter_text_matches(

546 self,

547 prompt: str,

547 - tokenizer: AnyTokenizer,

548 + tokenizer: TokenizerLike,

549 *,

550 start_idx: int = 0,

551 ) -> Generator[PromptTargetMatch]:

@@ -566,7 +567,7 @@ class ResolvedPromptUpdate:

567 def iter_matches(

568 self,

569 prompt: list[int] | str,

569 - tokenizer: AnyTokenizer,

570 + tokenizer: TokenizerLike,

571 *,

572 start_idx: int = 0,

573 ) -> Generator[PromptTargetMatch]:

@@ -675,7 +676,7 @@ _MatchToApply = tuple[tuple[str, int], tuple[PromptTargetMatch, int]]

676 def _find_matches(

677 prompt: _S,

678 mm_prompt_updates: "MultiModalPromptUpdates",

678 - tokenizer: AnyTokenizer,

679 + tokenizer: TokenizerLike,

680 *,

681 prev_end_idx: int = 0,

682 current_result: "MultiModalPromptUpdatesApplyResult",

@@ -740,7 +741,7 @@ def _all_items_found(

741 def _apply_matches(

742 prompt: _S,

743 mm_prompt_updates: "MultiModalPromptUpdates",

743 - tokenizer: AnyTokenizer,

744 + tokenizer: TokenizerLike,

745 ) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]:

746 mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()}

747

@@ -806,7 +807,7 @@ def _apply_matches(

807 def apply_token_matches(

808 prompt: list[int],

809 mm_prompt_updates: "MultiModalPromptUpdates",

809 - tokenizer: AnyTokenizer,

810 + tokenizer: TokenizerLike,

811 ) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]:

812 """

813 Apply the updates in `mm_prompt_updates` to `prompt`.

@@ -823,7 +824,7 @@ def apply_token_matches(

824 def apply_text_matches(

825 prompt: str,

826 mm_prompt_updates: "MultiModalPromptUpdates",

826 - tokenizer: AnyTokenizer,

827 + tokenizer: TokenizerLike,

828 ) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]:

829 """

830 Apply the updates in `mm_prompt_updates` to `prompt`.

@@ -840,7 +841,7 @@ def apply_text_matches(

841 def _iter_placeholders(

842 prompt: list[int],

843 mm_prompt_updates: "MultiModalPromptUpdates",

843 - tokenizer: AnyTokenizer,

844 + tokenizer: TokenizerLike,

845 ) -> Iterable[PlaceholderFeaturesInfo]:

846 """

847 Yield each set of placeholder tokens found in `prompt`.

@@ -909,7 +910,7 @@ def _iter_placeholders(

910 def find_mm_placeholders(

911 prompt: list[int],

912 mm_prompt_updates: "MultiModalPromptUpdates",

912 - tokenizer: AnyTokenizer,

913 + tokenizer: TokenizerLike,

914 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:

915 it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer)

916 return dict(full_groupby_modality(it))

@@ -930,7 +931,7 @@ class InputProcessingContext:

931 model_config: ModelConfig

932 """The configuration of the model."""

933

933 - tokenizer: AnyTokenizer

934 + tokenizer: TokenizerLike

935 """The tokenizer used to tokenize the inputs."""

936

937 @overload

@@ -1146,7 +1147,7 @@ class BaseProcessingInfo:

1147 def model_id(self) -> str:

1148 return self.ctx.model_config.model

1149

1149 - def get_tokenizer(self) -> AnyTokenizer:

1150 + def get_tokenizer(self) -> TokenizerLike:

1151 return self.ctx.tokenizer

1152

1153 def get_hf_config(self) -> PretrainedConfig:

vllm/multimodal/registry.py

+9 -5

@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast

6

7 from vllm.config.multimodal import BaseDummyOptions

8 from vllm.logger import init_logger

9 -from vllm.transformers_utils.tokenizer import AnyTokenizer, cached_tokenizer_from_config

9 +from vllm.tokenizers import TokenizerLike

10 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

11

12 from .cache import BaseMultiModalProcessorCache

13 from .processing import (

@@ -231,17 +232,20 @@ class MultiModalRegistry:

232 def _create_processing_ctx(

233 self,

234 model_config: "ModelConfig",

234 - tokenizer: AnyTokenizer | None = None,

235 + tokenizer: TokenizerLike | None = None,

236 ) -> InputProcessingContext:

236 - if tokenizer is None and not model_config.skip_tokenizer_init:

237 + if model_config.skip_tokenizer_init:

238 + tokenizer = cast(TokenizerLike, object())

239 + elif tokenizer is None:

240 tokenizer = cached_tokenizer_from_config(model_config)

241 +

242 return InputProcessingContext(model_config, tokenizer)

243

244 def _create_processing_info(

245 self,

246 model_config: "ModelConfig",

247 *,

244 - tokenizer: AnyTokenizer | None = None,

248 + tokenizer: TokenizerLike | None = None,

249 ) -> BaseProcessingInfo:

250 model_cls = self._get_model_cls(model_config)

251 factories = model_cls._processor_factory

@@ -252,7 +256,7 @@ class MultiModalRegistry:

256 self,

257 model_config: "ModelConfig",

258 *,

255 - tokenizer: AnyTokenizer | None = None,

259 + tokenizer: TokenizerLike | None = None,

260 cache: BaseMultiModalProcessorCache | None = None,

261 ) -> BaseMultiModalProcessor[BaseProcessingInfo]:

262 """

vllm/reasoning/abs_reasoning_parsers.py

+3 -3

@@ -19,12 +19,12 @@ if TYPE_CHECKING:

19 DeltaMessage,

20 ResponsesRequest,

21 )

22 - from vllm.transformers_utils.tokenizer import AnyTokenizer

22 + from vllm.tokenizers import TokenizerLike

23 else:

24 ChatCompletionRequest = Any

25 DeltaMessage = Any

26 ResponsesRequest = Any

27 - AnyTokenizer = Any

27 + TokenizerLike = Any

28

29 logger = init_logger(__name__)

30

@@ -37,7 +37,7 @@ class ReasoningParser:

37 It is used to extract reasoning content from the model output.

38 """

39

40 - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs):

40 + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):

41 self.model_tokenizer = tokenizer

42

43 @cached_property

vllm/reasoning/basic_parsers.py

+2 -2

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any

7

8 from vllm.entrypoints.openai.protocol import DeltaMessage

9 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

10 -from vllm.transformers_utils.tokenizer import AnyTokenizer

10 +from vllm.tokenizers import TokenizerLike

11

12 if TYPE_CHECKING:

13 from vllm.entrypoints.openai.protocol import (

@@ -43,7 +43,7 @@ class BaseThinkingReasoningParser(ReasoningParser):

43 """The token that ends reasoning content."""

44 raise NotImplementedError

45

46 - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs):

46 + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):

47 super().__init__(tokenizer, *args, **kwargs)

48

49 if not self.model_tokenizer:

vllm/reasoning/minimax_m2_reasoning_parser.py

+2 -2

@@ -11,7 +11,7 @@ from vllm.entrypoints.openai.protocol import (

11 from vllm.logger import init_logger

12 from vllm.reasoning.abs_reasoning_parsers import ReasoningParser

13 from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser

14 -from vllm.transformers_utils.tokenizer import AnyTokenizer

14 +from vllm.tokenizers import TokenizerLike

15

16 logger = init_logger(__name__)

17

@@ -37,7 +37,7 @@ class MiniMaxM2AppendThinkReasoningParser(ReasoningParser):

37 Reasoning parser for MiniMax M2 model.

38 """

39

40 - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs):

40 + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):

41 super().__init__(tokenizer, *args, **kwargs)

42 self.end_token_id = self.vocab.get("</think>")

43

vllm/reasoning/mistral_reasoning_parser.py

+1 -1

@@ -6,7 +6,7 @@ from functools import cached_property

6 from vllm.logger import init_logger

7 from vllm.reasoning import ReasoningParser

8 from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser

9 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

9 +from vllm.tokenizers import MistralTokenizer

10

11 logger = init_logger(__name__)

12

vllm/reasoning/olmo3_reasoning_parser.py

+2 -2

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING

9 import regex as re

10

11 if TYPE_CHECKING:

12 - from vllm.transformers_utils.tokenizer import AnyTokenizer

12 + from vllm.tokenizers import TokenizerLike

13

14 from vllm.entrypoints.openai.protocol import (

15 ChatCompletionRequest,

@@ -220,7 +220,7 @@ class Olmo3ReasoningParser(ReasoningParser):

220 token is missing from generation.

221 """

222

223 - def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs):

223 + def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs):

224 super().__init__(tokenizer, *args, **kwargs)

225

226 self.think_start = r"<think>"

vllm/sampling_params.py

+2 -2

@@ -13,7 +13,7 @@ from pydantic.dataclasses import dataclass

13

14 from vllm.logger import init_logger

15 from vllm.logits_process import LogitsProcessor

16 -from vllm.transformers_utils.tokenizer import AnyTokenizer

16 +from vllm.tokenizers import TokenizerLike

17 from vllm.v1.serial_utils import PydanticMsgspecMixin

18

19 logger = init_logger(__name__)

@@ -477,7 +477,7 @@ class SamplingParams(

477 eos_ids.update(self.stop_token_ids)

478 self.stop_token_ids = list(eos_ids)

479

480 - def update_from_tokenizer(self, tokenizer: AnyTokenizer) -> None:

480 + def update_from_tokenizer(self, tokenizer: TokenizerLike) -> None:

481 if not self.bad_words:

482 return

483 self._bad_words_token_ids = []

vllm/tokenizers/__init__.py new

+8

@@ -0,0 +1,8 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from .mistral import MistralTokenizer

5 +from .protocol import TokenizerLike

6 +from .registry import TokenizerRegistry

7 +

8 +__all__ = ["TokenizerLike", "MistralTokenizer", "TokenizerRegistry"]

vllm/tokenizers/mistral.py renamed

+7 -22

@@ -4,7 +4,8 @@

4 from typing import TYPE_CHECKING, Any, cast

5

6 from vllm.logger import init_logger

7 -from vllm.transformers_utils.tokenizer_base import TokenizerBase

7 +

8 +from .protocol import TokenizerLike

9

10 if TYPE_CHECKING:

11 from mistral_common.protocol.instruct.request import (

@@ -163,7 +164,7 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:

164 return tokenizer.unk_id

165

166

166 -class MistralTokenizer(TokenizerBase):

167 +class MistralTokenizer(TokenizerLike):

168 def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None:

169 from mistral_common.protocol.instruct.validator import ValidationMode

170 from mistral_common.tokens.tokenizers.sentencepiece import (

@@ -270,14 +271,6 @@ class MistralTokenizer(TokenizerBase):

271 def eos_token_id(self) -> int:

272 return self.tokenizer.eos_id

273

273 - @property

274 - def sep_token(self) -> str:

275 - raise NotImplementedError()

276 -

277 - @property

278 - def pad_token(self) -> str:

279 - return self.transformers_tokenizer.pad_token

280 -

274 @property

275 def is_fast(self) -> bool:

276 return True

@@ -292,11 +285,14 @@ class MistralTokenizer(TokenizerBase):

285

286 @property

287 def truncation_side(self) -> str:

295 - raise NotImplementedError()

288 + return self.transformers_tokenizer.truncation_side

289

290 def _is_special_token_id(self, token_id: int) -> bool:

291 return token_id in self._special_token_ids_set

292

293 + def __hash__(self) -> int:

294 + return hash(id(self))

295 +

296 def __len__(self) -> int:

297 return self.vocab_size

298

@@ -341,17 +337,6 @@ class MistralTokenizer(TokenizerBase):

337 # Mistral tokenizers have no added vocabulary

338 return {}

339

344 - def encode_one(

345 - self,

346 - text: str,

347 - truncation: bool = False,

348 - max_length: int | None = None,

349 - ) -> list[int]:

350 - # Mistral Tokenizers should not add special tokens

351 - return self.transformers_tokenizer.encode(

352 - text, add_special_tokens=False, truncation=truncation, max_length=max_length

353 - )

354 -

340 def encode(

341 self,

342 text: str,

vllm/tokenizers/protocol.py new

+105

@@ -0,0 +1,105 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +from typing import TYPE_CHECKING, Any, Protocol

5 +

6 +from typing_extensions import Self

7 +

8 +if TYPE_CHECKING:

9 + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

10 +

11 +

12 +class TokenizerLike(Protocol):

13 + @classmethod

14 + def from_pretrained(

15 + cls,

16 + pretrained_model_name_or_path: str,

17 + /,

18 + *,

19 + revision: str | None = None,

20 + ) -> Self:

21 + raise NotImplementedError

22 +

23 + @property

24 + def all_special_tokens(self) -> list[str]:

25 + raise NotImplementedError

26 +

27 + @property

28 + def all_special_ids(self) -> list[int]:

29 + raise NotImplementedError

30 +

31 + @property

32 + def bos_token_id(self) -> int:

33 + raise NotImplementedError

34 +

35 + @property

36 + def eos_token_id(self) -> int:

37 + raise NotImplementedError

38 +

39 + @property

40 + def is_fast(self) -> bool:

41 + raise NotImplementedError

42 +

43 + @property

44 + def vocab_size(self) -> int:

45 + raise NotImplementedError

46 +

47 + @property

48 + def max_token_id(self) -> int:

49 + raise NotImplementedError

50 +

51 + @property

52 + def truncation_side(self) -> str:

53 + raise NotImplementedError

54 +

55 + def __hash__(self) -> int:

56 + return hash(id(self))

57 +

58 + def __len__(self) -> int:

59 + return self.vocab_size

60 +

61 + def __call__(

62 + self,

63 + text: str | list[str] | list[int],

64 + text_pair: str | None = None,

65 + add_special_tokens: bool = False,

66 + truncation: bool = False,

67 + max_length: int | None = None,

68 + ):

69 + raise NotImplementedError

70 +

71 + def get_vocab(self) -> dict[str, int]:

72 + raise NotImplementedError

73 +

74 + def get_added_vocab(self) -> dict[str, int]:

75 + raise NotImplementedError

76 +

77 + def encode(

78 + self,

79 + text: str,

80 + truncation: bool | None = None,

81 + max_length: int | None = None,

82 + add_special_tokens: bool | None = None,

83 + ) -> list[int]:

84 + raise NotImplementedError

85 +

86 + def apply_chat_template(

87 + self,

88 + messages: list["ChatCompletionMessageParam"],

89 + tools: list[dict[str, Any]] | None = None,

90 + **kwargs,

91 + ) -> list[int]:

92 + raise NotImplementedError

93 +

94 + def convert_tokens_to_string(self, tokens: list[str]) -> str:

95 + raise NotImplementedError

96 +

97 + def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:

98 + raise NotImplementedError

99 +

100 + def convert_ids_to_tokens(

101 + self,

102 + ids: list[int],

103 + skip_special_tokens: bool = True,

104 + ) -> list[str]:

105 + raise NotImplementedError

vllm/tokenizers/registry.py new

+28

@@ -0,0 +1,28 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +import importlib

4 +

5 +from .protocol import TokenizerLike

6 +

7 +

8 +class TokenizerRegistry:

9 + # Tokenizer name -> (tokenizer module, tokenizer class)

10 + REGISTRY: dict[str, tuple[str, str]] = {}

11 +

12 + @staticmethod

13 + def register(name: str, module: str, class_name: str) -> None:

14 + TokenizerRegistry.REGISTRY[name] = (module, class_name)

15 +

16 + @staticmethod

17 + def get_tokenizer(

18 + tokenizer_name: str,

19 + *args,

20 + **kwargs,

21 + ) -> "TokenizerLike":

22 + tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name)

23 + if tokenizer_cls is None:

24 + raise ValueError(f"Tokenizer {tokenizer_name} not found.")

25 +

26 + tokenizer_module = importlib.import_module(tokenizer_cls[0])

27 + class_ = getattr(tokenizer_module, tokenizer_cls[1])

28 + return class_.from_pretrained(*args, **kwargs)

vllm/transformers_utils/config.py

+4 -3

@@ -26,8 +26,9 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME

26

27 from vllm import envs

28 from vllm.logger import init_logger

29 -from vllm.transformers_utils.config_parser_base import ConfigParserBase

30 -from vllm.transformers_utils.repo_utils import (

29 +

30 +from .config_parser_base import ConfigParserBase

31 +from .repo_utils import (

32 _get_hf_token,

33 file_or_path_exists,

34 get_hf_file_to_dict,

@@ -35,7 +36,7 @@ from vllm.transformers_utils.repo_utils import (

36 try_get_local_file,

37 with_retry,

38 )

38 -from vllm.transformers_utils.utils import (

39 +from .utils import (

40 check_gguf_file,

41 is_gguf,

42 is_remote_gguf,

vllm/transformers_utils/detokenizer_utils.py

+5 -5

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4

5 -from .tokenizer import AnyTokenizer

5 +from vllm.tokenizers import TokenizerLike

6

7

8 def _replace_none_with_empty(tokens: list[str | None]):

@@ -12,7 +12,7 @@ def _replace_none_with_empty(tokens: list[str | None]):

12

13

14 def _convert_tokens_to_string_with_added_encoders(

15 - tokenizer: AnyTokenizer,

15 + tokenizer: TokenizerLike,

16 output_tokens: list[str],

17 skip_special_tokens: bool,

18 spaces_between_special_tokens: bool,

@@ -57,7 +57,7 @@ INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5

57

58

59 def convert_prompt_ids_to_tokens(

60 - tokenizer: AnyTokenizer,

60 + tokenizer: TokenizerLike,

61 prompt_ids: list[int],

62 skip_special_tokens: bool = False,

63 ) -> tuple[list[str], int, int]:

@@ -81,7 +81,7 @@ def convert_prompt_ids_to_tokens(

81

82

83 def convert_ids_list_to_tokens(

84 - tokenizer: AnyTokenizer,

84 + tokenizer: TokenizerLike,

85 token_ids: list[int],

86 ) -> list[str]:

87 """Detokenize the input ids individually.

@@ -108,7 +108,7 @@ def convert_ids_list_to_tokens(

108 # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15

109 # under Apache 2.0 license

110 def detokenize_incrementally(

111 - tokenizer: AnyTokenizer,

111 + tokenizer: TokenizerLike,

112 all_input_ids: list[int],

113 prev_tokens: list[str] | None,

114 prefix_offset: int,

vllm/transformers_utils/gguf_utils.py

+2 -1

@@ -9,7 +9,8 @@ from gguf.constants import Keys, VisionProjectorType

9 from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig

10

11 from vllm.logger import init_logger

12 -from vllm.transformers_utils.repo_utils import list_filtered_repo_files

12 +

13 +from .repo_utils import list_filtered_repo_files

14

15 logger = init_logger(__name__)

16

vllm/transformers_utils/tokenizer.py

+33 -27

@@ -5,41 +5,48 @@ import contextlib

5 import copy

6 import importlib.util

7 import os

8 +import warnings

9 from functools import lru_cache

10 from pathlib import Path

10 -from typing import TYPE_CHECKING, Any, TypeAlias

11 +from typing import TYPE_CHECKING, Any

12

13 import huggingface_hub

13 -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast

14 +from transformers import AutoTokenizer, PreTrainedTokenizerBase

15 from typing_extensions import assert_never

16

17 from vllm import envs

18 from vllm.logger import init_logger

18 -from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config

19 -from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf

20 -from vllm.transformers_utils.repo_utils import list_filtered_repo_files

21 -from vllm.transformers_utils.tokenizers import MistralTokenizer

22 -from vllm.transformers_utils.utils import (

23 - check_gguf_file,

24 - is_gguf,

25 - is_remote_gguf,

26 - split_remote_gguf,

27 -)

19 +from vllm.tokenizers import MistralTokenizer, TokenizerLike, TokenizerRegistry

20 +

21 +from .config import get_sentence_transformer_tokenizer_config

22 +from .gguf_utils import get_gguf_file_path_from_hf

23 +from .repo_utils import list_filtered_repo_files

24 +from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf

25

26 if TYPE_CHECKING:

27 from vllm.config import ModelConfig

31 - from vllm.transformers_utils.tokenizer_base import TokenizerBase

32 -else:

33 - ModelConfig = Any

34 - TokenizerBase = Any

28 +

29

30 logger = init_logger(__name__)

31

38 -AnyTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast | TokenizerBase

32 +

33 +def __getattr__(name: str):

34 + if name == "AnyTokenizer":

35 + warnings.warn(

36 + "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been moved to "

37 + "`vllm.tokenizers.TokenizerLike`. "

38 + "The old name will be removed in v0.13.",

39 + DeprecationWarning,

40 + stacklevel=2,

41 + )

42 +

43 + return TokenizerLike

44 +

45 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

46

47

48 def decode_tokens(

42 - tokenizer: AnyTokenizer,

49 + tokenizer: TokenizerLike,

50 token_ids: list[int],

51 *,

52 skip_special_tokens: bool | None = None,

@@ -58,7 +65,7 @@ def decode_tokens(

65

66

67 def encode_tokens(

61 - tokenizer: AnyTokenizer,

68 + tokenizer: TokenizerLike,

69 text: str,

70 *,

71 truncation: bool | None = None,

@@ -86,7 +93,7 @@ def encode_tokens(

93 return tokenizer.encode(text, **kw_args)

94

95

89 -def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:

96 +def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:

97 """

98 By default, transformers will recompute multiple tokenizer properties

99 each time they are called, leading to a significant slowdown.

@@ -144,7 +151,7 @@ def get_tokenizer(

151 revision: str | None = None,

152 download_dir: str | None = None,

153 **kwargs,

147 -) -> AnyTokenizer:

154 +) -> TokenizerLike:

155 """Gets a tokenizer for the given model name via HuggingFace or ModelScope."""

156 if envs.VLLM_USE_MODELSCOPE:

157 # download model from ModelScope hub,

@@ -206,15 +213,13 @@ def get_tokenizer(

213 if len(files_list) > 0:

214 tokenizer_mode = "mistral"

215

209 - tokenizer: AnyTokenizer

216 + tokenizer: TokenizerLike

217 if tokenizer_mode == "mistral":

218 logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")

219 tokenizer = MistralTokenizer.from_pretrained(

220 str(tokenizer_name), revision=revision

221 )

222 elif tokenizer_mode == "custom":

216 - from vllm.transformers_utils.tokenizer_base import TokenizerRegistry

217 -

223 logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")

224 tokenizer = TokenizerRegistry.get_tokenizer(

225 str(tokenizer_name),

@@ -260,12 +265,13 @@ def get_tokenizer(

265 if isinstance(encoder_config, dict) and encoder_config.get(

266 "do_lower_case", False

267 ):

268 + assert isinstance(tokenizer, PreTrainedTokenizerBase)

269 special_tokens_map = {

270 k: v.lower() for k, v in tokenizer.special_tokens_map.items()

271 }

272 tokenizer.add_special_tokens(special_tokens_map)

273

268 - if not isinstance(tokenizer, PreTrainedTokenizerFast):

274 + if not tokenizer.is_fast:

275 logger.warning(

276 "Using a slow tokenizer. This might cause a significant "

277 "slowdown. Consider using a fast tokenizer instead."

@@ -279,7 +285,7 @@ cached_get_tokenizer = lru_cache(get_tokenizer)

285

286

287 def cached_tokenizer_from_config(

282 - model_config: ModelConfig,

288 + model_config: "ModelConfig",

289 **kwargs: Any,

290 ):

291 return cached_get_tokenizer(

@@ -291,7 +297,7 @@ def cached_tokenizer_from_config(

297 )

298

299

294 -def init_tokenizer_from_configs(model_config: ModelConfig):

300 +def init_tokenizer_from_configs(model_config: "ModelConfig"):

301 runner_type = model_config.runner_type

302 if runner_type == "generate" or runner_type == "draft":

303 truncation_side = "left"

vllm/transformers_utils/tokenizer_base.py

+23 -140

@@ -1,150 +1,33 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 -import importlib

5 -from abc import ABC, abstractmethod

6 -from typing import TYPE_CHECKING, Any

4 +import warnings

5

8 -if TYPE_CHECKING:

9 - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

6

7 +def __getattr__(name: str):

8 + if name == "TokenizerBase":

9 + from vllm.tokenizers import TokenizerLike

10

12 -class TokenizerBase(ABC):

13 - @property

14 - @abstractmethod

15 - def all_special_tokens(self) -> list[str]:

16 - raise NotImplementedError()

11 + warnings.warn(

12 + "`vllm.transformers_utils.tokenizer_base.TokenizerBase` has been "

13 + "moved to `vllm.tokenizers.TokenizerLike`. "

14 + "The old name will be removed in v0.13.",

15 + DeprecationWarning,

16 + stacklevel=2,

17 + )

18

18 - @property

19 - @abstractmethod

20 - def all_special_ids(self) -> list[int]:

21 - raise NotImplementedError()

19 + return TokenizerLike

20 + if name == "TokenizerRegistry":

21 + from vllm.tokenizers import TokenizerRegistry

22

23 - @property

24 - @abstractmethod

25 - def bos_token_id(self) -> int:

26 - raise NotImplementedError()

23 + warnings.warn(

24 + "`vllm.transformers_utils.tokenizer_base.TokenizerRegistry` has been "

25 + "moved to `vllm.tokenizers.TokenizerRegistry`. "

26 + "The old name will be removed in v0.13.",

27 + DeprecationWarning,

28 + stacklevel=2,

29 + )

30

28 - @property

29 - @abstractmethod

30 - def eos_token_id(self) -> int:

31 - raise NotImplementedError()

31 + return TokenizerRegistry

32

33 - @property

34 - @abstractmethod

35 - def sep_token(self) -> str:

36 - raise NotImplementedError()

37 -

38 - @property

39 - @abstractmethod

40 - def pad_token(self) -> str:

41 - raise NotImplementedError()

42 -

43 - @property

44 - @abstractmethod

45 - def is_fast(self) -> bool:

46 - raise NotImplementedError()

47 -

48 - @property

49 - @abstractmethod

50 - def vocab_size(self) -> int:

51 - raise NotImplementedError()

52 -

53 - @property

54 - @abstractmethod

55 - def max_token_id(self) -> int:

56 - raise NotImplementedError()

57 -

58 - @property

59 - @abstractmethod

60 - def truncation_side(self) -> str:

61 - raise NotImplementedError()

62 -

63 - def __len__(self) -> int:

64 - return self.vocab_size

65 -

66 - @abstractmethod

67 - def __call__(

68 - self,

69 - text: str | list[str] | list[int],

70 - text_pair: str | None = None,

71 - add_special_tokens: bool = False,

72 - truncation: bool = False,

73 - max_length: int | None = None,

74 - ):

75 - raise NotImplementedError()

76 -

77 - @abstractmethod

78 - def get_vocab(self) -> dict[str, int]:

79 - raise NotImplementedError()

80 -

81 - @abstractmethod

82 - def get_added_vocab(self) -> dict[str, int]:

83 - raise NotImplementedError()

84 -

85 - @abstractmethod

86 - def encode_one(

87 - self,

88 - text: str,

89 - truncation: bool = False,

90 - max_length: int | None = None,

91 - ) -> list[int]:

92 - raise NotImplementedError()

93 -

94 - @abstractmethod

95 - def encode(

96 - self,

97 - text: str,

98 - truncation: bool | None = None,

99 - max_length: int | None = None,

100 - add_special_tokens: bool | None = None,

101 - ) -> list[int]:

102 - raise NotImplementedError()

103 -

104 - @abstractmethod

105 - def apply_chat_template(

106 - self,

107 - messages: list["ChatCompletionMessageParam"],

108 - tools: list[dict[str, Any]] | None = None,

109 - **kwargs,

110 - ) -> list[int]:

111 - raise NotImplementedError()

112 -

113 - @abstractmethod

114 - def convert_tokens_to_string(self, tokens: list[str]) -> str:

115 - raise NotImplementedError()

116 -

117 - @abstractmethod

118 - def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str:

119 - raise NotImplementedError()

120 -

121 - @abstractmethod

122 - def convert_ids_to_tokens(

123 - self,

124 - ids: list[int],

125 - skip_special_tokens: bool = True,

126 - ) -> list[str]:

127 - raise NotImplementedError()

128 -

129 -

130 -class TokenizerRegistry:

131 - # Tokenizer name -> (tokenizer module, tokenizer class)

132 - REGISTRY: dict[str, tuple[str, str]] = {}

133 -

134 - @staticmethod

135 - def register(name: str, module: str, class_name: str) -> None:

136 - TokenizerRegistry.REGISTRY[name] = (module, class_name)

137 -

138 - @staticmethod

139 - def get_tokenizer(

140 - tokenizer_name: str,

141 - *args,

142 - **kwargs,

143 - ) -> TokenizerBase:

144 - tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name)

145 - if tokenizer_cls is None:

146 - raise ValueError(f"Tokenizer {tokenizer_name} not found.")

147 -

148 - tokenizer_module = importlib.import_module(tokenizer_cls[0])

149 - class_ = getattr(tokenizer_module, tokenizer_cls[1])

150 - return class_.from_pretrained(*args, **kwargs)

33 + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

vllm/transformers_utils/tokenizers/__init__.py deleted

-16

@@ -1,16 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from .mistral import (

5 - MistralTokenizer,

6 - maybe_serialize_tool_calls,

7 - truncate_tool_call_ids,

8 - validate_request_params,

9 -)

10 -

11 -__all__ = [

12 - "MistralTokenizer",

13 - "maybe_serialize_tool_calls",

14 - "truncate_tool_call_ids",

15 - "validate_request_params",

16 -]

vllm/v1/engine/async_llm.py

+9 -7

@@ -26,9 +26,10 @@ from vllm.plugins.io_processors import get_io_processor

26 from vllm.pooling_params import PoolingParams

27 from vllm.sampling_params import SamplingParams

28 from vllm.tasks import SupportedTask

29 +from vllm.tokenizers import TokenizerLike

30 from vllm.tracing import init_tracer

31 from vllm.transformers_utils.config import maybe_register_config_serialize_by_value

31 -from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs

32 +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs

33 from vllm.usage.usage_lib import UsageContext

34 from vllm.utils.async_utils import cancel_task_threadsafe

35 from vllm.utils.collection_utils import as_list

@@ -120,9 +121,10 @@ class AsyncLLM(EngineClient):

121 )

122

123 # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).

123 - stream_interval = self.vllm_config.scheduler_config.stream_interval

124 self.output_processor = OutputProcessor(

125 - self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval

125 + self.tokenizer,

126 + log_stats=self.log_stats,

127 + stream_interval=self.vllm_config.scheduler_config.stream_interval,

128 )

129 endpoint = self.observability_config.otlp_traces_endpoint

130 if endpoint is not None:

@@ -703,17 +705,17 @@ class AsyncLLM(EngineClient):

705 raise EngineGenerateError() from e

706

707 @property

706 - def tokenizer(self) -> AnyTokenizer | None:

708 + def tokenizer(self) -> TokenizerLike | None:

709 return self.input_processor.tokenizer

710

711 @tokenizer.setter

710 - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

712 + def tokenizer(self, tokenizer: TokenizerLike | None) -> None:

713 self.input_processor.tokenizer = tokenizer

714

713 - async def get_tokenizer(self) -> AnyTokenizer:

715 + async def get_tokenizer(self) -> TokenizerLike:

716 if self.tokenizer is None:

717 raise ValueError(

716 - "Unable to get tokenizer because skip_tokenizer_init is True"

718 + "Unable to get tokenizer because `skip_tokenizer_init=True`"

719 )

720

721 return self.tokenizer

vllm/v1/engine/detokenizer.py

+3 -3

@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizerFast

10

11 from vllm.logger import init_logger

12 from vllm.transformers_utils.detokenizer_utils import (

13 - AnyTokenizer,

13 + TokenizerLike,

14 convert_prompt_ids_to_tokens,

15 detokenize_incrementally,

16 )

@@ -45,7 +45,7 @@ class IncrementalDetokenizer:

45 @classmethod

46 def from_new_request(

47 cls,

48 - tokenizer: AnyTokenizer | None,

48 + tokenizer: TokenizerLike | None,

49 request: EngineCoreRequest,

50 ) -> "IncrementalDetokenizer":

51 assert request.sampling_params is not None

@@ -256,7 +256,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):

256

257

258 class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):

259 - def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest):

259 + def __init__(self, tokenizer: TokenizerLike, request: EngineCoreRequest):

260 super().__init__(request)

261

262 self.tokenizer = tokenizer

vllm/v1/engine/input_processor.py

+4 -5

@@ -19,8 +19,7 @@ from vllm.multimodal.processing import EncDecMultiModalProcessor

19 from vllm.multimodal.utils import argsort_mm_positions

20 from vllm.pooling_params import PoolingParams

21 from vllm.sampling_params import SamplingParams

22 -from vllm.transformers_utils.tokenizer import AnyTokenizer

23 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

22 +from vllm.tokenizers import MistralTokenizer, TokenizerLike

23 from vllm.utils import length_from_prompt_token_ids_or_embeds

24 from vllm.v1.engine import EngineCoreRequest

25 from vllm.v1.metrics.stats import MultiModalCacheStats

@@ -40,7 +39,7 @@ class InputProcessor:

39 def __init__(

40 self,

41 vllm_config: VllmConfig,

43 - tokenizer: AnyTokenizer | None,

42 + tokenizer: TokenizerLike | None,

43 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

44 ) -> None:

45 self.vllm_config = vllm_config

@@ -62,11 +61,11 @@ class InputProcessor:

61 )

62

63 @property

65 - def tokenizer(self) -> AnyTokenizer | None:

64 + def tokenizer(self) -> TokenizerLike | None:

65 return self.input_preprocessor.tokenizer

66

67 @tokenizer.setter

69 - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

68 + def tokenizer(self, tokenizer: TokenizerLike | None) -> None:

69 self.input_preprocessor.tokenizer = tokenizer

70

71 def _validate_logprobs(

vllm/v1/engine/llm_engine.py

+9 -7

@@ -23,8 +23,9 @@ from vllm.plugins.io_processors import get_io_processor

23 from vllm.pooling_params import PoolingParams

24 from vllm.sampling_params import SamplingParams

25 from vllm.tasks import SupportedTask

26 +from vllm.tokenizers import TokenizerLike

27 from vllm.tracing import init_tracer

27 -from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs

28 +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs

29 from vllm.usage.usage_lib import UsageContext

30 from vllm.v1.engine import EngineCoreRequest

31 from vllm.v1.engine.core_client import EngineCoreClient

@@ -95,9 +96,10 @@ class LLMEngine:

96 )

97

98 # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).

98 - stream_interval = self.vllm_config.scheduler_config.stream_interval

99 self.output_processor = OutputProcessor(

100 - self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval

100 + self.tokenizer,

101 + log_stats=self.log_stats,

102 + stream_interval=self.vllm_config.scheduler_config.stream_interval,

103 )

104 endpoint = self.observability_config.otlp_traces_endpoint

105 if endpoint is not None:

@@ -350,17 +352,17 @@ class LLMEngine:

352 return get_metrics_snapshot()

353

354 @property

353 - def tokenizer(self) -> AnyTokenizer | None:

355 + def tokenizer(self) -> TokenizerLike | None:

356 return self.input_processor.tokenizer

357

358 @tokenizer.setter

357 - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None:

359 + def tokenizer(self, tokenizer: TokenizerLike | None) -> None:

360 self.input_processor.tokenizer = tokenizer

361

360 - def get_tokenizer(self) -> AnyTokenizer:

362 + def get_tokenizer(self) -> TokenizerLike:

363 if self.tokenizer is None:

364 raise ValueError(

363 - "Unable to get tokenizer because skip_tokenizer_init is True"

365 + "Unable to get tokenizer because `skip_tokenizer_init=True`"

366 )

367

368 return self.tokenizer

vllm/v1/engine/logprobs.py

+3 -3

@@ -13,7 +13,7 @@ from vllm.logprobs import (

13 create_sample_logprobs,

14 )

15 from vllm.transformers_utils.detokenizer_utils import (

16 - AnyTokenizer,

16 + TokenizerLike,

17 convert_ids_list_to_tokens,

18 )

19 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest

@@ -28,7 +28,7 @@ NONES = itertools.repeat(None)

28 class LogprobsProcessor:

29 # Tokenizer for this request,

30 # None if detokenization is disabled.

31 - tokenizer: AnyTokenizer | None

31 + tokenizer: TokenizerLike | None

32

33 # Logprobs for this request

34 logprobs: SampleLogprobs | None

@@ -40,7 +40,7 @@ class LogprobsProcessor:

40 @classmethod

41 def from_new_request(

42 cls,

43 - tokenizer: AnyTokenizer | None,

43 + tokenizer: TokenizerLike | None,

44 request: EngineCoreRequest,

45 ) -> "LogprobsProcessor":

46 sampling_params = request.sampling_params

vllm/v1/engine/output_processor.py

+6 -3

@@ -15,8 +15,8 @@ from vllm.outputs import (

15 RequestOutput,

16 )

17 from vllm.sampling_params import RequestOutputKind

18 +from vllm.tokenizers import TokenizerLike

19 from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context

19 -from vllm.transformers_utils.tokenizer import AnyTokenizer

20 from vllm.utils import length_from_prompt_token_ids_or_embeds

21 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason

22 from vllm.v1.engine.detokenizer import IncrementalDetokenizer

@@ -139,7 +139,7 @@ class RequestState:

139 @classmethod

140 def from_new_request(

141 cls,

142 - tokenizer: AnyTokenizer,

142 + tokenizer: TokenizerLike | None,

143 request: EngineCoreRequest,

144 prompt: str | None,

145 parent_req: ParentRequest | None,

@@ -341,7 +341,10 @@ class OutputProcessor:

341 """Process EngineCoreOutputs into RequestOutputs."""

342

343 def __init__(

344 - self, tokenizer: AnyTokenizer, log_stats: bool, stream_interval: int = 1

344 + self,

345 + tokenizer: TokenizerLike | None,

346 + log_stats: bool,

347 + stream_interval: int = 1,

348 ):

349 self.log_stats = log_stats

350 self.tokenizer = tokenizer

vllm/v1/structured_output/backend_types.py

+3 -3

@@ -10,10 +10,10 @@ if TYPE_CHECKING:

10 import torch

11

12 from vllm.config import VllmConfig

13 - from vllm.transformers_utils.tokenizer import AnyTokenizer

13 + from vllm.tokenizers import TokenizerLike

14 else:

15 VllmConfig = object

16 - AnyTokenizer = object

16 + TokenizerLike = object

17

18

19 class StructuredOutputOptions(enum.Enum):

@@ -100,7 +100,7 @@ class StructuredOutputBackend(ABC):

100 """Engine-level backend for structured output requests."""

101

102 vllm_config: VllmConfig

103 - tokenizer: AnyTokenizer

103 + tokenizer: TokenizerLike

104 vocab_size: int

105

106 @abstractmethod

vllm/v1/structured_output/backend_xgrammar.py

+1 -1

@@ -10,7 +10,7 @@ import torch

10 import vllm.envs

11 from vllm.logger import init_logger

12 from vllm.sampling_params import SamplingParams

13 -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

13 +from vllm.tokenizers import MistralTokenizer

14 from vllm.utils.import_utils import LazyLoader

15 from vllm.v1.structured_output.backend_types import (

16 StructuredOutputBackend,

vllm/v1/structured_output/utils.py

+5 -5

@@ -24,7 +24,7 @@ if TYPE_CHECKING:

24 import transformers.models.gpt2.tokenization_gpt2 as tokenization_gpt2

25 import xgrammar as xgr

26

27 - from vllm.transformers_utils.tokenizer import AnyTokenizer

27 + from vllm.tokenizers import TokenizerLike

28 from vllm.v1.worker.gpu_input_batch import InputBatch

29 else:

30 xgr = LazyLoader("xgr", globals(), "xgrammar")

@@ -36,7 +36,7 @@ else:

36 "transformers.models.gpt2.tokenization_gpt2",

37 )

38

39 - AnyTokenizer = object

39 + TokenizerLike = object

40 SchedulerOutput = object

41 InputBatch = object

42

@@ -195,7 +195,7 @@ re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")

195

196

197 def _reduced_vocabulary(

198 - tokenizer: AnyTokenizer,

198 + tokenizer: TokenizerLike,

199 eos_token_id: int,

200 ) -> dict[bytes, list[int]]:

201 """Create a map from vocabulary tokens to lists of equivalent token ids.

@@ -222,7 +222,7 @@ def _reduced_vocabulary(

222 vocabulary: dict[bytes, list[int]] = {}

223 empty_token_ids: list[int] = []

224 for token, token_idx in tokenizer.get_vocab().items():

225 - if token in tokenizer.all_special_tokens: # type: ignore

225 + if token in tokenizer.all_special_tokens:

226 continue

227

228 token_str = convert_token_to_string(token)

@@ -261,7 +261,7 @@ def _reduced_vocabulary(

261 return vocabulary

262

263

264 -def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary:

264 +def get_outlines_vocabulary(tokenizer: TokenizerLike) -> oc.Vocabulary:

265 """Get the `Vocabulary` object for a given tokenizer."""

266 if hasattr(tokenizer, "_outlines_vocabulary"):

267 return tokenizer._outlines_vocabulary # type: ignore

[Misc] Refactor tokenizer interface (#29693)