dd94484577 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/backend_request_func.py

+2 -2

@@ -12,7 +12,7 @@ from dataclasses import dataclass, field

12 import aiohttp

13 import huggingface_hub.constants

14 from tqdm.asyncio import tqdm

15 -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast

15 +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend

16

17 # NOTE(simon): do not import vLLM here so the benchmark script

18 # can run without vLLM installed.

@@ -609,7 +609,7 @@ def get_tokenizer(

609 tokenizer_mode: str = "auto",

610 trust_remote_code: bool = False,

611 **kwargs,

612 -) -> PreTrainedTokenizer | PreTrainedTokenizerFast:

612 +) -> PythonBackend | TokenizersBackend:

613 if pretrained_model_name_or_path is not None and not os.path.exists(

614 pretrained_model_name_or_path

615 ):

docs/features/reasoning_outputs.md

+1 -1

@@ -439,7 +439,7 @@ Additionally, to enable structured output, you'll need to create a new `Reasoner

439 end_token: str = "</think>"

440

441 @classmethod

442 - def from_tokenizer(cls, tokenizer: PreTrainedTokenizer) -> Reasoner:

442 + def from_tokenizer(cls, tokenizer: PythonBackend) -> Reasoner:

443 return cls(

444 start_token_id=tokenizer.encode("<think>", add_special_tokens=False)[0],

445 end_token_id=tokenizer.encode("</think>", add_special_tokens=False)[0],

examples/features/prompt_embed/prompt_embed_offline.py

+4 -4

@@ -19,7 +19,7 @@ Run:

19 """

20

21 import torch

22 -from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizer

22 +from transformers import AutoModelForCausalLM, AutoTokenizer, PythonBackend

23

24 from vllm import LLM

25

@@ -34,7 +34,7 @@ def init_tokenizer_and_llm(model_name: str):

34

35 def get_prompt_embeds(

36 chat: list[dict[str, str]],

37 - tokenizer: PreTrainedTokenizer,

37 + tokenizer: PythonBackend,

38 embedding_layer: torch.nn.Module,

39 ):

40 token_ids = tokenizer.apply_chat_template(

@@ -45,7 +45,7 @@ def get_prompt_embeds(

45

46

47 def single_prompt_inference(

48 - llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module

48 + llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module

49 ):

50 chat = [{"role": "user", "content": "Please tell me about the capital of France."}]

51 prompt_embeds = get_prompt_embeds(chat, tokenizer, embedding_layer)

@@ -64,7 +64,7 @@ def single_prompt_inference(

64

65

66 def batch_prompt_inference(

67 - llm: LLM, tokenizer: PreTrainedTokenizer, embedding_layer: torch.nn.Module

67 + llm: LLM, tokenizer: PythonBackend, embedding_layer: torch.nn.Module

68 ):

69 chats = [

70 [{"role": "user", "content": "Please tell me about the capital of France."}],

requirements/test/cpu.txt

+2 -2

@@ -672,7 +672,7 @@ pathvalidate==3.2.1

672 # via pytablewriter

673 patsy==1.0.1

674 # via statsmodels

675 -peft==0.18.1

675 +peft==0.19.1

676 # via -r requirements/test/cuda.in

677 perceptron==0.1.4

678 # via -r requirements/test/cuda.in

@@ -1159,7 +1159,7 @@ tqdm==4.67.3

1159 # segmentation-models-pytorch

1160 # sentence-transformers

1161 # transformers

1162 -transformers==5.5.3

1162 +transformers==5.10.4

1163 # via

1164 # -r requirements/test/../common.txt

1165 # -r requirements/test/cuda.in

requirements/test/cuda.in

+2 -2

@@ -20,7 +20,7 @@ httpx

20 librosa # required for audio tests

21 vector_quantize_pytorch # required for minicpmo_26 test

22 vocos # required for minicpmo_26 test

23 -peft>=0.18.1 # required for phi-4-mm test

23 +peft>=0.19.1 # required for phi-4-mm test

24 pqdm

25 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests

26 sentence-transformers>=5.2.0 # required for embedding tests

@@ -39,7 +39,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_

39 datamodel_code_generator # required for minicpm3 test

40 lm-eval[api]>=0.4.12 # required for model evaluation test

41 mteb[bm25s]>=2, <3 # required for mteb test

42 -transformers==5.5.3

42 +transformers==5.10.4

43 tokenizers==0.22.2

44 schemathesis>=4.0.0 # Required for openai schema test.

45 # quantization

requirements/test/cuda.txt

+2 -2

@@ -755,7 +755,7 @@ pathvalidate==3.2.1

755 # via pytablewriter

756 patsy==1.0.1

757 # via statsmodels

758 -peft==0.18.1

758 +peft==0.19.1

759 # via -r requirements/test/cuda.in

760 perceptron==0.1.4

761 # via -r requirements/test/cuda.in

@@ -1261,7 +1261,7 @@ tqdm==4.67.3

1261 # segmentation-models-pytorch

1262 # sentence-transformers

1263 # transformers

1264 -transformers==5.5.3

1264 +transformers==5.10.4

1265 # via

1266 # -c requirements/common.txt

1267 # -r requirements/test/../common.txt

requirements/test/nightly-torch.txt

+1 -1

@@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test

29 datamodel_code_generator # required for minicpm3 test

30 lm-eval[api]>=0.4.12 # required for model evaluation test

31 mteb[bm25s]>=2, <3 # required for mteb test

32 -transformers==5.5.3

32 +transformers==5.10.4

33 tokenizers==0.22.2

34 schemathesis>=4.0.0 # Required for openai schema test.

35 # quantization

requirements/test/rocm.in

+2 -2

@@ -19,7 +19,7 @@ httpx

19 librosa # required for audio tests

20 vector_quantize_pytorch # required for minicpmo_26 test

21 vocos # required for minicpmo_26 test

22 -peft>=0.15.0 # required for phi-4-mm test

22 +peft>=0.19.1 # required for phi-4-mm test

23 pqdm

24 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests

25 sentence-transformers>=5.2.0 # required for embedding tests

@@ -35,7 +35,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_

35 datamodel_code_generator # required for minicpm3 test

36 lm-eval[api]>=0.4.12 # required for model evaluation test

37 mteb[bm25s]>=2, <3 # required for mteb test

38 -transformers==5.5.3

38 +transformers==5.10.4

39 tokenizers==0.22.2

40 schemathesis>=4.0.0 # Required for openai schema test

41 # quantization

requirements/test/rocm.txt

+2 -2

@@ -731,7 +731,7 @@ pathvalidate==3.3.1

731 # via pytablewriter

732 patsy==1.0.2

733 # via statsmodels

734 -peft==0.18.1

734 +peft==0.19.1

735 # via -r requirements/test/rocm.in

736 perceptron==0.1.4

737 # via -r requirements/test/rocm.in

@@ -1218,7 +1218,7 @@ tqdm==4.67.3

1218 # sentence-transformers

1219 # tilelang

1220 # transformers

1221 -transformers==5.5.3

1221 +transformers==5.10.4

1222 # via

1223 # -c requirements/common.txt

1224 # -r requirements/test/../common.txt

requirements/test/xpu.in

+1

@@ -17,6 +17,7 @@ accelerate

17 arctic-inference

18 lm_eval[api]>=0.4.12

19 modelscope<1.38

20 +transformers==5.10.4

21

22 # --- Audio Processing ---

23 librosa

requirements/test/xpu.txt

+2 -1

@@ -940,10 +940,11 @@ tqdm==4.67.3

940 # pqdm

941 # sentence-transformers

942 # transformers

943 -transformers==5.5.3

943 +transformers==5.10.4

944 # via

945 # -c requirements/common.txt

946 # -r requirements/test/../common.txt

947 + # -r requirements/test/xpu.in

948 # compressed-tensors

949 # sentence-transformers

950 # xgrammar

rust/src/text/src/backend/hf/model_files.rs

+1 -1

@@ -378,7 +378,7 @@ mod tests {

378 fs::write(dir.path().join("tokenizer.json"), "{}").expect("write tokenizer");

379 fs::write(

380 dir.path().join("tokenizer_config.json"),

381 - r#"{"tokenizer_class":"PreTrainedTokenizerFast"}"#,

381 + r#"{"tokenizer_class":"TokenizersBackend"}"#,

382 )

383 .expect("write tokenizer config");

384 fs::write(dir.path().join("config.json"), "{}").expect("write config");

tests/conftest.py

+2 -2

@@ -74,7 +74,7 @@ from torch._inductor.utils import fresh_cache

74

75

76 if TYPE_CHECKING:

77 - from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast

77 + from transformers import PythonBackend, TokenizersBackend

78 from transformers.generation.utils import GenerateOutput

79

80

@@ -499,7 +499,7 @@ class HfRunner:

499 self.model = model

500

501 if not skip_tokenizer_init:

502 - self.tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast" = (

502 + self.tokenizer: "PythonBackend | TokenizersBackend" = (

503 AutoTokenizer.from_pretrained(

504 tokenizer_name or model_name,

505 trust_remote_code=trust_remote_code,

tests/entrypoints/pooling/embed/test_online.py

+2 -2

@@ -369,7 +369,7 @@ async def test_chat_request(

369 assert output.object == "list"

370 assert len(output.data) == 1

371 assert output.model == MODEL_NAME

372 - assert output.usage.prompt_tokens == 34

372 + assert output.usage.prompt_tokens == 33

373

374 # test continue_final_message

375 response = requests.post(

@@ -401,7 +401,7 @@ async def test_chat_request(

401 assert output.object == "list"

402 assert len(output.data) == 1

403 assert output.model == MODEL_NAME

404 - assert output.usage.prompt_tokens == 36

404 + assert output.usage.prompt_tokens == 35

405

406 # test continue_final_message with add_generation_prompt

407 response = requests.post(

tests/models/multimodal/generation/test_common.py

-1

@@ -906,7 +906,6 @@ VLM_TEST_SETTINGS = {

906 max_model_len=4096,

907 use_tokenizer_eos=True,

908 auto_cls=AutoModelForImageTextToText,

909 - hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),

909 ),

910 "qwen2_vl": VLMTestInfo(

911 models=["Qwen/Qwen2-VL-2B-Instruct"],

tests/models/multimodal/generation/vlm_utils/model_utils.py

-91

@@ -1492,94 +1492,3 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

1492

1493 hf_model.model.generate = types.MethodType(_generate, hf_model.model)

1494 return hf_model

1495 -

1496 -

1497 -def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:

1498 - """Return hf_model_kwargs with a patched config for QianfanOCR."""

1499 - from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig

1500 -

1501 - config = QianfanOCRConfig.from_pretrained(model_name)

1502 - vc = config.vision_config

1503 - if isinstance(vc.image_size, int):

1504 - vc.image_size = (vc.image_size, vc.image_size)

1505 - if isinstance(vc.patch_size, int):

1506 - vc.patch_size = (vc.patch_size, vc.patch_size)

1507 - return {"config": config}

1508 -

1509 -

1510 -def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

1511 - """Patches an HfRunner instance to run QianfanOCR model inference.

1512 -

1513 - QianfanOCR shares the same architecture as InternVLChatModel, so the

1514 - patching logic mirrors ``internvl_patch_hf_runner``. The only difference

1515 - is that we load the config via vllm's registered ``QianfanOCRConfig``

1516 - instead of relying on ``trust_remote_code``.

1517 - """

1518 -

1519 - class QianfanOCRProcessor:

1520 - def __init__(self, hf_runner: HfRunner):

1521 - self.tokenizer = hf_runner.tokenizer

1522 -

1523 - from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig

1524 -

1525 - self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)

1526 - self.vision_config = self.config.vision_config

1527 - self.use_thumbnail = self.config.use_thumbnail

1528 - self.min_num = self.config.min_dynamic_patch

1529 - self.max_num = self.config.max_dynamic_patch

1530 - self.image_size = self.vision_config.image_size

1531 -

1532 - # Compute num_image_token from config instead of model attribute,

1533 - # since the transformers-native model doesn't expose it.

1534 - image_size = self.config.force_image_size or self.vision_config.image_size

1535 - patch_size = self.vision_config.patch_size

1536 - downsample_ratio = self.config.downsample_ratio

1537 - self.num_image_token = int(

1538 - (image_size // patch_size) ** 2 * (downsample_ratio**2)

1539 - )

1540 -

1541 - def __call__(

1542 - self,

1543 - text: str,

1544 - images: PIL.Image.Image | list[PIL.Image.Image] = None,

1545 - **kwargs,

1546 - ):

1547 - from vllm.transformers_utils.processors.internvl import (

1548 - image_to_pixel_values_internvl,

1549 - )

1550 -

1551 - IMG_START = "<img>"

1552 - IMG_END = "</img>"

1553 - IMG_CONTEXT = "<IMG_CONTEXT>"

1554 -

1555 - images = [images] if isinstance(images, PIL.Image.Image) else images

1556 - pixel_values_list = [

1557 - image_to_pixel_values_internvl(

1558 - image,

1559 - input_size=self.image_size,

1560 - min_num=self.min_num,

1561 - max_num=self.max_num,

1562 - use_thumbnail=self.use_thumbnail,

1563 - )

1564 - for image in images

1565 - ]

1566 - num_patches_list = [pv.shape[0] for pv in pixel_values_list]

1567 - pixel_values = torch.cat(pixel_values_list, dim=0)

1568 -

1569 - for num_patches in num_patches_list:

1570 - context_tokens = IMG_CONTEXT * self.num_image_token * num_patches

1571 - image_tokens = IMG_START + context_tokens + IMG_END

1572 - text = text.replace("<image>", image_tokens, 1)

1573 -

1574 - prompt = self.tokenizer(text, return_tensors="pt")

1575 - prompt.update({"pixel_values": pixel_values})

1576 - return prompt

1577 -

1578 - img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")

1579 - hf_model.model.img_context_token_id = img_context_token_id

1580 - hf_model.processor = QianfanOCRProcessor(hf_model)

1581 - hf_model.model.get_output_embeddings = (

1582 - lambda: hf_model.model.language_model.get_output_embeddings()

1583 - )

1584 - hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)

1585 - return hf_model

tests/models/registry.py

+1 -3

@@ -202,9 +202,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {

202 "AfmoeForCausalLM": _HfExamplesInfo("arcee-ai/Trinity-Nano-Preview"),

203 "ApertusForCausalLM": _HfExamplesInfo("swiss-ai/Apertus-8B-Instruct-2509"),

204 "ArceeForCausalLM": _HfExamplesInfo("arcee-ai/AFM-4.5B-Base"),

205 - "ArcticForCausalLM": _HfExamplesInfo(

206 - "Snowflake/snowflake-arctic-instruct", trust_remote_code=True

207 - ),

205 + "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct"),

206 "AXK1ForCausalLM": _HfExamplesInfo("skt/A.X-K1", trust_remote_code=True),

207 "BailingMoeForCausalLM": _HfExamplesInfo(

208 "inclusionAI/Ling-lite-1.5", trust_remote_code=True

tests/models/utils.py

+1

@@ -486,6 +486,7 @@ def dummy_hf_overrides(

486 "Gemma3nForConditionalGeneration",

487 "Gemma4ForCausalLM",

488 "Gemma4ForConditionalGeneration",

489 + "Gemma4MTPModel",

490 "DiffusionGemmaForBlockDiffusion",

491 )

492 else 1

tests/tokenizers_/test_basic.py

+3 -3

@@ -5,7 +5,7 @@ from typing import _get_protocol_attrs # type: ignore

5 import pytest

6 from transformers import (

7 PreTrainedTokenizerBase,

8 - PreTrainedTokenizerFast,

8 + TokenizersBackend,

9 )

10

11 from vllm.tokenizers import TokenizerLike, get_tokenizer

@@ -24,7 +24,7 @@ def _assert_tokenizer_like(tokenizer: object):

24

25 def test_tokenizer_like_protocol():

26 tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True)

27 - assert isinstance(tokenizer, PreTrainedTokenizerFast)

27 + assert isinstance(tokenizer, TokenizersBackend)

28 _assert_tokenizer_like(tokenizer)

29

30 tokenizer = get_tokenizer(

@@ -38,7 +38,7 @@ def test_tokenizer_like_protocol():

38 assert isinstance(tokenizer, HfTokenizer)

39

40 # Verify it's a fast tokenizer (required for FastIncrementalDetokenizer)

41 - assert isinstance(tokenizer, PreTrainedTokenizerFast)

41 + assert isinstance(tokenizer, TokenizersBackend)

42 assert "DSV32" in tokenizer.__class__.__name__

43 _assert_tokenizer_like(tokenizer)

44

tests/tokenizers_/test_detokenize.py

+5 -5

@@ -5,7 +5,7 @@ from collections.abc import Generator

5 from typing import Any

6

7 import pytest

8 -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast

8 +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend

9

10 from vllm.sampling_params import SamplingParams

11 from vllm.tokenizers.mistral import MistralTokenizer

@@ -153,13 +153,13 @@ def test_decode_streaming(

153 spaces_between_special_tokens,

154 fast,

155 ):

156 - if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):

156 + if fast and not isinstance(tokenizer, TokenizersBackend):

157 pytest.skip()

158

159 if skip_special_tokens and not spaces_between_special_tokens:

160 pytest.skip()

161

162 - if not fast and isinstance(tokenizer, PreTrainedTokenizerFast):

162 + if not fast and isinstance(tokenizer, TokenizersBackend):

163 # Fix up inconsistency in fast/slow tokenizer behaviour.

164 tokenizer.add_special_tokens(

165 {

@@ -173,7 +173,7 @@ def test_decode_streaming(

173

174 extra_decode_args = (

175 {}

176 - if not isinstance(tokenizer, PreTrainedTokenizer)

176 + if not isinstance(tokenizer, PythonBackend)

177 else {"spaces_between_special_tokens": spaces_between_special_tokens}

178 )

179

@@ -225,7 +225,7 @@ def test_decode_streaming(

225 @pytest.mark.parametrize("tokenizer_name", TOKENIZERS)

226 @pytest.mark.parametrize("fast", (True, False))

227 def test_oov_decode(tokenizer, fast):

228 - if fast and not isinstance(tokenizer, PreTrainedTokenizerFast):

228 + if fast and not isinstance(tokenizer, TokenizersBackend):

229 pytest.skip()

230

231 decoded_text, out_ids = _run_incremental_decode(

tests/v1/engine/utils.py

+5 -5

@@ -7,14 +7,14 @@ from typing import TypeAlias

7

8 import numpy as np

9 import torch

10 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast

10 +from transformers import PythonBackend, TokenizersBackend

11

12 from vllm.engine.arg_utils import EngineArgs

13 from vllm.v1.engine import EngineCoreOutput, FinishReason

14 from vllm.v1.metrics.stats import PrefillStats

15 from vllm.v1.outputs import LogprobsLists, LogprobsTensors

16

17 -GeneralTokenizerType: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast

17 +GeneralTokenizerType: TypeAlias = PythonBackend | TokenizersBackend

18

19 # Number of sample logprobs to request when testing sample logprobs

20 NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5

@@ -193,7 +193,7 @@ def _create_random_top_token_test_matrix(

193

194 def decode_token(

195 tok_id: int,

196 - tokenizer: PreTrainedTokenizer,

196 + tokenizer: PythonBackend,

197 ) -> str:

198 """Reproduce the process of detokenizing a token for testing purposes.

199

@@ -210,7 +210,7 @@ def decode_token(

210 def generate_dummy_sample_logprobs(

211 sampled_tokens_list: list,

212 num_logprobs: int,

213 - tokenizer: PreTrainedTokenizer,

213 + tokenizer: PythonBackend,

214 ) -> list[tuple[list[int], list[float], int]]:

215 """Generate dummy sample logprobs

216

@@ -259,7 +259,7 @@ def generate_dummy_sample_logprobs(

259 def generate_dummy_prompt_logprobs_tensors(

260 prompt_tokens_list: list,

261 num_logprobs: int,

262 - tokenizer: PreTrainedTokenizer,

262 + tokenizer: PythonBackend,

263 ) -> LogprobsTensors:

264 """Generate dummy prompt logprobs tensors

265

vllm/model_executor/models/funaudiochat.py

+4 -4

@@ -20,7 +20,7 @@ from typing import Any

20 import numpy as np

21 import torch

22 import torch.nn as nn

23 -from transformers import PreTrainedTokenizerFast, WhisperFeatureExtractor

23 +from transformers import TokenizersBackend, WhisperFeatureExtractor

24 from transformers.activations import get_activation

25 from transformers.feature_extraction_utils import BatchFeature

26 from transformers.modeling_outputs import BaseModelOutput

@@ -556,15 +556,15 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):

556 return WhisperFeatureExtractor.from_pretrained(self.model_id)

557

558 @cached_property

559 - def speech_tokenizer(self) -> PreTrainedTokenizerFast:

560 - return PreTrainedTokenizerFast.from_pretrained(

559 + def speech_tokenizer(self) -> TokenizersBackend:

560 + return TokenizersBackend.from_pretrained(

561 self.model_id, subfolder="speech_tokenizer"

562 )

563

564 def get_feature_extractor(self) -> WhisperFeatureExtractor:

565 return self.feature_extractor

566

567 - def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:

567 + def get_speech_tokenizer(self) -> TokenizersBackend:

568 return self.speech_tokenizer

569

570 def get_data_parser(self):

vllm/model_executor/models/minicpmv.py

+1

@@ -552,6 +552,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):

552 vendored_processor = MiniCPMVProcessor(

553 image_processor=hf_processor.image_processor,

554 tokenizer=hf_processor.tokenizer,

555 + version=self.get_model_version(),

556 )

557 hf_processor = vendored_processor

558

vllm/model_executor/models/minicpmv4_6.py

+31

@@ -35,6 +35,8 @@ from vllm.multimodal.parse import ImageProcessorItems, ImageSize, VideoProcessor

35 from vllm.multimodal.processing.processor import (

36 PromptReplacement,

37 PromptUpdateDetails,

38 + ResolvedPromptUpdate,

39 + _seq2text,

40 )

41 from vllm.sequence import IntermediateTensors

42

@@ -408,6 +410,35 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):

410 for modality, pattern in placeholders

411 ]

412

413 + def _recompute_cached_prompt_update(

414 + self, cached_update: ResolvedPromptUpdate, new_item_idx: int

415 + ) -> ResolvedPromptUpdate:

416 + new_update = super()._recompute_cached_prompt_update(

417 + cached_update, new_item_idx

418 + )

419 + # MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`

420 + # (the base class only rewrites the image modality).

421 + if cached_update.modality == "video":

422 + tokenizer = self.info.get_tokenizer()

423 + id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")

424 + id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")

425 + video_token = getattr(tokenizer, "video_token", "<|video_pad|>")

426 +

427 + text = _seq2text(tokenizer, cached_update.content.full)

428 + prev_item_idx = cached_update.item_idx

429 +

430 + new_update = new_update.with_content(

431 + PromptUpdateDetails.select_text(

432 + text.replace(

433 + f"{id_start}{prev_item_idx}{id_end}",

434 + f"{id_start}{new_item_idx}{id_end}",

435 + 1,

436 + ),

437 + video_token,

438 + )

439 + )

440 + return new_update

441 +

442 def _get_mm_fields_config(

443 self,

444 hf_inputs,

vllm/model_executor/models/qwen3_vl.py

+38 -22

@@ -33,7 +33,7 @@ import numpy as np

33 import torch

34 import torch.nn as nn

35 import torch.nn.functional as F

36 -from transformers import BatchFeature

36 +from transformers import BatchFeature, ProcessorMixin

37 from transformers.models.qwen2_vl import Qwen2VLImageProcessor

38 from transformers.models.qwen2_vl.image_processing_qwen2_vl import (

39 smart_resize as image_smart_resize,

@@ -1191,34 +1191,31 @@ def _replace_video_token_placeholders(

1191 target: list[int],

1192 replacements: list[list[int]],

1193 ) -> list[int]:

1194 - """Replace each 3-token video placeholder with its expanded sequence.

1194 + """Replace each video placeholder with its expanded sequence.

1195

1196 Args:

1197 prompt_ids: Token IDs of the original (unexpanded) prompt.

1198 - target: 3-element list ``[vision_start_id, video_pad_id,

1199 - vision_end_id]`` to search for.

1198 + target: Token-ID sequence to search for. New-style

1199 + (transformers>=5.10) processors expand only the bare

1200 + `[video_pad_id]`; older ones expand the full

1201 + `[vision_start_id, video_pad_id, vision_end_id]` triplet.

1202 replacements: Per-video expanded token sequences, in prompt order.

1203

1204 Returns:

1203 - Token IDs with every placeholder triplet replaced.

1205 + Token IDs with every placeholder replaced.

1206 """

1207 result: list[int] = []

1208 repl_idx = 0

1209 i = 0

1210 n = len(prompt_ids)

1209 - t0, t1, t2 = target

1211 + k = len(target)

1212 num_repl = len(replacements)

1213

1214 while i < n:

1213 - if (

1214 - i + 2 < n

1215 - and prompt_ids[i] == t0

1216 - and prompt_ids[i + 1] == t1

1217 - and prompt_ids[i + 2] == t2

1218 - ):

1215 + if prompt_ids[i : i + k] == target:

1216 result.extend(replacements[repl_idx])

1217 repl_idx += 1

1221 - i += 3

1218 + i += k

1219 else:

1220 result.append(prompt_ids[i])

1221 i += 1

@@ -1230,6 +1227,15 @@ def _replace_video_token_placeholders(

1227

1228

1229 class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):

1230 + @staticmethod

1231 + def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:

1232 + """Transformers>=5.10 processors override `replace_video_token`

1233 + to expand only the bare video token, keeping the prompt's outer

1234 + `<|vision_start|>`/`<|vision_end|>` markers."""

1235 + mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)

1236 + proc_impl = getattr(type(hf_processor), "replace_video_token", None)

1237 + return proc_impl is not None and proc_impl is not mixin_impl

1238 +

1239 def _call_hf_processor(

1240 self,

1241 prompt: str,

@@ -1365,14 +1371,17 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1371 tok_kwargs=tok_kwargs,

1372 )

1373

1368 - # Replace each placeholder triplet with pre-computed video tokens.

1374 + # Replace each placeholder with pre-computed video tokens.

1375 if video_input_ids_lst:

1376 hf_config = self.info.get_hf_config()

1371 - video_target = [

1372 - hf_config.vision_start_token_id,

1373 - hf_config.video_token_id,

1374 - hf_config.vision_end_token_id,

1375 - ]

1377 + if self._expands_only_video_token(self.info.get_hf_processor()):

1378 + video_target = [hf_config.video_token_id]

1379 + else:

1380 + video_target = [

1381 + hf_config.vision_start_token_id,

1382 + hf_config.video_token_id,

1383 + hf_config.vision_end_token_id,

1384 + ]

1385 input_ids = processed_outputs.pop("input_ids")

1386 if not isinstance(input_ids, list):

1387 input_ids = input_ids.tolist()

@@ -1464,17 +1473,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])

1473 select_token_id=select_token_id,

1474 )

1475

1476 + if self._expands_only_video_token(hf_processor):

1477 + # transformers>=5.10 expands only the bare video_token

1478 + video_target = hf_processor.video_token

1479 + else:

1480 + # Old-style processors expand the full placeholder

1481 + # NOTE: We match string on purpose since searching sequence of

1482 + # token ids takes more time.

1483 + video_target = "<|vision_start|><|video_pad|><|vision_end|>"

1484 +

1485 return [

1486 PromptReplacement(

1487 modality="image",

1488 target=hf_processor.image_token,

1489 replacement=get_image_replacement_qwen3vl,

1490 ),

1473 - # NOTE: We match string on purpose since searching sequence of

1474 - # token ids takes more time.

1491 PromptReplacement(

1492 modality="video",

1477 - target="<|vision_start|><|video_pad|><|vision_end|>",

1493 + target=video_target,

1494 replacement=get_video_replacement_qwen3vl,

1495 ),

1496 ]

vllm/model_executor/models/transformers/base.py

+2 -2

@@ -276,13 +276,13 @@ class Base(

276 - Any quantization config specific mappings

277 """

278 self.hf_to_vllm_mapper = WeightsMapper()

279 - orig_to_new_renamings = self.hf_to_vllm_mapper.orig_to_new_renamings

279 + orig_to_new_renaming = self.hf_to_vllm_mapper.orig_to_new_renaming

280 orig_to_new_regex = self.hf_to_vllm_mapper.orig_to_new_regex

281

282 for mapping in get_model_conversion_mapping(self.model):

283 # Handle weights which have been renamed in Transformers

284 if isinstance(mapping, WeightRenaming):

285 - orig_to_new_renamings.append(mapping)

285 + orig_to_new_renaming.append(mapping)

286 # TODO: Handle WeightConverter to enable layer merging

287

288 # Handle unexpected weights which should be ignored

vllm/model_executor/models/utils.py

+9 -7

@@ -11,7 +11,6 @@ import regex as re

11 import torch

12 import torch.nn as nn

13 from torch.nn.modules.module import register_module_module_registration_hook

14 -from transformers import PretrainedConfig

14

15 from vllm.config import VllmConfig

16 from vllm.distributed import (

@@ -33,6 +32,9 @@ from vllm.utils.torch_utils import (

32 )

33

34 if TYPE_CHECKING:

35 + from transformers import PretrainedConfig

36 + from transformers.conversion_mapping import WeightRenaming

37 +

38 from vllm.model_executor.layers.quantization import QuantizationConfig

39

40 logger = init_logger(__name__)

@@ -46,7 +48,7 @@ class WeightsMapper:

48

49 If a key maps to a value of `None`, the corresponding weight is ignored."""

50

49 - orig_to_new_renamings: list[Any] = field(default_factory=list)

51 + orig_to_new_renaming: list["WeightRenaming"] = field(default_factory=list)

52 orig_to_new_regex: Mapping[re.Pattern, str | None] = field(default_factory=dict)

53 orig_to_new_substr: Mapping[str, str | None] = field(default_factory=dict)

54 orig_to_new_stacked: Mapping[str, tuple[str, ShardId]] = field(default_factory=dict)

@@ -56,9 +58,9 @@ class WeightsMapper:

58 def __or__(self, other: "WeightsMapper") -> "WeightsMapper":

59 """Combine two `WeightsMapper`s by merging their mappings."""

60 return WeightsMapper(

59 - orig_to_new_renamings=[

60 - *self.orig_to_new_renamings,

61 - *other.orig_to_new_renamings,

61 + orig_to_new_renaming=[

62 + *self.orig_to_new_renaming,

63 + *other.orig_to_new_renaming,

64 ],

65 orig_to_new_regex={**self.orig_to_new_regex, **other.orig_to_new_regex},

66 orig_to_new_substr={**self.orig_to_new_substr, **other.orig_to_new_substr},

@@ -92,7 +94,7 @@ class WeightsMapper:

94 "k_scale to v_scale"

95 )

96

95 - for renaming in self.orig_to_new_renamings:

97 + for renaming in self.orig_to_new_renaming:

98 key, _ = renaming.rename_source_key(key)

99

100 for pattern, new_key in self.orig_to_new_regex.items():

@@ -426,7 +428,7 @@ def init_vllm_registered_model(

428 vllm_config: VllmConfig,

429 *,

430 prefix: str = "",

429 - hf_config: PretrainedConfig | None = None,

431 + hf_config: "PretrainedConfig | None" = None,

432 architectures: list[str] | None = None,

433 ) -> nn.Module:

434 """

vllm/reasoning/abs_reasoning_parsers.py

+1 -1

@@ -41,7 +41,7 @@ class ReasoningParser:

41

42 @cached_property

43 def vocab(self) -> dict[str, int]:

44 - # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab

44 + # NOTE: Only TokenizersBackend is guaranteed to have .vocab

45 # whereas all tokenizers have .get_vocab()

46 return self.model_tokenizer.get_vocab()

47

vllm/renderers/hf.py

+5 -13

@@ -228,20 +228,12 @@ def _try_get_processor_chat_template(

228 if cache_key in _PROCESSOR_CHAT_TEMPLATES:

229 return _PROCESSOR_CHAT_TEMPLATES[cache_key]

230

231 - from transformers import (

232 - PreTrainedTokenizer,

233 - PreTrainedTokenizerFast,

234 - ProcessorMixin,

235 - )

231 + from transformers import ProcessorMixin, PythonBackend, TokenizersBackend

232

233 try:

234 processor = cached_get_processor(

235 tokenizer.name_or_path,

240 - processor_cls=(

241 - PreTrainedTokenizer,

242 - PreTrainedTokenizerFast,

243 - ProcessorMixin,

244 - ),

236 + processor_cls=(PythonBackend, TokenizersBackend, ProcessorMixin),

237 trust_remote_code=trust_remote_code,

238 )

239 if (

@@ -619,15 +611,15 @@ _cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)

611

612 @lru_cache

613 def _get_hf_base_chat_template_params() -> frozenset[str]:

622 - from transformers import PreTrainedTokenizer

614 + from transformers import PythonBackend

615

616 # Get standard parameters from HuggingFace's base tokenizer class.

625 - # This dynamically extracts parameters from PreTrainedTokenizer's

617 + # This dynamically extracts parameters from PythonBackend's

618 # apply_chat_template method, ensuring compatibility with tokenizers

619 # that use **kwargs to receive standard parameters.

620

621 # Read signature from HF's base class - the single source of truth

630 - base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)

622 + base_sig = inspect.signature(PythonBackend.apply_chat_template)

623

624 # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders

625 return frozenset(

vllm/tokenizers/deepseek_v32.py

+2 -2

@@ -3,7 +3,7 @@

3 import copy

4 from typing import Any

5

6 -from transformers import PreTrainedTokenizerFast

6 +from transformers import TokenizersBackend

7

8 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

9

@@ -85,5 +85,5 @@ def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:

85 class DeepseekV32Tokenizer(TokenizerLike):

86 @classmethod

87 def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:

88 - tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)

88 + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)

89 return get_cached_tokenizer(get_deepseek_v32_tokenizer(tokenizer))

vllm/tokenizers/deepseek_v4.py

+2 -2

@@ -3,7 +3,7 @@

3 import copy

4 from typing import Any

5

6 -from transformers import PreTrainedTokenizerFast

6 +from transformers import TokenizersBackend

7

8 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam

9

@@ -92,5 +92,5 @@ def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:

92 class DeepseekV4Tokenizer(TokenizerLike):

93 @classmethod

94 def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:

95 - tokenizer = PreTrainedTokenizerFast.from_pretrained(*args, **kwargs)

95 + tokenizer = TokenizersBackend.from_pretrained(*args, **kwargs)

96 return get_cached_tokenizer(get_deepseek_v4_tokenizer(tokenizer))

vllm/tokenizers/hf.py

+40 -5

@@ -6,13 +6,13 @@ import queue

6 from pathlib import Path

7 from typing import TypeAlias, TypeVar

8

9 -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast

9 +from transformers import AutoTokenizer, PythonBackend, TokenizersBackend

10

11 from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config

12

13 from .protocol import TokenizerLike

14

15 -HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast

15 +HfTokenizer: TypeAlias = PythonBackend | TokenizersBackend

16 _T = TypeVar("_T", bound=TokenizerLike)

17

18

@@ -24,7 +24,7 @@ class ThreadSafeHFTokenizerMixin:

24

25 def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):

26 """

27 - If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer

27 + If `tokenizer` is a `TokenizersBackend`, modify the tokenizer

28 in-place to make the public interface thread-safe by routing calls

29 through a deep-copied tokenizer pool.

30

@@ -34,14 +34,14 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):

34 methods like ``add_special_tokens`` or ``add_tokens``.

35 - Adjacent method calls could happen on different deep copies.

36 """

37 - if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(

37 + if not isinstance(tokenizer, TokenizersBackend) or isinstance(

38 tokenizer, ThreadSafeHFTokenizerMixin

39 ):

40 return tokenizer

41

42 og_tokenizer = copy.copy(tokenizer)

43

44 - tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()

44 + tokenizer_pool: queue.Queue[TokenizersBackend] = queue.Queue()

45 for _ in range(copies):

46 tokenizer_pool.put(copy.deepcopy(og_tokenizer))

47

@@ -116,6 +116,16 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:

116 tokenizer_all_special_tokens = tokenizer.all_special_tokens

117 tokenizer_vocab = tokenizer.get_vocab()

118 tokenizer_len = len(tokenizer)

119 + # The underlying tokenizer class could be MistralCommonBackend,

120 + # which does not implement is_fast in Transformers

121 + tokenizer_is_fast = getattr(tokenizer, "is_fast", True)

122 +

123 + # MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization.

124 + mistral_tekkenizer = None

125 + if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None):

126 + from vllm.tokenizers.mistral import mistral_common_tekkenizer

127 +

128 + mistral_tekkenizer = mistral_common_tekkenizer(tokenizer)

129

130 max_token_id = max(tokenizer_vocab.values())

131 max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)

@@ -145,6 +155,31 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:

155 def max_chars_per_token(self) -> int:

156 return max_chars_per_token

157

158 + @property

159 + def is_fast(self) -> bool:

160 + return tokenizer_is_fast

161 +

162 + def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False):

163 + if mistral_tekkenizer is not None:

164 + from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens

165 +

166 + return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids)

167 + return super().convert_ids_to_tokens(

168 + ids, skip_special_tokens=skip_special_tokens

169 + )

170 +

171 + def convert_tokens_to_string(self, tokens: list[str]) -> str:

172 + if mistral_tekkenizer is not None:

173 + from vllm.tokenizers.mistral import tekken_convert_tokens_to_string

174 +

175 + return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens)

176 + try:

177 + return super().convert_tokens_to_string(tokens)

178 + except NotImplementedError:

179 + # The underlying tokenizer class could be MistralCommonBackend,

180 + # which does not implement convert_tokens_to_string in Transformers

181 + return "".join(tokens)

182 +

183 def get_vocab(self) -> dict[str, int]:

184 return tokenizer_vocab

185

vllm/tokenizers/mistral.py

+35 -8

@@ -188,6 +188,39 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:

188 return tokenizer.unk_id

189

190

191 +def mistral_common_tekkenizer(tokenizer: object) -> "Tekkenizer | None":

192 + """Return the underlying `Tekkenizer` for a `MistralCommonBackend`."""

193 + mistral = getattr(tokenizer, "tokenizer", None)

194 + instruct = getattr(mistral, "instruct_tokenizer", None)

195 + tekken = getattr(instruct, "tokenizer", None)

196 + return tekken if isinstance(tekken, Tekkenizer) else None

197 +

198 +

199 +def tekken_convert_ids_to_tokens(

200 + tokenizer: "Tekkenizer", ids: Sequence[int]

201 +) -> list[str | bytes]:

202 + """Convert ids to pieces, using raw `bytes` for byte-fallback tokens."""

203 + tokens: list[str | bytes] = [tokenizer.id_to_piece(i) for i in ids]

204 + if any("�" in t for t in tokens):

205 + tokens = [

206 + tokenizer.id_to_byte_piece(i, SpecialTokenPolicy.KEEP)

207 + if i >= tokenizer.num_special_tokens

208 + else tokenizer.decode([i], SpecialTokenPolicy.KEEP)

209 + for i in ids

210 + ]

211 + return tokens

212 +

213 +

214 +def tekken_convert_tokens_to_string(

215 + tokenizer: "Tekkenizer", tokens: Sequence[str | bytes]

216 +) -> str:

217 + """Reassemble pieces from `tekken_convert_ids_to_tokens` into text."""

218 + if any(isinstance(t, bytes) for t in tokens):

219 + ids = [_tekken_token_to_id(tokenizer, t) for t in tokens]

220 + return tokenizer.decode(ids, SpecialTokenPolicy.KEEP)

221 + return "".join(cast(Sequence[str], tokens))

222 +

223 +

224 class MistralTokenizer(TokenizerLike):

225 IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral

226

@@ -453,14 +486,8 @@ class MistralTokenizer(TokenizerLike):

486 if (t in to_decode_special_tokens or t not in self._special_tokens_set)

487 ]

488

456 - if any(isinstance(t, bytes) for t in tokens):

457 - # we need to encode and decode all tokens again

458 - ids = [_tekken_token_to_id(self.tokenizer, t) for t in tokens]

459 - # We filtered unwanted special tokens before

460 - # so we can decode the rest.

461 - decoded = self.tokenizer.decode(ids, SpecialTokenPolicy.KEEP)

462 - else:

463 - decoded = "".join(tokens)

489 + # We filtered unwanted special tokens before so we can decode the rest.

490 + decoded = tekken_convert_tokens_to_string(self.tokenizer, tokens)

491 else:

492 # make sure certain special tokens like Tool calls are

493 # not decoded

vllm/tokenizers/registry.py

+1

@@ -32,6 +32,7 @@ logger = init_logger(__name__)

32 # - Add model type to MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS in transformers (better)

33 # - Fix tokenizer_class on the hub for the affected models (best)

34 _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {

35 + "internlm2",

36 "step3_vl",

37 "step3p7",

38 "unlimited-ocr",

vllm/tool_parsers/abstract_tool_parser.py

+1 -1

@@ -112,7 +112,7 @@ class ToolParser:

112

113 @cached_property

114 def vocab(self) -> dict[str, int]:

115 - # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab

115 + # NOTE: Only TokenizersBackend is guaranteed to have .vocab

116 # whereas all tokenizers have .get_vocab()

117 return self.model_tokenizer.get_vocab()

118

vllm/transformers_utils/config.py

+1

@@ -71,6 +71,7 @@ class LazyConfigDict(dict):

71

72 _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(

73 afmoe="AfmoeConfig",

74 + arctic="ArcticConfig",

75 bagel="BagelConfig",

76 umm="CheersConfig",

77 chatglm="ChatGLMConfig",

vllm/transformers_utils/configs/__init__.py

+2

@@ -16,6 +16,7 @@ import importlib

16

17 _CLASS_TO_MODULE: dict[str, str] = {

18 "AfmoeConfig": "vllm.transformers_utils.configs.afmoe",

19 + "ArcticConfig": "vllm.transformers_utils.configs.arctic",

20 "AXK1Config": "vllm.transformers_utils.configs.AXK1",

21 "BagelConfig": "vllm.transformers_utils.configs.bagel",

22 "CheersConfig": "vllm.transformers_utils.configs.cheers",

@@ -92,6 +93,7 @@ _CLASS_TO_MODULE: dict[str, str] = {

93

94 __all__ = [

95 "AfmoeConfig",

96 + "ArcticConfig",

97 "AXK1Config",

98 "BagelConfig",

99 "CheersConfig",

vllm/transformers_utils/processors/fireredlid.py

+1 -1

@@ -232,7 +232,7 @@ class FireRedLIDProcessor(ProcessorMixin):

232 """

233

234 feature_extractor_class = "FireRedLIDFeatureExtractor"

235 - tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")

235 + tokenizer_class = ("PythonBackend", "TokenizersBackend")

236

237 def __init__(self, feature_extractor, tokenizer):

238 super().__init__(feature_extractor, tokenizer)

vllm/transformers_utils/processors/glm4v.py

+2 -2

@@ -3,7 +3,7 @@

3

4 # Adapted from

5 # https://github.com/zai-org/CogAgent

6 -from transformers import PreTrainedTokenizer

6 +from transformers import PythonBackend

7 from transformers.image_processing_utils_fast import BaseImageProcessorFast

8 from transformers.image_utils import PILImageResampling

9 from transformers.processing_utils import ProcessorMixin

@@ -30,7 +30,7 @@ class GLM4VProcessor(ProcessorMixin):

30 def __init__(

31 self,

32 image_processor: GLM4VImageProcessorFast,

33 - tokenizer: PreTrainedTokenizer,

33 + tokenizer: PythonBackend,

34 ) -> None:

35 self.image_processor = image_processor

36 self.tokenizer = tokenizer

vllm/transformers_utils/processors/internvl.py

+8 -3

@@ -12,7 +12,12 @@ import numpy.typing as npt

12 import torch

13 import torchvision.transforms as T

14 from PIL import Image

15 -from transformers import BatchFeature, TensorType

15 +from transformers import (

16 + BaseVideoProcessor,

17 + BatchFeature,

18 + ImageProcessingMixin,

19 + TensorType,

20 +)

21 from transformers.processing_utils import ProcessorMixin

22

23 from vllm.multimodal.image import convert_image_mode

@@ -215,7 +220,7 @@ def video_to_pixel_values_internvl(

220 return pixel_values

221

222

218 -class InternVLImageProcessor:

223 +class InternVLImageProcessor(ImageProcessingMixin):

224 def __init__(

225 self,

226 image_size: int,

@@ -312,7 +317,7 @@ class InternVLImageProcessor:

317 return BatchFeature(image_inputs, tensor_type=return_tensors)

318

319

315 -class InternVLVideoProcessor:

320 +class InternVLVideoProcessor(BaseVideoProcessor):

321 def __init__(

322 self,

323 image_size: int,

vllm/transformers_utils/processors/isaac.py

+7 -2

@@ -7,7 +7,12 @@ import numpy as np

7 import torch

8 import torch.nn.functional as F

9 from PIL import Image

10 -from transformers import BatchFeature, ProcessorMixin, TensorType

10 +from transformers import (

11 + BatchFeature,

12 + ImageProcessingMixin,

13 + ProcessorMixin,

14 + TensorType,

15 +)

16 from transformers.processing_utils import ProcessingKwargs

17 from typing_extensions import Unpack

18

@@ -322,7 +327,7 @@ class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-

327 }

328

329

325 -class IsaacImageProcessor:

330 +class IsaacImageProcessor(ImageProcessingMixin):

331 model_input_names = ["pixel_values", "image_grid_thw"]

332

333 def __init__(

vllm/transformers_utils/processors/minicpmo.py

+2 -2

@@ -268,7 +268,7 @@ class MiniCPMOProcessor(ProcessorMixin):

268 def batch_decode(self, *args, **kwargs):

269 """

270 This method forwards all its arguments to LlamaTokenizerFast's

271 - [`~PreTrainedTokenizer.batch_decode`]. Please refer to the

271 + [`~PythonBackend.batch_decode`]. Please refer to the

272 docstring of this method for more information.

273 """

274 output_ids = args[0]

@@ -289,7 +289,7 @@ class MiniCPMOProcessor(ProcessorMixin):

289 def decode(self, *args, **kwargs):

290 """

291 This method forwards all its arguments to LlamaTokenizerFast's

292 - [`~PreTrainedTokenizer.decode`]. Please refer to the docstring

292 + [`~PythonBackend.decode`]. Please refer to the docstring

293 of this method for more information.

294 """

295 result = args[0]

vllm/transformers_utils/processors/minicpmv.py

+5 -10

@@ -56,14 +56,9 @@ class MiniCPMVProcessor(ProcessorMixin):

56 image_processor_class = "AutoImageProcessor"

57 tokenizer_class = "AutoTokenizer"

58

59 - def __init__(self, image_processor=None, tokenizer=None):

59 + def __init__(self, image_processor=None, tokenizer=None, version=None):

60 super().__init__(image_processor, tokenizer)

61 - # Newer (transformers v5.7+) MiniCPM-V image processors, e.g.

62 - # MiniCPMV4_6ImageProcessor, no longer carry a `version` attribute.

63 - # Fall back to None instead of hard-crashing: `version` is only used

64 - # to special-case the 2.5 tokenization path in `_convert`, and any

65 - # value other than 2.5 takes the default branch anyway.

66 - self.version = getattr(image_processor, "version", None)

61 + self.version = version

62

63 def __call__(

64 self,

@@ -100,7 +95,7 @@ class MiniCPMVProcessor(ProcessorMixin):

95 def batch_decode(self, *args, **kwargs):

96 """

97 This method forwards all its arguments to LlamaTokenizerFast's

103 - [`~PreTrainedTokenizer.batch_decode`]. Please refer to the

98 + [`~PythonBackend.batch_decode`]. Please refer to the

99 docstring of this method for more information.

100 """

101 output_ids = args[0]

@@ -133,7 +128,7 @@ class MiniCPMVProcessor(ProcessorMixin):

128 def decode(self, *args, **kwargs):

129 """

130 This method forwards all its arguments to LlamaTokenizerFast's

136 - [`~PreTrainedTokenizer.decode`]. Please refer to the docstring

131 + [`~PythonBackend.decode`]. Please refer to the docstring

132 of this method for more information.

133 """

134 result = args[0]

@@ -161,7 +156,7 @@ class MiniCPMVProcessor(ProcessorMixin):

156

157 def _convert(self, input_str, max_inp_length: int | None = None):

158 add_bos = getattr(self.tokenizer, "add_bos_token", False)

164 - if self.version == 2.5 or add_bos:

159 + if self.version == (2, 5) or add_bos:

160 input_ids = self.tokenizer.encode(input_str)

161 else:

162 bos_id = getattr(

vllm/transformers_utils/processors/moondream3.py

+2 -2

@@ -195,7 +195,7 @@ class Moondream3Processor(ProcessorMixin):

195 The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'

196 instead of having tokenizer files in the model repo.

197 """

198 - from transformers import AutoTokenizer, PreTrainedTokenizerFast

198 + from transformers import AutoTokenizer, TokenizersBackend

199 from transformers.utils import cached_file

200

201 tokenizer = kwargs.pop("tokenizer", None)

@@ -237,7 +237,7 @@ class Moondream3Processor(ProcessorMixin):

237 "tokenizer.json",

238 **cached_file_kwargs,

239 )

240 - return PreTrainedTokenizerFast(

240 + return TokenizersBackend(

241 tokenizer_file=tokenizer_file,

242 clean_up_tokenization_spaces=False,

243 )

vllm/transformers_utils/processors/openvla.py

+2 -2

@@ -7,7 +7,7 @@ from typing import Any

7 import numpy as np

8 import torch

9 from PIL import Image

10 -from transformers.processing_utils import ProcessorMixin

10 +from transformers import ImageProcessingMixin, ProcessorMixin

11

12 IMAGENET_MEAN = np.array([0.484375, 0.455078125, 0.40625], dtype=np.float32)

13 IMAGENET_STD = np.array([0.228515625, 0.2236328125, 0.224609375], dtype=np.float32)

@@ -65,7 +65,7 @@ def preprocess_openvla_image(image: Any, image_size: int) -> torch.Tensor:

65 return torch.from_numpy(pixel_values)

66

67

68 -class OpenVLAImageProcessor:

68 +class OpenVLAImageProcessor(ImageProcessingMixin):

69 def __init__(self, *, image_size: int) -> None:

70 self.image_size = image_size

71

vllm/transformers_utils/processors/ovis.py

+2 -2

@@ -417,14 +417,14 @@ class OvisProcessor(ProcessorMixin):

417

418 def batch_decode(self, *args, **kwargs):

419 """

420 - This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please

420 + This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please

421 refer to the docstring of this method for more information.

422 """

423 return self.tokenizer.batch_decode(*args, **kwargs)

424

425 def decode(self, *args, **kwargs):

426 """

427 - This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to

427 + This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to

428 the docstring of this method for more information.

429 """

430 return self.tokenizer.decode(*args, **kwargs)

vllm/transformers_utils/processors/pixtral.py

+2 -2

@@ -4,13 +4,13 @@ import torch

4 from mistral_common.protocol.instruct.chunk import ImageChunk

5 from mistral_common.tokens.tokenizers.multimodal import ImageEncoder

6 from PIL import Image

7 -from transformers import BatchFeature, ProcessorMixin, TensorType

7 +from transformers import BatchFeature, ImageProcessingMixin, ProcessorMixin, TensorType

8 from transformers.image_utils import ImageInput

9

10 from vllm.tokenizers.mistral import MistralTokenizer

11

12

13 -class MistralCommonImageProcessor:

13 +class MistralCommonImageProcessor(ImageProcessingMixin):

14 """

15 Provide a HF-compatible interface for

16 `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.

vllm/transformers_utils/processors/step3_vl.py

+7 -2

@@ -8,7 +8,12 @@ import torch

8 from PIL import Image

9 from torchvision import transforms

10 from torchvision.transforms.functional import InterpolationMode

11 -from transformers import BatchFeature, ProcessorMixin, TensorType

11 +from transformers import (

12 + BatchFeature,

13 + ImageProcessingMixin,

14 + ProcessorMixin,

15 + TensorType,

16 +)

17

18 from vllm.tokenizers import TokenizerLike

19

@@ -240,7 +245,7 @@ class ImagePatcher:

245 )

246

247

243 -class Step3VLImageProcessor:

248 +class Step3VLImageProcessor(ImageProcessingMixin):

249 def __init__(

250 self,

251 image_size: int = 728,

vllm/transformers_utils/processors/voxtral.py

+7 -2

@@ -6,13 +6,18 @@ from math import ceil

6 import numpy as np

7 import torch

8 from mistral_common.tokens.tokenizers.audio import AudioEncoder

9 -from transformers import BatchFeature, ProcessorMixin, TensorType

9 +from transformers import (

10 + BatchFeature,

11 + ProcessorMixin,

12 + SequenceFeatureExtractor,

13 + TensorType,

14 +)

15 from transformers.audio_utils import AudioInput

16

17 from vllm.tokenizers.mistral import MistralTokenizer

18

19

15 -class MistralCommonFeatureExtractor:

20 +class MistralCommonFeatureExtractor(SequenceFeatureExtractor):

21 """

22 Provide a HF-compatible interface for

23 `mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.

vllm/v1/engine/detokenizer.py

+3 -3

@@ -6,7 +6,7 @@ import tokenizers

6 import tokenizers.decoders

7 from packaging import version

8 from tokenizers import Tokenizer

9 -from transformers import PreTrainedTokenizerFast

9 +from transformers import TokenizersBackend

10

11 from vllm.logger import init_logger

12 from vllm.tokenizers import TokenizerLike

@@ -57,7 +57,7 @@ class IncrementalDetokenizer:

57 # No tokenizer => skipping detokenization.

58 return IncrementalDetokenizer()

59

60 - if USE_FAST_DETOKENIZER and isinstance(tokenizer, PreTrainedTokenizerFast):

60 + if USE_FAST_DETOKENIZER and isinstance(tokenizer, TokenizersBackend):

61 # Fast tokenizer => use tokenizers library DecodeStream.

62 return FastIncrementalDetokenizer(tokenizer, request)

63

@@ -165,7 +165,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):

165

166

167 class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):

168 - def __init__(self, tokenizer: PreTrainedTokenizerFast, request: EngineCoreRequest):

168 + def __init__(self, tokenizer: TokenizersBackend, request: EngineCoreRequest):

169 super().__init__(request)

170

171 sampling_params = request.sampling_params

vllm/v1/structured_output/backend_guidance.py

+5

@@ -8,6 +8,7 @@ from dataclasses import dataclass

8 from typing import TYPE_CHECKING, Any

9

10 import torch

11 +from transformers import MistralCommonBackend

12

13 from vllm.logger import init_logger

14 from vllm.sampling_params import SamplingParams

@@ -95,6 +96,10 @@ class GuidanceBackend(StructuredOutputBackend):

96

97 if is_mistral_tokenizer(self.tokenizer):

98 self.ll_tokenizer = self.tokenizer.llg_tokenizer

99 + elif isinstance(self.tokenizer, MistralCommonBackend):

100 + from mistral_common.guidance.tokenizer import from_mistral_tokenizer

101 +

102 + self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer)

103 else:

104 self.ll_tokenizer = llguidance_hf.from_tokenizer(

105 self.tokenizer, max(self.vocab_size, len(self.tokenizer))

Bump Transformers version to 5.10.4 (#41359)