1d0c0d209c · tduncan/Nvidia-Optimization-Stack-VLLM

tests/models/registry.py

+2 -5

@@ -1020,18 +1020,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {

1020 min_transformers_version="4.57",

1021 ),

1022 "Qwen3ASRForConditionalGeneration": _HfExamplesInfo(

1023 - "Qwen/Qwen3-ASR-1.7B",

1023 + "Qwen/Qwen3-ASR-0.6B",

1024 max_model_len=4096,

1025 min_transformers_version="4.57",

1026 - is_available_online=False,

1026 ),

1027 "Qwen3ASRRealtimeGeneration": _HfExamplesInfo(

1029 - "Qwen/Qwen3-ASR-1.7B",

1028 + "Qwen/Qwen3-ASR-0.6B",

1029 max_model_len=4096,

1030 min_transformers_version="4.57",

1032 - enforce_eager=True,

1031 hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},

1034 - is_available_online=False,

1032 ),

1033 "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),

1034 "SkyworkR1VChatModel": _HfExamplesInfo(

vllm/model_executor/models/deepseek_vl2.py

+1 -2

@@ -48,7 +48,6 @@ from vllm.transformers_utils.configs.deepseek_vl2 import (

48 MlpProjectorConfig,

49 VisionEncoderConfig,

50 )

51 -from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor

51 from vllm.utils.tensor_schema import TensorSchema, TensorShape

52 from vllm.utils.torch_utils import set_default_torch_dtype

53

@@ -160,7 +159,7 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):

159 return self.ctx.get_hf_config(DeepseekVLV2Config)

160

161 def get_hf_processor(self, **kwargs: object):

163 - return self.ctx.get_hf_processor(DeepseekVLV2Processor, **kwargs)

162 + return self.ctx.get_hf_processor(**kwargs)

163

164 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

165 return {"image": None}

vllm/model_executor/models/fireredasr2.py

+1 -1

@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (

41 PromptUpdateDetails,

42 )

43 from vllm.transformers_utils.processor import cached_processor_from_config

44 -from vllm.transformers_utils.processors.fireredasr2_processor import (

44 +from vllm.transformers_utils.processors.fireredasr2 import (

45 FireRedASR2FeatureExtractor,

46 )

47 from vllm.utils.tensor_schema import TensorSchema, TensorShape

vllm/model_executor/models/funasr.py

+1 -1

@@ -50,7 +50,7 @@ from vllm.multimodal.processing import (

50 PromptUpdate,

51 )

52 from vllm.transformers_utils.processor import cached_processor_from_config

53 -from vllm.transformers_utils.processors.funasr_processor import FunASRFeatureExtractor

53 +from vllm.transformers_utils.processors.funasr import FunASRFeatureExtractor

54 from vllm.utils.tensor_schema import TensorSchema, TensorShape

55

56 from .interfaces import (

vllm/transformers_utils/processor.py

+31 -1

@@ -20,7 +20,9 @@ from transformers.video_processing_utils import BaseVideoProcessor

20 from typing_extensions import TypeVar

21

22 from vllm.logger import init_logger

23 +from vllm.transformers_utils import processors

24 from vllm.transformers_utils.gguf_utils import is_gguf

25 +from vllm.transformers_utils.repo_utils import get_hf_file_to_dict

26 from vllm.transformers_utils.utils import convert_model_repo_to_path

27 from vllm.utils.func_utils import get_allowed_kwarg_only_overrides

28

@@ -139,6 +141,22 @@ def _merge_mm_kwargs(

141 return allowed_kwargs

142

143

144 +def get_processor_cls_name_from_config(

145 + processor_name: str,

146 + revision: str | None = "main",

147 +) -> str | None:

148 + config_file = [

149 + "processor_config.json",

150 + "preprocessor_config.json",

151 + "tokenizer_config.json",

152 + ]

153 + for file in config_file:

154 + config = get_hf_file_to_dict(file, processor_name, revision=revision)

155 + if config and "processor_class" in config:

156 + return config["processor_class"]

157 + return None

158 +

159 +

160 def get_processor(

161 processor_name: str,

162 *args: Any,

@@ -152,8 +170,20 @@ def get_processor(

170 revision = "main"

171 try:

172 processor_name = convert_model_repo_to_path(processor_name)

173 + registered_cls_name = get_processor_cls_name_from_config(

174 + processor_name, revision=revision

175 + )

176 + registered_processor_cls = (

177 + getattr(processors, registered_cls_name, None)

178 + if registered_cls_name

179 + else None

180 + )

181 + registered_processor_cls = cast(type[_P] | None, registered_processor_cls)

182 + # Use registered processor class when it's available

183 + # and explicit processor_cls is not set.

184 if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin:

156 - processor = AutoProcessor.from_pretrained(

185 + _processor_cls = registered_processor_cls or AutoProcessor

186 + processor = _processor_cls.from_pretrained(

187 processor_name,

188 *args,

189 revision=revision,

vllm/transformers_utils/processors/__init__.py

+28 -10

@@ -8,16 +8,20 @@ reasons:

8 - There is a need to override the existing processor to support vLLM.

9 """

10

11 -from vllm.transformers_utils.processors.bagel import BagelProcessor

12 -from vllm.transformers_utils.processors.deepseek_vl2 import DeepseekVLV2Processor

13 -from vllm.transformers_utils.processors.fireredasr2_processor import (

14 - FireRedASR2Processor,

15 -)

16 -from vllm.transformers_utils.processors.funasr_processor import FunASRProcessor

17 -from vllm.transformers_utils.processors.hunyuan_vl import HunYuanVLProcessor

18 -from vllm.transformers_utils.processors.hunyuan_vl_image import HunYuanVLImageProcessor

19 -from vllm.transformers_utils.processors.ovis import OvisProcessor

20 -from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor

11 +import importlib

12 +

13 +_CLASS_TO_MODULE: dict[str, str] = {

14 + "BagelProcessor": "vllm.transformers_utils.processors.bagel",

15 + "DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",

16 + "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",

17 + "FunASRProcessor": "vllm.transformers_utils.processors.funasr",

18 + "HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",

19 + "HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",

20 + "OvisProcessor": "vllm.transformers_utils.processors.ovis",

21 + "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",

22 + "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",

23 +}

24 +

25

26 __all__ = [

27 "BagelProcessor",

@@ -28,4 +32,18 @@ __all__ = [

32 "HunYuanVLImageProcessor",

33 "OvisProcessor",

34 "Ovis2_5Processor",

35 + "Qwen3ASRProcessor",

36 ]

37 +

38 +

39 +def __getattr__(name: str):

40 + if name in _CLASS_TO_MODULE:

41 + module_name = _CLASS_TO_MODULE[name]

42 + module = importlib.import_module(module_name)

43 + return getattr(module, name)

44 +

45 + raise AttributeError(f"module 'processors' has no attribute '{name}'")

46 +

47 +

48 +def __dir__():

49 + return sorted(list(__all__))

vllm/transformers_utils/processors/bagel.py

-4

@@ -3,7 +3,6 @@

4 """BAGEL processor for image and text inputs."""

5

6 -from transformers import AutoProcessor

6 from transformers.feature_extraction_utils import BatchFeature

7 from transformers.image_utils import ImageInput

8 from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack

@@ -79,6 +78,3 @@ class BagelProcessor(ProcessorMixin):

78 tokenizer_input_names = self.tokenizer.model_input_names

79 image_processor_input_names = self.image_processor.model_input_names

80 return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))

82 -

83 -

84 -AutoProcessor.register("BagelProcessor", BagelProcessor)

vllm/transformers_utils/processors/deepseek_ocr.py

+1 -4

@@ -8,7 +8,7 @@ from typing import Literal

8 import torch

9 import torchvision.transforms as T

10 from PIL import Image, ImageOps

11 -from transformers import AutoProcessor, BatchFeature, LlamaTokenizerFast

11 +from transformers import BatchFeature, LlamaTokenizerFast

12 from transformers.processing_utils import ProcessorMixin

13

14 # TODO(Isotr0py): change modes for variants

@@ -453,6 +453,3 @@ class DeepseekOCRProcessor(ProcessorMixin):

453 num_image_tokens,

454 image_shapes,

455 )

456 -

457 -

458 -AutoProcessor.register("DeepseekOCRProcessor", DeepseekOCRProcessor)

vllm/transformers_utils/processors/deepseek_vl2.py

+1 -4

@@ -29,7 +29,7 @@ from typing import Any

29 import torch

30 import torchvision.transforms as T

31 from PIL import Image, ImageOps

32 -from transformers import AutoProcessor, BatchFeature, LlamaTokenizerFast

32 +from transformers import BatchFeature, LlamaTokenizerFast

33 from transformers.processing_utils import ProcessorMixin

34

35

@@ -401,6 +401,3 @@ class DeepseekVLV2Processor(ProcessorMixin):

401 images_spatial_crop,

402 num_image_tokens,

403 )

404 -

405 -

406 -AutoProcessor.register("DeepseekVLV2Processor", DeepseekVLV2Processor)

vllm/transformers_utils/processors/fireredasr2.py renamed

-2

@@ -8,7 +8,6 @@ import torch

8 import torch.nn.functional as F

9 from transformers import (

10 AutoFeatureExtractor,

11 - AutoProcessor,

11 BatchFeature,

12 )

13 from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor

@@ -345,4 +344,3 @@ class FireRedASR2Processor(ProcessorMixin):

344 AutoFeatureExtractor.register(

345 "FireRedASR2FeatureExtractor", FireRedASR2FeatureExtractor

346 )

348 -AutoProcessor.register("FireRedASR2Processor", FireRedASR2Processor)

vllm/transformers_utils/processors/funasr.py renamed

-2

@@ -9,7 +9,6 @@ import torchaudio.compliance.kaldi as kaldi

9 from torch.nn.utils.rnn import pad_sequence

10 from transformers import (

11 AutoFeatureExtractor,

12 - AutoProcessor,

12 BatchFeature,

13 )

14 from transformers.feature_extraction_sequence_utils import SequenceFeatureExtractor

@@ -503,4 +502,3 @@ class FunASRProcessor(ProcessorMixin):

502

503

504 AutoFeatureExtractor.register("FunASRFeatureExtractor", FunASRFeatureExtractor)

506 -AutoProcessor.register("FunASRProcessor", FunASRProcessor)

vllm/transformers_utils/processors/hunyuan_vl.py

-4

@@ -5,7 +5,6 @@

5

6 import numpy as np

7 import torch

8 -from transformers import AutoProcessor

8 from transformers.feature_extraction_utils import BatchFeature

9 from transformers.image_utils import ImageInput

10 from transformers.processing_utils import ProcessorMixin

@@ -225,6 +224,3 @@ def split_image_into_patch_blocks(

224 patches = img.reshape(-1, 3, patch_size, patch_size)

225

226 return patches

228 -

229 -

230 -AutoProcessor.register("HunYuanVLProcessor", HunYuanVLProcessor)

vllm/transformers_utils/processors/ovis.py

+1 -4

@@ -26,7 +26,7 @@ from functools import cached_property

26

27 import PIL

28 import torch

29 -from transformers import AutoProcessor, BatchFeature

29 +from transformers import BatchFeature

30 from transformers.image_utils import ImageInput

31 from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack

32 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

@@ -453,6 +453,3 @@ class OvisProcessor(ProcessorMixin):

453 dict.fromkeys(tokenizer_input_names + image_processor_input_names)

454 )

455 return names_from_processor + ["second_per_grid_ts"]

456 -

457 -

458 -AutoProcessor.register("OvisProcessor", OvisProcessor)

vllm/transformers_utils/processors/ovis2_5.py

+1 -4

@@ -6,7 +6,7 @@ from functools import cached_property

6 import numpy as np

7 import PIL

8 import torch

9 -from transformers import AutoProcessor, BatchFeature

9 +from transformers import BatchFeature

10 from transformers.image_utils import ImageInput

11 from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack

12 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

@@ -476,6 +476,3 @@ class Ovis2_5Processor(ProcessorMixin):

476 visual_placeholders,

477 torch.tensor([[grid_t, grid_h, grid_w]]),

478 )

479 -

480 -

481 -AutoProcessor.register("Ovis2_5Processor", Ovis2_5Processor)

vllm/transformers_utils/processors/qwen3_asr.py

-3

@@ -227,6 +227,3 @@ class Qwen3ASRProcessor(ProcessorMixin):

227 + ["feature_attention_mask"]

228 )

229 )

230 -

231 -

232 -AutoProcessor.register("Qwen3ASRProcessor", Qwen3ASRProcessor)

[Misc] Lazy import registered processors (#36024)