A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 13:15 UTC
[Bugfix] Standardize custom HF Processor init (#37289)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 17, 2026 at 23:38 UTC
c781fbbab3c52551aa565a0f2e9052107447bdb7
7 files changed
+39
-33
vllm/model_executor/models/deepseek_ocr.py
+3
-1
@@ -196,8 +196,10 @@ class DeepseekOCRProcessingInfo(BaseProcessingInfo):
196
crop_mode=CROP_MODE,
197
strategy="v1",
198
)
199
+
200
return self.ctx.get_hf_processor(
200
- DeepseekOCRProcessor, **{**kwargs, **v1_processor_config}
201
+ DeepseekOCRProcessor,
202
+ **{**v1_processor_config, **kwargs},
203
)
204
205
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/model_executor/models/deepseek_ocr2.py
+3
-1
@@ -76,8 +76,10 @@ class DeepseekOCR2ProcessingInfo(BaseProcessingInfo):
76
crop_mode=CROP_MODE,
77
strategy="v2",
78
)
79
+
80
return self.ctx.get_hf_processor(
80
- DeepseekOCRProcessor, **{**kwargs, **v2_processor_config}
81
+ DeepseekOCRProcessor,
82
+ **{**v2_processor_config, **kwargs},
83
)
84
85
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/model_executor/models/glm4v.py
+11
-3
@@ -47,7 +47,10 @@ from vllm.multimodal.processing import (
47
)
48
from vllm.sequence import IntermediateTensors
49
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
50
-from vllm.transformers_utils.processors.glm4v import GLM4VProcessor
50
+from vllm.transformers_utils.processors.glm4v import (
51
+ GLM4VImageProcessorFast,
52
+ GLM4VProcessor,
53
+)
54
from vllm.utils.tensor_schema import TensorSchema, TensorShape
55
56
from .chatglm import ChatGLMBaseModel, ChatGLMModel, GLMTransformer
@@ -387,15 +390,20 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
390
def get_hf_config(self):
391
return self.ctx.get_hf_config(ChatGLMConfig)
392
390
- def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
393
+ def get_image_processor(self, **kwargs):
394
config = self.get_hf_config()
395
vision_config = config.vision_config
396
+
397
image_size = vision_config["image_size"]
398
+ kwargs.setdefault("size", {"width": image_size, "height": image_size})
399
400
+ return GLM4VImageProcessorFast(**kwargs)
401
+
402
+ def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
403
return self.ctx.init_processor(
404
GLM4VProcessor,
405
tokenizer=self.get_tokenizer(),
398
- **{**kwargs, "image_size": image_size},
406
+ image_processor=self.get_image_processor(**kwargs),
407
)
408
409
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/model_executor/models/qwen_vl.py
+11
-3
@@ -44,7 +44,10 @@ from vllm.multimodal.processing import (
44
PromptUpdateDetails,
45
)
46
from vllm.sequence import IntermediateTensors
47
-from vllm.transformers_utils.processors.qwen_vl import QwenVLProcessor
47
+from vllm.transformers_utils.processors.qwen_vl import (
48
+ QwenVLImageProcessorFast,
49
+ QwenVLProcessor,
50
+)
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
53
from .interfaces import (
@@ -432,15 +435,20 @@ class QwenVLModel(QWenModel):
435
436
437
class QwenVLProcessingInfo(BaseProcessingInfo):
435
- def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
438
+ def get_image_processor(self, **kwargs):
439
config = self.get_hf_config()
440
vision_config = config.visual
441
+
442
image_size = vision_config["image_size"]
443
+ kwargs.setdefault("size", {"width": image_size, "height": image_size})
444
445
+ return QwenVLImageProcessorFast(**kwargs)
446
+
447
+ def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
448
return self.ctx.init_processor(
449
QwenVLProcessor,
450
tokenizer=self.get_tokenizer(),
443
- **{**kwargs, "image_size": image_size},
451
+ image_processor=self.get_image_processor(**kwargs),
452
)
453
454
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/tokenizers/qwen_vl.py
+4
@@ -61,6 +61,10 @@ def get_qwen_vl_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
61
62
63
class QwenVLTokenizer(TokenizerLike):
64
+ image_start_tag: str
65
+ image_end_tag: str
66
+ image_pad_tag: str
67
+
68
@classmethod
69
def from_pretrained(cls, *args, **kwargs) -> HfTokenizer:
70
tokenizer = AutoTokenizer.from_pretrained(*args, **kwargs)
vllm/transformers_utils/processors/glm4v.py
+2
-7
@@ -29,13 +29,8 @@ class GLM4VProcessor(ProcessorMixin):
29
30
def __init__(
31
self,
32
+ image_processor: GLM4VImageProcessorFast,
33
tokenizer: PreTrainedTokenizer,
33
- image_size: int,
34
- image_processor: GLM4VImageProcessorFast | None = None,
34
) -> None:
36
- self.tokenizer = tokenizer
37
- if image_processor is None:
38
- image_processor = GLM4VImageProcessorFast(
39
- size={"width": image_size, "height": image_size}
40
- )
35
self.image_processor = image_processor
36
+ self.tokenizer = tokenizer
vllm/transformers_utils/processors/qwen_vl.py
+5
-18
@@ -31,25 +31,12 @@ class QwenVLProcessor(ProcessorMixin):
31
32
def __init__(
33
self,
34
+ image_processor: QwenVLImageProcessorFast,
35
tokenizer: QwenVLTokenizer,
35
- image_size: int,
36
- image_processor: QwenVLImageProcessorFast | None = None,
36
) -> None:
38
- self.tokenizer = tokenizer
39
- if image_processor is None:
40
- image_processor = QwenVLImageProcessorFast(
41
- size={"width": image_size, "height": image_size}
42
- )
37
self.image_processor = image_processor
38
+ self.tokenizer = tokenizer
39
45
- @property
46
- def image_start_tag(self) -> str:
47
- return self.tokenizer.image_start_tag # type: ignore[attr-defined]
48
-
49
- @property
50
- def image_end_tag(self) -> str:
51
- return self.tokenizer.image_end_tag # type: ignore[attr-defined]
52
-
53
- @property
54
- def image_pad_tag(self) -> str:
55
- return self.tokenizer.image_pad_tag # type: ignore[attr-defined]
40
+ self.image_start_tag = tokenizer.image_start_tag
41
+ self.image_end_tag = tokenizer.image_end_tag
42
+ self.image_pad_tag = tokenizer.image_pad_tag