A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix InternS1 video processing after Transformers v4.56 (#25644)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Sep 25, 2025 at 22:46 UTC
03858e6d1c852798ac7e2aa4e02f17c248e2bbed
4 files changed
+68
-3
.buildkite/test-pipeline.yaml
+2
-1
@@ -770,8 +770,9 @@ steps:
770
- pytest -v -s tests/models/multimodal/processing/
771
- pytest -v -s tests/models/multimodal/test_mapping.py
772
- python3 examples/offline_inference/basic/chat.py
773
- - python3 examples/offline_inference/audio_language.py --model-type whisper
773
- python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
774
+ # Whisper needs spawn method to avoid deadlock
775
+ - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
776
777
- label: Blackwell Test # 38 min
778
timeout_in_minutes: 60
tests/models/multimodal/processing/test_common.py
+1
@@ -213,6 +213,7 @@ _IGNORE_MM_KEYS = {
213
MM_DATA_PATCHES = {
214
# GLM4.1V and Qwen3-VL requires video metadata to be included in the input
215
"glm4v": glm4_1v_patch_mm_data,
216
+ "glm4v_moe": glm4_1v_patch_mm_data,
217
"qwen3_vl": qwen3_vl_patch_mm_data,
218
"qwen3_vl_moe": qwen3_vl_patch_mm_data,
219
}
vllm/model_executor/models/interns1.py
+10
-1
@@ -16,6 +16,8 @@ from transformers import BatchFeature, InternVLProcessor, PretrainedConfig
16
from transformers.activations import ACT2FN
17
from transformers.models.got_ocr2.image_processing_got_ocr2_fast import (
18
GotOcr2ImageProcessorFast)
19
+from transformers.models.internvl.video_processing_internvl import (
20
+ InternVLVideoProcessor)
21
22
from vllm.config import VllmConfig
23
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -31,6 +33,8 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
33
PromptUpdate, PromptUpdateDetails)
34
from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
+from vllm.transformers_utils.processor import (
37
+ cached_video_processor_from_config)
38
from vllm.utils.tensor_schema import TensorSchema, TensorShape
39
40
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
@@ -152,7 +156,12 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
156
"""ProcessingInfo for InternS1-style models."""
157
158
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
155
- return self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
159
+ hf_processor = self.ctx.get_hf_processor(InternVLProcessor, **kwargs)
160
+ hf_processor.video_processor = cached_video_processor_from_config(
161
+ self.ctx.model_config,
162
+ processor_cls=InternVLVideoProcessor,
163
+ **kwargs)
164
+ return hf_processor
165
166
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
167
return {"image": None, "video": None}
vllm/transformers_utils/processor.py
+55
-1
@@ -5,10 +5,11 @@ from functools import lru_cache
5
from typing import TYPE_CHECKING, Any, Optional, Union, cast
6
7
from transformers import (AutoFeatureExtractor, AutoImageProcessor,
8
- AutoProcessor)
8
+ AutoProcessor, AutoVideoProcessor)
9
from transformers.feature_extraction_utils import FeatureExtractionMixin
10
from transformers.image_processing_utils import BaseImageProcessor
11
from transformers.processing_utils import ProcessorMixin
12
+from transformers.video_processing_utils import BaseVideoProcessor
13
from typing_extensions import TypeVar
14
15
from vllm.utils import get_allowed_kwarg_only_overrides
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
18
from vllm.config import ModelConfig
19
20
_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
21
+_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor)
22
23
24
class HashableDict(dict):
@@ -243,3 +245,55 @@ def cached_image_processor_from_config(
245
trust_remote_code=model_config.trust_remote_code,
246
**_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
247
)
248
+
249
+
250
+def get_video_processor(
251
+ processor_name: str,
252
+ *args: Any,
253
+ revision: Optional[str] = None,
254
+ trust_remote_code: bool = False,
255
+ processor_cls_overrides: Optional[type[_V]] = None,
256
+ **kwargs: Any,
257
+):
258
+ """Load a video processor for the given model name via HuggingFace."""
259
+ try:
260
+ processor_cls = processor_cls_overrides or AutoVideoProcessor
261
+ processor = processor_cls.from_pretrained(
262
+ processor_name,
263
+ *args,
264
+ revision=revision,
265
+ trust_remote_code=trust_remote_code,
266
+ **kwargs)
267
+ except ValueError as e:
268
+ # If the error pertains to the processor class not existing or not
269
+ # currently being imported, suggest using the --trust-remote-code flag.
270
+ # Unlike AutoTokenizer, AutoVideoProcessor does not separate such errors
271
+ if not trust_remote_code:
272
+ err_msg = (
273
+ "Failed to load the video processor. If the video processor is "
274
+ "a custom processor not yet available in the HuggingFace "
275
+ "transformers library, consider setting "
276
+ "`trust_remote_code=True` in LLM or using the "
277
+ "`--trust-remote-code` flag in the CLI.")
278
+ raise RuntimeError(err_msg) from e
279
+ else:
280
+ raise e
281
+
282
+ return cast(BaseVideoProcessor, processor)
283
+
284
+
285
+cached_get_video_processor = lru_cache(get_video_processor)
286
+
287
+
288
+def cached_video_processor_from_config(
289
+ model_config: "ModelConfig",
290
+ processor_cls: Optional[type[_V]] = None,
291
+ **kwargs: Any,
292
+):
293
+ return cached_get_video_processor(
294
+ model_config.model,
295
+ revision=model_config.revision,
296
+ trust_remote_code=model_config.trust_remote_code,
297
+ processor_cls_overrides=processor_cls, # type: ignore[arg-type]
298
+ **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
299
+ )