A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 23:30 UTC
[Model] Use helper function to run MM processors with token inputs (where applicable) (#38018)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 26, 2026 at 16:44 UTC
502c41a8f65e50f394882c6f957ef22b0bdef5a4
12 files changed
+215
-145
vllm/model_executor/models/keye.py
+11
@@ -1210,6 +1210,17 @@ class KeyeDummyInputsBuilder(KeyeBaseDummyInputsBuilder[KeyeProcessingInfo]):
1210
1211
1212
class KeyeMultiModalProcessor(BaseMultiModalProcessor[KeyeProcessingInfo]):
1213
+ def _call_hf_processor(
1214
+ self,
1215
+ prompt: str,
1216
+ mm_data: Mapping[str, object],
1217
+ mm_kwargs: Mapping[str, object],
1218
+ tok_kwargs: Mapping[str, object],
1219
+ ) -> BatchFeature:
1220
+ # Override to use the text path instead of token path to use the
1221
+ # video-specific logic in processing_keye.py
1222
+ return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
1223
+
1224
def _get_prompt_updates(
1225
self,
1226
mm_items: MultiModalDataItems,
vllm/model_executor/models/keye_vl1_5.py
+11
@@ -371,6 +371,17 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
371
372
373
class KeyeVL1_5MultiModalProcessor(BaseMultiModalProcessor[KeyeVL1_5ProcessingInfo]):
374
+ def _call_hf_processor(
375
+ self,
376
+ prompt: str,
377
+ mm_data: Mapping[str, object],
378
+ mm_kwargs: Mapping[str, object],
379
+ tok_kwargs: Mapping[str, object],
380
+ ) -> BatchFeature:
381
+ # Override to use the text path instead of token path to use the
382
+ # video-specific logic in processing_keye.py
383
+ return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
384
+
385
def _get_prompt_updates(
386
self,
387
mm_items: MultiModalDataItems,
vllm/model_executor/models/kimi_k25.py
+11
@@ -215,6 +215,17 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
215
grid_thws=MultiModalFieldConfig.batched("vision_chunk"),
216
)
217
218
+ def _call_hf_processor(
219
+ self,
220
+ prompt: str,
221
+ mm_data: Mapping[str, object],
222
+ mm_kwargs: Mapping[str, object],
223
+ tok_kwargs: Mapping[str, object],
224
+ ) -> BatchFeature:
225
+ # Override to use the text path instead of token path because vision chunk
226
+ # is not considered
227
+ return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
228
+
229
def _get_prompt_updates(
230
self,
231
mm_items: MultiModalDataItems,
vllm/model_executor/models/pixtral.py
+23
-1
@@ -12,7 +12,7 @@ import torch.nn.functional as F
12
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
13
from mistral_common.protocol.instruct.messages import UserMessage
14
from mistral_common.protocol.instruct.request import ChatCompletionRequest
15
-from transformers import PixtralVisionConfig
15
+from transformers import BatchFeature, PixtralVisionConfig
16
from transformers.models.pixtral.image_processing_pixtral import (
17
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
18
)
@@ -62,6 +62,7 @@ from vllm.sequence import IntermediateTensors
62
from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.tokenizers.mistral import MistralTokenizer
64
from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
65
+from vllm.utils.collection_utils import is_list_of
66
from vllm.utils.tensor_schema import TensorSchema, TensorShape
67
68
from .interfaces import (
@@ -213,6 +214,27 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
214
) -> Mapping[str, MultiModalFieldConfig]:
215
return dict(images=MultiModalFieldConfig.batched("image"))
216
217
+ def _call_hf_processor(
218
+ self,
219
+ prompt: str,
220
+ mm_data: Mapping[str, object],
221
+ mm_kwargs: Mapping[str, object],
222
+ tok_kwargs: Mapping[str, object],
223
+ ) -> BatchFeature:
224
+ outputs = super()._call_hf_processor(
225
+ prompt=prompt,
226
+ mm_data=mm_data,
227
+ mm_kwargs=mm_kwargs,
228
+ # Avoid padding issue
229
+ tok_kwargs={**tok_kwargs, "return_tensors": None},
230
+ )
231
+
232
+ # Missing batch dimension
233
+ if is_list_of(outputs["input_ids"], int):
234
+ outputs["input_ids"] = [outputs["input_ids"]]
235
+
236
+ return outputs
237
+
238
def _get_prompt_updates(
239
self,
240
mm_items: MultiModalDataItems,
vllm/model_executor/models/qwen2_5_vl.py
+11
@@ -929,6 +929,17 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
929
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
930
)
931
932
+ def _call_hf_processor(
933
+ self,
934
+ prompt: str,
935
+ mm_data: Mapping[str, object],
936
+ mm_kwargs: Mapping[str, object],
937
+ tok_kwargs: Mapping[str, object],
938
+ ) -> BatchFeature:
939
+ # Override to use the text path instead of token path to use the
940
+ # video-specific logic in processing_qwen2_5_vl.py
941
+ return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
942
+
943
def _get_prompt_updates(
944
self,
945
mm_items: MultiModalDataItems,
vllm/model_executor/models/voxtral.py
+11
-3
@@ -56,6 +56,7 @@ from vllm.sequence import IntermediateTensors
56
from vllm.tokenizers import cached_tokenizer_from_config
57
from vllm.tokenizers.mistral import MistralTokenizer
58
from vllm.transformers_utils.processors.voxtral import MistralCommonVoxtralProcessor
59
+from vllm.utils.collection_utils import is_list_of
60
61
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
62
from .utils import init_vllm_registered_model, maybe_prefix
@@ -208,7 +209,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
209
) -> None:
210
# mistral_common's tokenizer's does not follow HF's placeholder norms
211
# skip validation here
211
- ...
212
+ pass
213
214
def _call_hf_processor(
215
self,
@@ -224,13 +225,20 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
225
# MistralCommonVoxtralProcessor accepts "audio"
226
mm_data["audio"] = audios
227
227
- return super()._call_hf_processor(
228
+ outputs = super()._call_hf_processor(
229
prompt=prompt,
230
mm_data=mm_data,
231
mm_kwargs=mm_kwargs,
231
- tok_kwargs=tok_kwargs,
232
+ # Avoid padding issue
233
+ tok_kwargs={**tok_kwargs, "return_tensors": None},
234
)
235
236
+ # Missing batch dimension
237
+ if is_list_of(outputs["input_ids"], int):
238
+ outputs["input_ids"] = [outputs["input_ids"]]
239
+
240
+ return outputs
241
+
242
def _get_prompt_updates(
243
self,
244
mm_items: MultiModalDataItems,
vllm/multimodal/processing/context.py
+3
-2
@@ -196,7 +196,7 @@ class InputProcessingContext:
196
197
tokenizer = self.tokenizer
198
if is_mistral_tokenizer(tokenizer):
199
- tokenizer = tokenizer.transformers_tokenizer
199
+ tokenizer = tokenizer.transformers_tokenizer # type: ignore[union-attr]
200
201
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
202
merged_kwargs.pop("tokenizer", None)
@@ -263,9 +263,10 @@ class InputProcessingContext:
263
requires_kw_only=False,
264
allow_var_kwargs=True,
265
)
266
+ allowed_kwargs.setdefault("return_tensors", "pt")
267
268
try:
268
- output = hf_processor(**data, **allowed_kwargs, return_tensors="pt")
269
+ output = hf_processor(**data, **allowed_kwargs)
270
except Exception as exc:
271
# See https://github.com/huggingface/tokenizers/issues/537
272
if (
vllm/multimodal/processing/processor.py
+41
-10
@@ -5,8 +5,15 @@ from collections import defaultdict
5
from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field, replace
7
from enum import Enum
8
-from functools import lru_cache
9
-from typing import TYPE_CHECKING, Generic, NamedTuple, Protocol, TypeAlias, cast
8
+from functools import lru_cache, partial
9
+from typing import (
10
+ TYPE_CHECKING,
11
+ Generic,
12
+ NamedTuple,
13
+ Protocol,
14
+ TypeAlias,
15
+ cast,
16
+)
17
18
import regex as re
19
import torch
@@ -21,6 +28,7 @@ from vllm.inputs import (
28
)
29
from vllm.logger import init_logger
30
from vllm.tokenizers import TokenizerLike
31
+from vllm.transformers_utils.processor import call_hf_processor_mm_only
32
from vllm.utils.collection_utils import flatten_2d_lists, full_groupby
33
34
from ..inputs import (
@@ -1150,7 +1158,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1158
)
1159
processed_data.update(passthrough_data)
1160
1153
- (prompt_ids,) = processed_data.pop("input_ids").tolist()
1161
+ input_ids = processed_data.pop("input_ids")
1162
+ if not isinstance(input_ids, list):
1163
+ input_ids = input_ids.tolist()
1164
+
1165
+ (prompt_ids,) = input_ids
1166
1167
is_update_applied = self._hf_processor_applies_updates(
1168
prompt_text=prompt_text,
@@ -1213,16 +1225,35 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1225
[`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]
1226
to go along with the multi-modal data.
1227
"""
1216
- mm_counts = mm_items.get_all_counts()
1228
+ # Custom logic based on text inputs
1229
+ if type(self)._call_hf_processor != BaseMultiModalProcessor._call_hf_processor:
1230
+ mm_counts = mm_items.get_all_counts()
1231
+
1232
+ _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1233
+ prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1234
+ mm_items=mm_items,
1235
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1236
+ tokenization_kwargs=tokenization_kwargs,
1237
+ )
1238
1218
- _, mm_processed_data, _ = self._apply_hf_processor_text_mm(
1219
- prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
1220
- mm_items=mm_items,
1221
- hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1222
- tokenization_kwargs=tokenization_kwargs,
1239
+ return mm_processed_data
1240
+
1241
+ valid_mm_items = mm_items.select(
1242
+ {k for k, c in mm_items.get_all_counts().items() if c > 0}
1243
)
1244
+ processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
1245
+
1246
+ processed_data = self.info.ctx.call_hf_processor(
1247
+ partial(
1248
+ call_hf_processor_mm_only,
1249
+ self.info.get_hf_processor(**hf_processor_mm_kwargs),
1250
+ ),
1251
+ processor_data,
1252
+ dict(**hf_processor_mm_kwargs, **tokenization_kwargs),
1253
+ )
1254
+ processed_data.update(passthrough_data)
1255
1225
- return mm_processed_data
1256
+ return processed_data
1257
1258
def _apply_hf_processor_main(
1259
self,
vllm/transformers_utils/processor.py
+44
-1
@@ -11,12 +11,16 @@ from transformers import (
11
AutoImageProcessor,
12
AutoProcessor,
13
AutoVideoProcessor,
14
+ BatchFeature,
15
processing_utils,
16
)
17
+from transformers.audio_utils import AudioInput
18
from transformers.feature_extraction_utils import FeatureExtractionMixin
19
from transformers.image_processing_utils import BaseImageProcessor
20
+from transformers.image_utils import ImageInput
21
from transformers.processing_utils import ProcessorMixin
22
from transformers.video_processing_utils import BaseVideoProcessor
23
+from transformers.video_utils import VideoInput
24
from typing_extensions import TypeVar
25
26
from vllm.logger import init_logger
@@ -272,7 +276,6 @@ def get_processor_kwargs_keys(
276
"images_kwargs",
277
"videos_kwargs",
278
"audio_kwargs",
275
- "common_kwargs",
279
}
280
281
try:
@@ -523,3 +526,43 @@ def cached_video_processor_from_config(
526
processor_cls_overrides=processor_cls, # type: ignore[arg-type]
527
**_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs),
528
)
529
+
530
+
531
+def call_hf_processor_mm_only(
532
+ processor: ProcessorMixin,
533
+ images: ImageInput | None = None,
534
+ videos: VideoInput | None = None,
535
+ audio: AudioInput | None = None,
536
+ **kwargs,
537
+) -> BatchFeature:
538
+ output_kwargs = processor._merge_kwargs(
539
+ get_processor_kwargs_type(processor),
540
+ **kwargs,
541
+ )
542
+
543
+ if audio is not None and (
544
+ feature_extractor := getattr(processor, "feature_extractor", None)
545
+ ):
546
+ audio_inputs = feature_extractor(audio, **output_kwargs["audio_kwargs"])
547
+ audio_inputs["feature_attention_mask"] = audio_inputs.pop("attention_mask")
548
+ else:
549
+ audio_inputs = {}
550
+
551
+ if images is not None and (
552
+ image_processor := getattr(processor, "image_processor", None)
553
+ ):
554
+ images_inputs = image_processor(images=images, **output_kwargs["images_kwargs"])
555
+ else:
556
+ images_inputs = {}
557
+
558
+ if videos is not None and (
559
+ video_processor := getattr(processor, "video_processor", None)
560
+ ):
561
+ videos_inputs = video_processor(videos=videos, **output_kwargs["videos_kwargs"])
562
+ else:
563
+ videos_inputs = {}
564
+
565
+ return BatchFeature(
566
+ data={**audio_inputs, **images_inputs, **videos_inputs},
567
+ tensor_type=kwargs.get("return_tensors"),
568
+ )
vllm/transformers_utils/processors/isaac.py
+32
-16
@@ -1,16 +1,15 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from __future__ import annotations
4
-
3
import math
6
-from typing import Any
4
+from typing import Any, TypedDict
5
6
import numpy as np
7
import torch
8
import torch.nn.functional as F
9
from PIL import Image
10
from transformers import BatchFeature, ProcessorMixin, TensorType
13
-from typing_extensions import TypedDict, Unpack
11
+from transformers.processing_utils import ProcessingKwargs
12
+from typing_extensions import Unpack
13
14
from vllm.tokenizers.hf import HfTokenizer
15
@@ -308,15 +307,22 @@ def process_vision_for_patches(
307
return patches, dims_virtual
308
309
311
-class IsaacImageProcessorKwargs(TypedDict, total=False):
310
+class IsaacImagesKwargs(TypedDict, total=False):
311
patch_size: int
312
max_num_patches: int
313
min_num_patches: int
314
pixel_shuffle_scale: int
315
316
317
+class IsaacProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg]
318
+ images_kwargs: IsaacImagesKwargs
319
+ _defaults = {
320
+ "text_kwargs": {"padding": False},
321
+ "images_kwargs": {},
322
+ }
323
+
324
+
325
class IsaacImageProcessor:
319
- valid_kwargs = IsaacImageProcessorKwargs
326
model_input_names = ["pixel_values", "image_grid_thw"]
327
328
def __init__(
@@ -335,7 +341,7 @@ class IsaacImageProcessor:
341
self,
342
images: Image.Image | list[Image.Image],
343
return_tensors: str | TensorType | None = None,
338
- **kwargs: Unpack[IsaacImageProcessorKwargs],
344
+ **kwargs: Unpack[IsaacImagesKwargs],
345
) -> BatchFeature:
346
"""Preprocess images into format compatible with vLLM input processing."""
347
if not isinstance(images, list):
@@ -349,10 +355,16 @@ class IsaacImageProcessor:
355
356
patches, dims_virtual = process_vision_for_patches(
357
image_tensor,
352
- patch_size=self.patch_size,
353
- max_num_patches=self.vision_max_num_patches,
354
- min_num_patches=self.vision_min_num_patches,
355
- pixel_shuffle_scale=self.pixel_shuffle_scale,
358
+ patch_size=kwargs.get("patch_size", self.patch_size),
359
+ max_num_patches=kwargs.get(
360
+ "max_num_patches", self.vision_max_num_patches
361
+ ),
362
+ min_num_patches=kwargs.get(
363
+ "min_num_patches", self.vision_min_num_patches
364
+ ),
365
+ pixel_shuffle_scale=kwargs.get(
366
+ "pixel_shuffle_scale", self.pixel_shuffle_scale
367
+ ),
368
)
369
370
# Isaac packs a dummy temporal dim for images
@@ -405,13 +417,17 @@ class IsaacProcessor(ProcessorMixin):
417
text: str | list[str] | None = None,
418
images: Image.Image | list[Image.Image] | None = None,
419
return_tensors: str | TensorType | None = None,
408
- **kwargs,
420
+ **kwargs: Unpack[IsaacProcessorKwargs], # type: ignore[misc]
421
) -> BatchFeature:
422
+ output_kwargs = self._merge_kwargs(
423
+ IsaacProcessorKwargs,
424
+ tokenizer_init_kwargs=self.tokenizer.init_kwargs,
425
+ **kwargs,
426
+ )
427
+
428
if images is not None:
429
image_inputs = self.image_processor(
412
- images,
413
- return_tensors=return_tensors,
414
- **kwargs,
430
+ images, **output_kwargs["images_kwargs"]
431
)
432
image_grid_thw = image_inputs["image_grid_thw"]
433
else:
@@ -435,7 +451,7 @@ class IsaacProcessor(ProcessorMixin):
451
index += 1
452
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
453
438
- text_inputs = self.tokenizer(text, return_tensors=return_tensors)
454
+ text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
455
else:
456
text_inputs = {}
457
vllm/transformers_utils/processors/pixtral.py
+9
-58
@@ -5,10 +5,7 @@ from mistral_common.protocol.instruct.chunk import ImageChunk
5
from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
6
from PIL import Image
7
from transformers import BatchFeature, ProcessorMixin, TensorType
8
-from transformers.audio_utils import AudioInput
8
from transformers.image_utils import ImageInput
10
-from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
11
-from transformers.video_utils import VideoInput
9
10
from vllm.tokenizers.mistral import MistralTokenizer
11
@@ -55,62 +52,16 @@ class MistralCommonPixtralProcessor(ProcessorMixin):
52
53
def __init__(self, tokenizer: MistralTokenizer) -> None:
54
self.tokenizer = tokenizer.transformers_tokenizer
55
+
56
+ # Back-compatibility for Transformers v4
57
+ if not hasattr(self.tokenizer, "init_kwargs"):
58
+ self.tokenizer.init_kwargs = {}
59
+
60
self.image_processor = MistralCommonImageProcessor(
61
tokenizer.instruct.mm_encoder
62
)
63
62
- self._image_special_ids = self.image_processor.mm_encoder.special_ids
63
-
64
- @property
65
- def image_break_id(self) -> int:
66
- return self._image_special_ids.img_break
67
-
68
- @property
69
- def image_token_id(self) -> int:
70
- return self._image_special_ids.img
71
-
72
- @property
73
- def image_end_id(self) -> int:
74
- return self._image_special_ids.img_end
75
-
76
- def __call__(
77
- self,
78
- images: ImageInput | None = None,
79
- text: TextInput
80
- | PreTokenizedInput
81
- | list[TextInput]
82
- | list[PreTokenizedInput]
83
- | None = None,
84
- videos: VideoInput | None = None,
85
- audio: AudioInput | None = None,
86
- **kwargs,
87
- ):
88
- if images is None and text is None and videos is None and audio is None:
89
- raise ValueError(
90
- f"You need to provide at least one input to "
91
- f"call {self.__class__.__name__}"
92
- )
93
-
94
- kwargs = self._merge_kwargs(
95
- self.valid_processor_kwargs,
96
- tokenizer_init_kwargs={},
97
- **kwargs,
98
- )
99
- kwargs["text_kwargs"]["return_tensors"] = "pt"
100
- kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
101
-
102
- attribute_to_kwargs = {
103
- "tokenizer": (text, "text_kwargs"),
104
- "image_processor": (images, "images_kwargs"),
105
- "video_processor": (videos, "videos_kwargs"),
106
- "feature_extractor": (audio, "audio_kwargs"),
107
- }
108
- outputs = {}
109
- for attribute_name in self.attributes:
110
- attribute = getattr(self, attribute_name, None)
111
- input_data, input_kwargs = attribute_to_kwargs[attribute_name]
112
- if input_data is not None and attribute is not None:
113
- attribute_output = attribute(input_data, **kwargs[input_kwargs])
114
- outputs.update(attribute_output)
115
-
116
- return BatchFeature(outputs)
64
+ image_special_ids = self.image_processor.mm_encoder.special_ids
65
+ self.image_break_id = image_special_ids.img_break
66
+ self.image_token_id = image_special_ids.img
67
+ self.image_end_id = image_special_ids.img_end
vllm/transformers_utils/processors/voxtral.py
+8
-54
@@ -8,9 +8,6 @@ import torch
8
from mistral_common.tokens.tokenizers.audio import AudioEncoder
9
from transformers import BatchFeature, ProcessorMixin, TensorType
10
from transformers.audio_utils import AudioInput
11
-from transformers.image_utils import ImageInput
12
-from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
13
-from transformers.video_utils import VideoInput
11
12
from vllm.tokenizers.mistral import MistralTokenizer
13
@@ -62,58 +59,15 @@ class MistralCommonVoxtralProcessor(ProcessorMixin):
59
60
def __init__(self, tokenizer: MistralTokenizer) -> None:
61
self.tokenizer = tokenizer.transformers_tokenizer
62
+
63
+ # Back-compatibility for Transformers v4
64
+ if not hasattr(self.tokenizer, "init_kwargs"):
65
+ self.tokenizer.init_kwargs = {}
66
+
67
self.feature_extractor = MistralCommonFeatureExtractor(
68
tokenizer.instruct.audio_encoder
69
)
70
69
- self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
70
-
71
- @property
72
- def audio_token_id(self) -> int:
73
- return self._audio_special_ids.audio
74
-
75
- @property
76
- def begin_audio_token_id(self) -> int:
77
- return self._audio_special_ids.begin_audio
78
-
79
- def __call__(
80
- self,
81
- images: ImageInput | None = None,
82
- text: TextInput
83
- | PreTokenizedInput
84
- | list[TextInput]
85
- | list[PreTokenizedInput]
86
- | None = None,
87
- videos: VideoInput | None = None,
88
- audio: AudioInput | None = None,
89
- **kwargs,
90
- ):
91
- if images is None and text is None and videos is None and audio is None:
92
- raise ValueError(
93
- f"You need to provide at least one input to "
94
- f"call {self.__class__.__name__}"
95
- )
96
-
97
- kwargs = self._merge_kwargs(
98
- self.valid_processor_kwargs,
99
- tokenizer_init_kwargs={},
100
- **kwargs,
101
- )
102
- kwargs["text_kwargs"]["return_tensors"] = "pt"
103
- kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
104
-
105
- attribute_to_kwargs = {
106
- "tokenizer": (text, "text_kwargs"),
107
- "image_processor": (images, "images_kwargs"),
108
- "video_processor": (videos, "videos_kwargs"),
109
- "feature_extractor": (audio, "audio_kwargs"),
110
- }
111
- outputs = {}
112
- for attribute_name in self.attributes:
113
- attribute = getattr(self, attribute_name, None)
114
- input_data, input_kwargs = attribute_to_kwargs[attribute_name]
115
- if input_data is not None and attribute is not None:
116
- attribute_output = attribute(input_data, **kwargs[input_kwargs])
117
- outputs.update(attribute_output)
118
-
119
- return BatchFeature(outputs)
71
+ audio_special_ids = self.feature_extractor.audio_encoder.special_ids
72
+ self.audio_token_id = audio_special_ids.audio
73
+ self.begin_audio_token_id = audio_special_ids.begin_audio