A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[VLM] Merged multimodal processor for Qwen2-Audio (#11303)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 19, 2024 at 14:14 UTC
6142ef0adafd76f6b33ff0adb9a097e45a5df279
11 files changed
+416
-360
examples/offline_inference_audio_language.py
+6
@@ -18,6 +18,10 @@ question_per_audio_count = {
18
2: "What sport and what nursery rhyme are referenced?"
19
}
20
21
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
22
+# lower-end GPUs.
23
+# Unless specified, these settings have been tested to work on a single L4.
24
+
25
26
# Ultravox 0.3
27
def run_ultravox(question: str, audio_count: int):
@@ -33,6 +37,8 @@ def run_ultravox(question: str, audio_count: int):
37
add_generation_prompt=True)
38
39
llm = LLM(model=model_name,
40
+ max_model_len=4096,
41
+ max_num_seqs=5,
42
trust_remote_code=True,
43
limit_mm_per_prompt={"audio": audio_count})
44
stop_token_ids = None
tests/models/decoder_only/audio_language/test_ultravox.py
+4
-5
@@ -5,6 +5,7 @@ import pytest
5
import pytest_asyncio
6
from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8
+from vllm.multimodal.audio import resample_audio
9
from vllm.sequence import SampleLogprobs
10
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
@@ -130,16 +131,14 @@ def run_test(
131
dtype=dtype,
132
postprocess_inputs=process,
133
auto_cls=AutoModel) as hf_model:
133
- import librosa
134
-
134
hf_outputs_per_audio = [
135
hf_model.generate_greedy_logprobs_limit(
136
[hf_prompt],
137
max_tokens,
138
num_logprobs=num_logprobs,
140
- audios=[(librosa.resample(audio[0],
141
- orig_sr=audio[1],
142
- target_sr=16000), 16000)])
139
+ audios=[(resample_audio(audio[0],
140
+ orig_sr=audio[1],
141
+ target_sr=16000), 16000)])
142
for _, hf_prompt, audio in prompts_and_audios
143
]
144
vllm/inputs/registry.py
+70
-30
@@ -1,11 +1,11 @@
1
import functools
2
from collections import UserDict
3
from dataclasses import dataclass
4
-from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5
- Optional, Protocol, Type)
4
+from typing import (TYPE_CHECKING, Any, Callable, Mapping, NamedTuple,
5
+ Optional, Protocol, Union)
6
7
from torch import nn
8
-from transformers import PretrainedConfig, ProcessorMixin
8
+from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
9
from typing_extensions import TypeVar, assert_never
10
11
from vllm.logger import init_logger
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
26
logger = init_logger(__name__)
27
28
C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
29
+P = TypeVar("P", bound=ProcessorMixin, default=ProcessorMixin)
30
31
32
@dataclass(frozen=True)
@@ -38,24 +39,28 @@ class InputContext:
39
model_config: "ModelConfig"
40
"""The configuration of the model."""
41
41
- def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
42
+ def get_hf_config(
43
+ self,
44
+ typ: Union[type[C], tuple[type[C], ...]] = PretrainedConfig,
45
+ /,
46
+ ) -> C:
47
"""
48
Get the HuggingFace configuration
49
(:class:`transformers.PretrainedConfig`) of the model,
50
additionally checking its type.
51
52
Raises:
48
- TypeError: If the model is not of the specified type.
53
+ TypeError: If the configuration is not of the specified type.
54
"""
55
hf_config = self.model_config.hf_config
51
- if not isinstance(hf_config, hf_config_type):
56
+ if not isinstance(hf_config, typ):
57
raise TypeError("Invalid type of HuggingFace config. "
53
- f"Expected type: {hf_config_type}, but "
58
+ f"Expected type: {typ}, but "
59
f"found type: {type(hf_config)}")
60
61
return hf_config
62
58
- def get_hf_image_processor_config(self) -> Dict[str, Any]:
63
+ def get_hf_image_processor_config(self) -> dict[str, Any]:
64
"""
65
Get the HuggingFace image processor configuration of the model.
66
"""
@@ -74,18 +79,37 @@ class InputContext:
79
80
return mm_config
81
77
- def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
82
+ def get_hf_processor(
83
+ self,
84
+ typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
85
+ /,
86
+ **kwargs: object,
87
+ ) -> P:
88
+ """
89
+ Get the HuggingFace processor
90
+ (:class:`transformers.ProcessorMixin`) of the model,
91
+ additionally checking its type.
92
+
93
+ Raises:
94
+ TypeError: If the processor is not of the specified type.
95
+ """
96
base_kwargs = self.model_config.mm_processor_kwargs
97
if base_kwargs is None:
98
base_kwargs = {}
99
100
merged_kwargs = {**base_kwargs, **kwargs}
101
84
- return cached_get_processor(
102
+ hf_processor = cached_get_processor(
103
self.model_config.model,
104
trust_remote_code=self.model_config.trust_remote_code,
105
**merged_kwargs,
106
)
107
+ if not isinstance(hf_processor, typ):
108
+ raise TypeError("Invalid type of HuggingFace processor. "
109
+ f"Expected type: {typ}, but "
110
+ f"found type: {type(hf_processor)}")
111
+
112
+ return hf_processor
113
114
115
@dataclass(frozen=True)
@@ -93,39 +117,55 @@ class InputProcessingContext(InputContext):
117
tokenizer: AnyTokenizer
118
"""The tokenizer used to tokenize the inputs."""
119
96
- def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
97
- base_kwargs = self.model_config.mm_processor_kwargs
98
- if base_kwargs is None:
99
- base_kwargs = {}
100
-
101
- merged_kwargs = {**base_kwargs, **kwargs}
102
-
103
- return cached_get_processor(
104
- self.model_config.model,
105
- tokenizer=self.tokenizer, # Override the tokenizer with ours
106
- trust_remote_code=self.model_config.trust_remote_code,
107
- **merged_kwargs,
120
+ def get_hf_processor(
121
+ self,
122
+ typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
123
+ /,
124
+ **kwargs: object,
125
+ ) -> P:
126
+ return super().get_hf_processor(
127
+ typ,
128
+ tokenizer=self.tokenizer,
129
+ **kwargs,
130
)
131
110
- def resolve_hf_processor_call_kwargs(
132
+ def call_hf_processor(
133
self,
134
hf_processor: ProcessorMixin,
135
+ prompt: str,
136
+ processor_data: Mapping[str, object],
137
inference_kwargs: Mapping[str, object],
114
- ) -> Mapping[str, object]:
138
+ ) -> BatchFeature:
139
assert callable(hf_processor)
140
141
base_kwargs = self.model_config.mm_processor_kwargs
142
if base_kwargs is None:
143
base_kwargs = {}
144
121
- return resolve_mm_processor_kwargs(
145
+ merged_kwargs = resolve_mm_processor_kwargs(
146
base_kwargs,
147
inference_kwargs,
148
hf_processor,
149
+ requires_kw_only=False,
150
+ allow_var_kwargs=True,
151
)
152
153
+ try:
154
+ return hf_processor(
155
+ text=prompt,
156
+ **processor_data,
157
+ **merged_kwargs,
158
+ return_tensors="pt",
159
+ )
160
+ except Exception as exc:
161
+ data = dict(text=prompt, **processor_data)
162
+ msg = (f"Failed to apply {type(hf_processor).__name__} "
163
+ f"on data={data} with kwargs={merged_kwargs}")
164
+
165
+ raise RuntimeError(msg) from exc
166
+
167
128
-N = TypeVar("N", bound=Type[nn.Module])
168
+N = TypeVar("N", bound=type[nn.Module])
169
170
171
class DummyData(NamedTuple):
@@ -232,7 +272,7 @@ class InputRegistry:
272
273
return wrapper
274
235
- def _get_dummy_data_factory(self, model_cls: Type[nn.Module]):
275
+ def _get_dummy_data_factory(self, model_cls: type[nn.Module]):
276
return self._dummy_factories_by_model_type \
277
.get(model_cls, self._default_dummy_data_factory)
278
@@ -257,7 +297,7 @@ class InputRegistry:
297
298
return wrapper
299
260
- def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
300
+ def _get_dummy_encoder_data_factory(self, model_cls: type[nn.Module]):
301
return self._dummy_encoder_factories_by_model_type \
302
.get(model_cls, self._default_dummy_data_factory)
303
@@ -368,14 +408,14 @@ class InputRegistry:
408
409
return wrapper
410
371
- def _get_model_input_processor(self, model_cls: Type[nn.Module]):
411
+ def _get_model_input_processor(self, model_cls: type[nn.Module]):
412
return self._input_processors_by_model_type \
413
.get(model_cls, self._default_input_processor)
414
415
def _ensure_mm_kwargs(
416
self,
417
inputs: SingletonInputs,
378
- mm_processor_kwargs: Dict[str, Any],
418
+ mm_processor_kwargs: dict[str, Any],
419
):
420
if inputs["type"] == "token":
421
# In case the input processor for that model fails to set it
vllm/model_executor/models/llava.py
+2
-2
@@ -133,8 +133,8 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
133
hf_processor.__is_patched__ = True # type: ignore
134
135
def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136
- hf_processor = self.ctx.get_hf_processor()
137
- assert isinstance(hf_processor, (LlavaProcessor, PixtralProcessor))
136
+ hf_processor = self.ctx.get_hf_processor(
137
+ (LlavaProcessor, PixtralProcessor))
138
139
if isinstance(hf_processor, PixtralProcessor):
140
self._patch_pixtral_processor(hf_processor)
vllm/model_executor/models/phi3v.py
+11
-5
@@ -34,7 +34,6 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
from vllm.multimodal.inputs import NestedTensors
36
from vllm.multimodal.processing import (BaseMultiModalProcessor,
37
- MultiModalDataDict,
37
MultiModalDataItems, ProcessorInputs,
38
PromptReplacement)
39
from vllm.sequence import IntermediateTensors
@@ -330,20 +329,27 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
329
return self.ctx.get_hf_processor(num_crops=num_crops)
330
return self.ctx.get_hf_processor()
331
333
- def _apply_hf_processor(
332
+ def _call_hf_processor(
333
self,
334
+ hf_processor: ProcessorMixin,
335
prompt: str,
336
- mm_data: MultiModalDataDict,
336
+ processor_data: Mapping[str, object],
337
mm_processor_kwargs: Mapping[str, object],
338
) -> BatchFeature:
339
- processed_outputs = super()._apply_hf_processor(
340
- prompt, mm_data, mm_processor_kwargs)
339
+ processed_outputs = super()._call_hf_processor(
340
+ hf_processor,
341
+ prompt=prompt,
342
+ processor_data=processor_data,
343
+ mm_processor_kwargs=mm_processor_kwargs,
344
+ )
345
+
346
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
347
# which will cause OverflowError when decoding the prompt_ids.
348
# Therefore, we need to do an early replacement here
349
token_ids = processed_outputs['input_ids']
350
token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
351
processed_outputs['input_ids'] = token_ids
352
+
353
return processed_outputs
354
355
def _get_prompt_replacements(
vllm/model_executor/models/qwen2_audio.py
+113
-190
@@ -19,45 +19,43 @@
19
# See the License for the specific language governing permissions and
20
# limitations under the License.
21
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22
-from functools import cached_property, lru_cache
23
-from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
- Union)
22
+from functools import cached_property
23
+from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24
+ TypedDict, Union)
25
26
-import librosa
26
import numpy as np
27
import torch
28
import torch.nn as nn
30
-from transformers import Qwen2AudioEncoder
29
+from transformers import BatchFeature, ProcessorMixin
30
+from transformers.models.qwen2_audio import (Qwen2AudioConfig,
31
+ Qwen2AudioEncoder,
32
+ Qwen2AudioProcessor)
33
+from transformers.models.whisper import WhisperFeatureExtractor
34
35
from vllm.attention import AttentionMetadata
36
from vllm.config import VllmConfig
34
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35
- InputContext, token_inputs)
36
-from vllm.logger import init_logger
37
+from vllm.inputs import InputContext
38
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40
+from vllm.multimodal import MULTIMODAL_REGISTRY
41
from vllm.multimodal.inputs import NestedTensors
41
-from vllm.multimodal.utils import consecutive_placeholder_ranges
42
-from vllm.sequence import IntermediateTensors, SequenceData
42
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
+ MultiModalDataItems, ProcessorInputs,
44
+ PromptReplacement)
45
+from vllm.sequence import IntermediateTensors
46
47
from .interfaces import SupportsMultiModal, SupportsPP
48
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
49
maybe_prefix, merge_multimodal_embeddings)
50
48
-logger = init_logger(__name__)
49
-
51
52
# # === Audio Inputs === #
53
class Qwen2AudioInputs(TypedDict):
54
input_features: torch.Tensor
54
- """Shape:
55
- `(num_audios, num_mel_bins, 3000)`
56
- """
55
+ """Shape: `(num_audios, num_mel_bins, 3000)`"""
56
57
feature_attention_mask: torch.Tensor
59
- """Shape: `(num_audios, 3000)`
60
- """
58
+ """Shape: `(num_audios, 3000)`"""
59
60
61
# === Audio Encoder === #
@@ -74,187 +72,114 @@ class Qwen2AudioMultiModalProjector(nn.Module):
72
return hidden_states
73
74
77
-def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
78
- mm_counts: Mapping[str, int]):
79
- num_audios = mm_counts["audio"]
80
- max_tokens_per_audio = get_max_qwen2_audio_audio_tokens(ctx)
81
- max_llm_audio_tokens = max_tokens_per_audio * num_audios
82
- if seq_len - max_llm_audio_tokens - 2 < 0:
83
- raise RuntimeError(
84
- f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
85
- "please increase max_model_len or reduce audio limit by "
86
- "--limit-mm-per-prompt.")
87
-
88
- audio_token_index = ctx.model_config.hf_config.audio_token_index
89
-
90
- dummy_seqdata = SequenceData.from_prompt_token_counts(
91
- (audio_token_index, max_llm_audio_tokens),
92
- (0, seq_len - max_llm_audio_tokens),
93
- )
94
- dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
95
- return DummyData(
96
- dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}, {
97
- "audio":
98
- consecutive_placeholder_ranges(num_items=num_audios,
99
- item_size=max_tokens_per_audio)
100
- })
101
-
102
-
103
-def get_processor(
104
- processor_name: str,
105
- *args,
106
- trust_remote_code: bool = False,
107
- **kwargs,
108
-):
109
- """Gets a processor for the given model name via HuggingFace.
110
-
111
- Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
112
- """
113
- # don't put this import at the top level
114
- # it will call torch.cuda.device_count()
115
- from transformers import AutoProcessor
116
-
117
- try:
118
- processor = AutoProcessor.from_pretrained(
119
- processor_name,
120
- *args,
121
- trust_remote_code=trust_remote_code,
122
- **kwargs)
123
- except ValueError as e:
124
- # If the error pertains to the processor class not existing or not
125
- # currently being imported, suggest using the --trust-remote-code flag.
126
- # Unlike AutoTokenizer, AutoProcessor does not separate such errors
127
- if not trust_remote_code:
128
- err_msg = (
129
- "Failed to load the processor. If the processor is "
130
- "a custom processor not yet available in the HuggingFace "
131
- "transformers library, consider setting "
132
- "`trust_remote_code=True` in LLM or using the "
133
- "`--trust-remote-code` flag in the CLI.")
134
- raise RuntimeError(err_msg) from e
135
- else:
136
- raise e
75
+# From Qwen2AudioEncoder._get_feat_extract_output_lengths
76
+def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
77
+ feat_lengths = (input_lengths - 1) // 2 + 1
78
+ output_lengths = (feat_lengths - 2) // 2 + 1
79
+ return feat_lengths, output_lengths
80
138
- return processor
81
82
+def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
83
+ hf_config = ctx.get_hf_config(Qwen2AudioConfig)
84
+ max_source_position = hf_config.audio_config.max_source_positions
85
+ output_lengths = (max_source_position - 2) // 2 + 1
86
+ return output_lengths
87
141
-cached_get_processor = lru_cache(get_processor)
88
89
+class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
90
144
-def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
145
- """
146
- Computes the output length of the convolutional layers
147
- and the output length of the audio encoder
148
- """
149
- input_lengths = (input_lengths - 1) // 2 + 1
150
- output_lengths = (input_lengths - 2) // 2 + 1
151
- return input_lengths, output_lengths
91
+ def _get_hf_processor(self) -> Qwen2AudioProcessor:
92
+ return self.ctx.get_hf_processor(Qwen2AudioProcessor)
93
94
+ def _get_feature_extractor(self) -> WhisperFeatureExtractor:
95
+ return self._get_hf_processor().feature_extractor # type: ignore
96
154
-def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
155
- max_source_position = (
156
- ctx.model_config.hf_config.audio_config.max_source_positions)
157
- output_lengths = (max_source_position - 2) // 2 + 1
158
- return output_lengths
97
+ def _get_processor_data(
98
+ self,
99
+ mm_items: MultiModalDataItems,
100
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
101
+ # resample audio to the model's sampling rate
102
+ feature_extractor = self._get_feature_extractor()
103
+ mm_items.resample_audios(feature_extractor.sampling_rate)
104
105
+ return super()._get_processor_data(mm_items)
106
161
-def input_processor_for_qwen2_audio(
162
- ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
163
- multi_modal_data = inputs.get("multi_modal_data")
164
- if multi_modal_data is None or "audio" not in multi_modal_data:
165
- return inputs
166
-
167
- audios = multi_modal_data["audio"]
168
- if not isinstance(audios, list):
169
- audios = [audios]
170
-
171
- if len(audios) == 0:
172
- return inputs
173
-
174
- processor = cached_get_processor(ctx.model_config.model)
175
- resampled_audios = [
176
- librosa.resample(audio,
177
- orig_sr=sampling_rate,
178
- target_sr=processor.feature_extractor.sampling_rate)
179
- for audio, sampling_rate in audios
180
- ]
181
- audio_input_lengths = np.array(
182
- [min(3000, _.shape[0] // 160 + 1) for _ in resampled_audios])
183
-
184
- audio_feat_lengths, audio_output_lengths = _get_feat_extract_output_lengths(
185
- audio_input_lengths)
186
-
187
- audio_token_index = ctx.model_config.hf_config.audio_token_index
188
-
189
- input_ids = inputs['prompt_token_ids']
190
-
191
- new_input_ids = []
192
- audio_num = input_ids.count(audio_token_index)
193
- assert len(audio_input_lengths) == audio_num, \
194
- (f'The text input contains {audio_num} audio tokens, '
195
- f'but {len(audio_input_lengths)} audios provided')
196
- start = 0
197
- for audio_idx in range(audio_num):
198
- end = input_ids.index(audio_token_index, start)
199
- new_input_ids.extend(input_ids[start:end]) # text part
200
-
201
- new_input_ids.extend([audio_token_index] *
202
- audio_output_lengths[audio_idx])
203
- start = end + 1
204
- new_input_ids.extend(input_ids[start:])
205
-
206
- return token_inputs(
207
- prompt_token_ids=new_input_ids,
208
- prompt=inputs.get("prompt"),
209
- multi_modal_data=multi_modal_data,
210
- )
211
-
212
-
213
-def input_mapper_for_qwen2_audio(
214
- ctx: InputContext,
215
- multi_modal_data: Union[np.ndarray, List[np.ndarray]],
216
-) -> MultiModalKwargs:
217
- """Input mapper for Qwen2-Audio."""
218
- if not isinstance(multi_modal_data, list):
219
- multi_modal_data = [multi_modal_data]
220
-
221
- if len(multi_modal_data) == 0:
222
- return MultiModalKwargs()
223
-
224
- processor = cached_get_processor(ctx.model_config.model)
225
- audio_feature_extractor = processor.feature_extractor
226
- if audio_feature_extractor is None:
227
- raise RuntimeError(
228
- "No HuggingFace audio_feature_extractor is available "
229
- "to process the audio object")
230
-
231
- try:
232
- resampled_audios = [
233
- librosa.resample(
234
- audio,
235
- orig_sr=sampling_rate,
236
- target_sr=processor.feature_extractor.sampling_rate)
237
- for audio, sampling_rate in multi_modal_data
107
+ def _call_hf_processor(
108
+ self,
109
+ hf_processor: ProcessorMixin,
110
+ prompt: str,
111
+ processor_data: Mapping[str, object],
112
+ mm_processor_kwargs: Mapping[str, object],
113
+ ) -> BatchFeature:
114
+ processor_data = dict(processor_data)
115
+ audios = processor_data.pop("audios", [])
116
+
117
+ if audios:
118
+ processor_data["audios"] = audios
119
+
120
+ feature_extractor = self._get_feature_extractor()
121
+ mm_processor_kwargs = dict(
122
+ **mm_processor_kwargs,
123
+ sampling_rate=feature_extractor.sampling_rate,
124
+ )
125
+ else:
126
+ # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
127
+ pass
128
+
129
+ return super()._call_hf_processor(
130
+ hf_processor,
131
+ prompt=prompt,
132
+ processor_data=processor_data,
133
+ mm_processor_kwargs=mm_processor_kwargs,
134
+ )
135
+
136
+ def _get_prompt_replacements(
137
+ self,
138
+ mm_items: MultiModalDataItems,
139
+ hf_inputs: BatchFeature,
140
+ mm_processor_kwargs: Mapping[str, object],
141
+ ) -> list[PromptReplacement]:
142
+ hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
143
+ placeholder = hf_config.audio_token_index
144
+
145
+ feature_attention_mask = hf_inputs.get("feature_attention_mask")
146
+ if feature_attention_mask is None:
147
+ audio_output_lengths = []
148
+ else:
149
+ _, audio_output_lengths = _get_feat_extract_output_lengths(
150
+ feature_attention_mask.sum(-1))
151
+
152
+ def get_replacement_qwen2_audio(item_idx: int):
153
+ return [placeholder] * audio_output_lengths[item_idx]
154
+
155
+ return [
156
+ PromptReplacement(
157
+ modality="audio",
158
+ target=[placeholder],
159
+ replacement=get_replacement_qwen2_audio,
160
+ )
161
]
239
- batch_data = audio_feature_extractor(resampled_audios,
240
- sampling_rate=16000,
241
- return_attention_mask=True,
242
- padding="max_length",
243
- return_tensors="pt").data
244
- batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
245
- except Exception:
246
- logger.error("Failed to process audio (%s)", multi_modal_data)
247
- raise
248
-
249
- return MultiModalKwargs(batch_data)
250
-
251
-
252
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_audio)
253
-@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_audio)
254
-@MULTIMODAL_REGISTRY.register_input_mapper("audio",
255
- input_mapper_for_qwen2_audio)
162
+
163
+ def _get_dummy_mm_inputs(
164
+ self,
165
+ mm_counts: Mapping[str, int],
166
+ ) -> ProcessorInputs:
167
+ audio_len = get_max_qwen2_audio_audio_tokens(self.ctx)
168
+
169
+ audio_count = mm_counts["audio"]
170
+ audio = np.zeros(audio_len)
171
+ data = {"audio": [audio] * audio_count}
172
+
173
+ return ProcessorInputs(
174
+ prompt_text="<|AUDIO|>" * audio_count,
175
+ mm_data=data,
176
+ mm_processor_kwargs={},
177
+ )
178
+
179
+
180
@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
181
"audio", get_max_qwen2_audio_audio_tokens)
182
+@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
183
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
184
SupportsPP):
185
@@ -289,9 +214,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
214
215
return get_sampler()
216
292
- def _validate_and_reshape_mm_tensor(self,
293
- mm_input: Union[torch.Tensor,
294
- List[torch.Tensor]],
217
+ def _validate_and_reshape_mm_tensor(self, mm_input: object,
218
name: str) -> torch.Tensor:
219
if not isinstance(mm_input, (torch.Tensor, list)):
220
raise ValueError(f"Incorrect type of {name}. "
vllm/model_executor/models/ultravox.py
+56
-57
@@ -3,7 +3,7 @@
3
4
import math
5
from functools import cached_property, lru_cache
6
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
6
+from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7
Tuple, TypedDict, Union)
8
9
import numpy as np
@@ -11,7 +11,7 @@ import torch
11
import torch.utils.checkpoint
12
from torch import nn
13
from torch.nn import functional as F
14
-from transformers import BatchFeature
14
+from transformers import BatchFeature, ProcessorMixin
15
from transformers.models.whisper import WhisperFeatureExtractor
16
from transformers.models.whisper.modeling_whisper import WhisperEncoder
17
@@ -25,11 +25,11 @@ from vllm.model_executor.model_loader.loader import DefaultModelLoader
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- MultiModalDataDict,
28
MultiModalDataItems, ProcessorInputs,
29
PromptReplacement)
30
from vllm.sequence import IntermediateTensors
31
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
32
+from vllm.utils import is_list_of
33
34
from .interfaces import SupportsMultiModal, SupportsPP
35
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -61,8 +61,8 @@ def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
61
62
63
def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
64
- return cached_feature_extractor(
65
- ctx.get_hf_config(UltravoxConfig).audio_model_id)
64
+ hf_config = ctx.get_hf_config(UltravoxConfig)
65
+ return cached_feature_extractor(hf_config.audio_model_id)
66
67
68
def get_ultravox_max_audio_tokens(ctx: InputContext):
@@ -73,72 +73,71 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
73
class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
74
75
def _get_feature_extractor(self) -> WhisperFeatureExtractor:
76
- return self._get_hf_processor().audio_processor.feature_extractor
76
+ hf_processor = self._get_hf_processor()
77
+ return hf_processor.audio_processor.feature_extractor # type: ignore
78
78
- def _resample_audio(
79
+ def _get_processor_data(
80
self,
80
- audio: np.ndarray,
81
- sr: int,
82
- ) -> Dict[str, Union[np.ndarray, int]]:
81
+ mm_items: MultiModalDataItems,
82
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
83
# resample audio to the model's sampling rate
84
feature_extractor = self._get_feature_extractor()
85
- if sr != feature_extractor.sampling_rate:
86
- try:
87
- import librosa
88
- except ImportError as exc:
89
- raise ImportError(
90
- "Please install vllm[audio] for audio support.") from exc
91
- audio = librosa.resample(audio,
92
- orig_sr=sr,
93
- target_sr=feature_extractor.sampling_rate)
94
- sr = feature_extractor.sampling_rate
95
- return {"audio": audio, "sampling_rate": sr}
96
-
97
- def _apply_hf_processor(
85
+ mm_items.resample_audios(feature_extractor.sampling_rate)
86
+
87
+ return super()._get_processor_data(mm_items)
88
+
89
+ def _call_hf_processor(
90
self,
91
+ hf_processor: ProcessorMixin,
92
prompt: str,
100
- mm_data: MultiModalDataDict,
93
+ processor_data: Mapping[str, object],
94
mm_processor_kwargs: Mapping[str, object],
95
) -> BatchFeature:
103
- if not mm_data or not mm_data.get("audio", None):
104
- return super()._apply_hf_processor(prompt, mm_data,
105
- mm_processor_kwargs)
96
+ processor_data = dict(processor_data)
97
+ audios = processor_data.pop("audios", [])
98
+
99
+ if not audios:
100
+ return super()._call_hf_processor(
101
+ hf_processor,
102
+ prompt=prompt,
103
+ processor_data=processor_data,
104
+ mm_processor_kwargs=mm_processor_kwargs,
105
+ )
106
107
- audio_data = mm_data["audio"]
108
- if not isinstance(audio_data, list):
109
- audio_data = [audio_data]
107
+ feature_extractor = self._get_feature_extractor()
108
+ mm_processor_kwargs = dict(
109
+ **mm_processor_kwargs,
110
+ sampling_rate=feature_extractor.sampling_rate,
111
+ )
112
+
113
+ # Already resampled by _get_processor_data
114
+ assert is_list_of(audios, np.ndarray)
115
116
# Ultravox processor doesn't support multiple inputs,
117
# therefore we need to input text and audio one by one
113
- tokenizer = self._get_tokenizer()
118
audio_features, audio_token_len = [], []
115
- processed_inputs = {}
116
- for audio, sr in audio_data:
117
- data = self._resample_audio(audio, sr)
118
- processed_inputs = super()._apply_hf_processor(
119
- prompt, data, mm_processor_kwargs)
120
- prompt = tokenizer.decode(processed_inputs["input_ids"][0],
121
- skip_special_tokens=False)
122
- audio_features.append(
123
- processed_inputs.pop("audio_values").squeeze(0))
124
- audio_token_len.append(
125
- processed_inputs.pop("audio_token_len").item())
126
-
127
- return dict(
128
- **processed_inputs,
119
+ shared_outputs = {}
120
+ for audio in audios:
121
+ # NOTE: Ultravox processor accepts "audio" instead of "audios"
122
+ item_processor_data = dict(**processor_data, audio=audio)
123
+
124
+ item_outputs = super()._call_hf_processor(
125
+ hf_processor,
126
+ prompt=prompt,
127
+ processor_data=item_processor_data,
128
+ mm_processor_kwargs=mm_processor_kwargs,
129
+ )
130
+
131
+ audio_features.append(item_outputs.pop("audio_values")[0])
132
+ audio_token_len.append(item_outputs.pop("audio_token_len").item())
133
+ shared_outputs = item_outputs
134
+
135
+ combined_outputs = dict(
136
+ **shared_outputs,
137
audio_features=audio_features,
138
audio_token_len=audio_token_len,
139
)
132
-
133
- def _get_processor_data(
134
- self,
135
- mm_data: MultiModalDataDict,
136
- ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
137
- # Ultravox uses "audio" instead of "audios" as calling keyword
138
- processor_data, passthrough_data = super()._get_processor_data(mm_data)
139
- if "audios" in processor_data:
140
- processor_data["audio"] = processor_data.pop("audios")
141
- return processor_data, passthrough_data
140
+ return BatchFeature(combined_outputs)
141
142
def _get_prompt_replacements(
143
self,
@@ -147,7 +146,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
146
mm_processor_kwargs: Mapping[str, object],
147
) -> list[PromptReplacement]:
148
hf_processor = self._get_hf_processor()
150
- placeholder = hf_processor.audio_token_replacement
149
+ placeholder = hf_processor.audio_token_replacement # type: ignore
150
151
def get_replacement_ultravox(item_idx: int):
152
audio_token_len = hf_inputs["audio_token_len"][item_idx]
@@ -171,7 +170,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
170
171
audio_count = mm_counts["audio"]
172
audio = np.zeros(audio_len)
174
- data = {"audio": [(audio, sampling_rate)] * audio_count}
173
+ data = {"audio": [audio] * audio_count}
174
175
return ProcessorInputs(
176
prompt_text="<|audio|>" * audio_count,
vllm/multimodal/audio.py
+18
@@ -1,3 +1,6 @@
1
+import numpy as np
2
+import numpy.typing as npt
3
+
4
from vllm.inputs.registry import InputContext
5
6
from .base import MultiModalPlugin
@@ -21,3 +24,18 @@ class AudioPlugin(MultiModalPlugin):
24
def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
25
raise NotImplementedError(
26
"There is no default maximum multimodal tokens")
27
+
28
+
29
+def resample_audio(
30
+ audio: npt.NDArray[np.floating],
31
+ *,
32
+ orig_sr: float,
33
+ target_sr: float,
34
+) -> npt.NDArray[np.floating]:
35
+ try:
36
+ import librosa
37
+ except ImportError as exc:
38
+ msg = "Please install vllm[audio] for audio support."
39
+ raise ImportError(msg) from exc
40
+
41
+ return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
vllm/multimodal/inputs.py
+13
-12
@@ -15,31 +15,32 @@ _T = TypeVar("_T")
15
# yapf: disable
16
ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
17
"""
18
-A :class:`transformers.image_utils.ImageInput` representing a single image,
19
-which can be passed to a HuggingFace :code:`ImageProcessor`.
18
+A :class:`transformers.image_utils.ImageInput` representing a single image
19
+item, which can be passed to a HuggingFace :code:`ImageProcessor`.
20
"""
21
22
VideoItem: TypeAlias = Union[
23
- List[Image],
23
+ list[Image],
24
np.ndarray,
25
torch.Tensor,
26
- List[np.ndarray],
27
- List[torch.Tensor],
26
+ list[np.ndarray],
27
+ list[torch.Tensor],
28
]
29
"""
30
-
31
-A :class:`transformers.image_utils.VideoInput` representing a single video,
32
-which can be passed to a HuggingFace :code:`VideoProcessor`.
30
+A :class:`transformers.image_utils.VideoInput` representing a single video
31
+item, which can be passed to a HuggingFace :code:`VideoProcessor`.
32
"""
33
34
AudioItem: TypeAlias = Union[
35
np.ndarray,
37
- List[float],
38
- Tuple[np.ndarray, float], # DEPRECATED: Use mm_processor_kwargs instead
36
+ list[float],
37
+ # `(audio, sampling_rate)`: If the audio's sampling rate is different
38
+ # from that expected by the model, we need to resample it.
39
+ tuple[np.ndarray, float],
40
]
41
"""
41
-Represents a single audio that can be inputted to a HuggingFace
42
-:code:`AudioProcessor`.
42
+Represents a single audio
43
+item, which can be passed to a HuggingFace :code:`AudioProcessor`.
44
"""
45
# yapf: enable
46
vllm/multimodal/processing.py
+100
-52
@@ -17,6 +17,7 @@ from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18
from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
19
20
+from .audio import resample_audio
21
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
22
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
23
VideoItem)
@@ -30,7 +31,7 @@ _PromptSeq = Union[str, list[int]]
31
@dataclass
32
class PromptReplacement:
33
modality: str
33
- """The modality for which the replacement is made"""
34
+ """The modality for which the replacement is made."""
35
36
target: _PromptSeq
37
"""The text or token sequence to find and replace."""
@@ -211,20 +212,48 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
212
corresponds to a list.
213
"""
214
215
+ @staticmethod
216
+ def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
217
+ """
218
+ Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
219
+ """
220
+ multi_data = MultiModalDataItems()
221
+
222
+ for k, v in data.items():
223
+ # yapf: disable
224
+ if k == "video":
225
+ # Special case since even a single item can be a list
226
+ multi_data[k] = ( # type: ignore[index]
227
+ v if is_list_of(v, (list, torch.Tensor)) else [v]
228
+ )
229
+ elif k in ("image", "audio"):
230
+ multi_data[k] = ( # type: ignore[index]
231
+ v if isinstance(v, (list, torch.Tensor)) else [v]
232
+ )
233
+ else:
234
+ multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
235
+ # yapf: enable
236
+
237
+ return multi_data
238
+
239
+ # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
240
+ # `self.images` doesn't update this dictionary, which may be confusing
241
+ # We annotate the getter methods as `Sequence` to prevent others from
242
+ # trying to update the list in this way
243
@property
215
- def image(self) -> list[ImageItem]:
216
- return self["image"]
244
+ def images(self) -> Sequence[ImageItem]:
245
+ return self.get("image", [])
246
247
@property
219
- def video(self) -> list[VideoItem]:
220
- return self["video"]
248
+ def videos(self) -> Sequence[VideoItem]:
249
+ return self.get("video", [])
250
251
@property
223
- def audio(self) -> list[AudioItem]:
224
- return self["audio"]
252
+ def audios(self) -> Sequence[AudioItem]:
253
+ return self.get("audio", [])
254
255
def get_image_size(self, item_idx: int) -> ImageSize:
227
- image = self.image[item_idx]
256
+ image = self.images[item_idx]
257
258
if isinstance(image, Image):
259
return ImageSize(*image.size)
@@ -234,25 +263,41 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
263
264
assert_never(image)
265
266
+ def get_audio_with_sr(
267
+ self,
268
+ item_idx: int,
269
+ *,
270
+ default_sr: float,
271
+ ) -> tuple[np.ndarray, float]:
272
+ audio = self.audios[item_idx]
273
+
274
+ if isinstance(audio, tuple):
275
+ return audio
276
+ if isinstance(audio, list):
277
+ return np.array(audio), default_sr
278
+ if isinstance(audio, np.ndarray):
279
+ return audio, default_sr
280
+
281
+ assert_never(audio)
282
+
283
+ def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
284
+ """
285
+ If :code:`drop_sr=True`, the audio items in this dictionary are updated
286
+ to be NumPy arrays which implicitly means that their sampling rate is
287
+ the same as the model's expected sampling rate; otherwise, they remain
288
+ as :code:`(audio, new_sr)` tuples.
289
+ """
290
+ if not self.audios:
291
+ return
292
+
293
+ new_audios = []
294
+ for item_idx in range(len(self.audios)):
295
+ audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
296
+ audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
297
238
-def to_multi_format(data: MultiModalDataDict) -> MultiModalDataItems:
239
- """
240
- Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
241
- """
242
- multi_data = MultiModalDataItems()
243
-
244
- for k, v in data.items():
245
- # yapf: disable
246
- if k == "video":
247
- # Special case since even a single item can be a list
248
- multi_data[k] = v if is_list_of(v, list) else [v] # type: ignore[index]
249
- elif k in ("image", "audio"):
250
- multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
251
- else:
252
- multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
253
- # yapf: enable
298
+ new_audios.append(audio if drop_sr else (audio, new_sr))
299
255
- return multi_data
300
+ self["audio"] = new_audios
301
302
303
class _TokenMatch(NamedTuple):
@@ -596,18 +641,20 @@ class BaseMultiModalProcessor(ABC):
641
642
def _get_processor_data(
643
self,
599
- mm_data: MultiModalDataDict,
600
- ) -> BatchFeature:
644
+ mm_items: MultiModalDataItems,
645
+ ) -> tuple[dict[str, Any], dict[str, Any]]:
646
processor_data = dict[str, Any]()
647
passthrough_data = dict[str, Any]()
603
- for k, v in mm_data.items():
648
+
649
+ for k, v in mm_items.items():
650
# TODO: Make a separate modality for embedding inputs
651
# to avoid confusion
652
if k in ("image", "video", "audio"):
653
if isinstance(v, torch.Tensor) and v.ndim == 3:
654
# Pass through embedding inputs (single)
655
passthrough_data[f"{k}_embeds"] = [v]
610
- elif is_list_of(v, torch.Tensor) and v[0].ndim == 2:
656
+ elif (is_list_of(v, torch.Tensor) and len(v) > 0
657
+ and v[0].ndim == 2):
658
# Pass through embedding inputs (multi)
659
passthrough_data[f"{k}_embeds"] = v
660
else:
@@ -615,40 +662,41 @@ class BaseMultiModalProcessor(ABC):
662
processor_data[f"{k}s"] = v
663
else:
664
processor_data[k] = v
665
+
666
return processor_data, passthrough_data
667
668
+ def _call_hf_processor(
669
+ self,
670
+ hf_processor: ProcessorMixin,
671
+ prompt: str,
672
+ processor_data: Mapping[str, object],
673
+ mm_processor_kwargs: Mapping[str, object],
674
+ ) -> BatchFeature:
675
+ return self.ctx.call_hf_processor(
676
+ hf_processor,
677
+ prompt,
678
+ processor_data,
679
+ mm_processor_kwargs,
680
+ )
681
+
682
def _apply_hf_processor(
683
self,
684
prompt: str,
623
- mm_data: MultiModalDataDict,
685
+ mm_items: MultiModalDataItems,
686
mm_processor_kwargs: Mapping[str, object],
687
) -> BatchFeature:
688
# some mm_processor_kwargs may be used in processor initialization
689
# instead of processor call
690
hf_processor = self._get_hf_processor(**mm_processor_kwargs)
691
630
- processor_data, passthrough_data = self._get_processor_data(mm_data)
692
+ processor_data, passthrough_data = self._get_processor_data(mm_items)
693
632
- assert callable(hf_processor)
633
- mm_processor_kwargs = self.ctx.resolve_hf_processor_call_kwargs(
694
+ hf_inputs = self._call_hf_processor(
695
hf_processor,
635
- mm_processor_kwargs,
696
+ prompt=prompt,
697
+ processor_data=processor_data,
698
+ mm_processor_kwargs=mm_processor_kwargs,
699
)
637
-
638
- try:
639
- hf_inputs = hf_processor(
640
- text=prompt, # type: ignore
641
- **processor_data,
642
- **mm_processor_kwargs,
643
- return_tensors="pt",
644
- )
645
- except Exception as exc:
646
- data = dict(text=prompt, **processor_data)
647
-
648
- raise RuntimeError(
649
- f"Failed to apply {type(hf_processor).__name__} "
650
- f"on data={data} with kwargs={mm_processor_kwargs}") from exc
651
-
700
hf_inputs.update(passthrough_data)
701
702
return hf_inputs
@@ -730,14 +778,13 @@ class BaseMultiModalProcessor(ABC):
778
3. Extract information about the placeholder tokens from the
779
processed token IDs.
780
"""
733
- tokenizer = self._get_tokenizer()
781
+ mm_items = MultiModalDataItems.from_dict(mm_data)
782
735
- hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
783
+ hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
784
mm_processor_kwargs)
785
prompt_ids, = hf_inputs.pop("input_ids").tolist()
786
mm_kwargs = MultiModalKwargs(hf_inputs)
787
740
- mm_items = to_multi_format(mm_data)
788
prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
789
mm_processor_kwargs)
790
all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
@@ -749,6 +796,7 @@ class BaseMultiModalProcessor(ABC):
796
prompt_ids, mm_item_counts)
797
798
if all_placeholders:
799
+ tokenizer = self._get_tokenizer()
800
prompt_text = _decode(tokenizer, prompt_ids)
801
else:
802
(
vllm/utils.py
+23
-7
@@ -775,7 +775,7 @@ def get_dtype_size(dtype: torch.dtype) -> int:
775
# `collections` helpers
776
def is_list_of(
777
value: object,
778
- typ: Type[T],
778
+ typ: Union[type[T], tuple[type[T], ...]],
779
*,
780
check: Literal["first", "all"] = "first",
781
) -> TypeIs[List[T]]:
@@ -1282,6 +1282,7 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1282
def supports_kw(
1283
callable: Callable[..., object],
1284
kw_name: str,
1285
+ *,
1286
requires_kw_only: bool = False,
1287
allow_var_kwargs: bool = True,
1288
) -> bool:
@@ -1326,6 +1327,8 @@ def resolve_mm_processor_kwargs(
1327
init_kwargs: Optional[Mapping[str, object]],
1328
inference_kwargs: Optional[Mapping[str, object]],
1329
callable: Callable[..., object],
1330
+ *,
1331
+ requires_kw_only: bool = True,
1332
allow_var_kwargs: bool = False,
1333
) -> Dict[str, Any]:
1334
"""Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
@@ -1344,11 +1347,17 @@ def resolve_mm_processor_kwargs(
1347
runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
1348
callable,
1349
overrides=inference_kwargs,
1347
- allow_var_kwargs=allow_var_kwargs)
1350
+ requires_kw_only=requires_kw_only,
1351
+ allow_var_kwargs=allow_var_kwargs,
1352
+ )
1353
1354
# Filter init time multimodal processor kwargs provided
1355
init_mm_kwargs = get_allowed_kwarg_only_overrides(
1351
- callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs)
1356
+ callable,
1357
+ overrides=init_kwargs,
1358
+ requires_kw_only=requires_kw_only,
1359
+ allow_var_kwargs=allow_var_kwargs,
1360
+ )
1361
1362
# Merge the final processor kwargs, prioritizing inference
1363
# time values over the initialization time values.
@@ -1359,6 +1368,8 @@ def resolve_mm_processor_kwargs(
1368
def get_allowed_kwarg_only_overrides(
1369
callable: Callable[..., object],
1370
overrides: Optional[Mapping[str, object]],
1371
+ *,
1372
+ requires_kw_only: bool = True,
1373
allow_var_kwargs: bool = False,
1374
) -> Dict[str, Any]:
1375
"""
@@ -1390,16 +1401,21 @@ def get_allowed_kwarg_only_overrides(
1401
for kwarg_name, val in overrides.items()
1402
if supports_kw(callable,
1403
kwarg_name,
1393
- requires_kw_only=True,
1404
+ requires_kw_only=requires_kw_only,
1405
allow_var_kwargs=allow_var_kwargs)
1406
}
1407
1408
# If anything is dropped, log a warning
1409
dropped_keys = overrides.keys() - filtered_overrides.keys()
1410
if dropped_keys:
1400
- logger.warning(
1401
- "The following intended overrides are not keyword-only args "
1402
- "and and will be dropped: %s", dropped_keys)
1411
+ if requires_kw_only:
1412
+ logger.warning(
1413
+ "The following intended overrides are not keyword-only args "
1414
+ "and and will be dropped: %s", dropped_keys)
1415
+ else:
1416
+ logger.warning(
1417
+ "The following intended overrides are not keyword args "
1418
+ "and and will be dropped: %s", dropped_keys)
1419
1420
return filtered_overrides
1421