@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6142ef0ada
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[VLM] Merged multimodal processor for Qwen2-Audio (#11303)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 19, 2024 at 14:14 UTC 6142ef0adafd76f6b33ff0adb9a097e45a5df279
11 files changed +416 -360
examples/offline_inference_audio_language.py
+6
@@ -18,6 +18,10 @@ question_per_audio_count = {
18 2: "What sport and what nursery rhyme are referenced?"
19 }
20
21 +# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
22 +# lower-end GPUs.
23 +# Unless specified, these settings have been tested to work on a single L4.
24 +
25
26 # Ultravox 0.3
27 def run_ultravox(question: str, audio_count: int):
@@ -33,6 +37,8 @@ def run_ultravox(question: str, audio_count: int):
37 add_generation_prompt=True)
38
39 llm = LLM(model=model_name,
40 + max_model_len=4096,
41 + max_num_seqs=5,
42 trust_remote_code=True,
43 limit_mm_per_prompt={"audio": audio_count})
44 stop_token_ids = None
tests/models/decoder_only/audio_language/test_ultravox.py
+4 -5
@@ -5,6 +5,7 @@ import pytest
5 import pytest_asyncio
6 from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8 +from vllm.multimodal.audio import resample_audio
9 from vllm.sequence import SampleLogprobs
10 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11
@@ -130,16 +131,14 @@ def run_test(
131 dtype=dtype,
132 postprocess_inputs=process,
133 auto_cls=AutoModel) as hf_model:
133 - import librosa
134 -
134 hf_outputs_per_audio = [
135 hf_model.generate_greedy_logprobs_limit(
136 [hf_prompt],
137 max_tokens,
138 num_logprobs=num_logprobs,
140 - audios=[(librosa.resample(audio[0],
141 - orig_sr=audio[1],
142 - target_sr=16000), 16000)])
139 + audios=[(resample_audio(audio[0],
140 + orig_sr=audio[1],
141 + target_sr=16000), 16000)])
142 for _, hf_prompt, audio in prompts_and_audios
143 ]
144
vllm/inputs/registry.py
+70 -30
@@ -1,11 +1,11 @@
1 import functools
2 from collections import UserDict
3 from dataclasses import dataclass
4 -from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, NamedTuple,
5 - Optional, Protocol, Type)
4 +from typing import (TYPE_CHECKING, Any, Callable, Mapping, NamedTuple,
5 + Optional, Protocol, Union)
6
7 from torch import nn
8 -from transformers import PretrainedConfig, ProcessorMixin
8 +from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
9 from typing_extensions import TypeVar, assert_never
10
11 from vllm.logger import init_logger
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
26 logger = init_logger(__name__)
27
28 C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
29 +P = TypeVar("P", bound=ProcessorMixin, default=ProcessorMixin)
30
31
32 @dataclass(frozen=True)
@@ -38,24 +39,28 @@ class InputContext:
39 model_config: "ModelConfig"
40 """The configuration of the model."""
41
41 - def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
42 + def get_hf_config(
43 + self,
44 + typ: Union[type[C], tuple[type[C], ...]] = PretrainedConfig,
45 + /,
46 + ) -> C:
47 """
48 Get the HuggingFace configuration
49 (:class:`transformers.PretrainedConfig`) of the model,
50 additionally checking its type.
51
52 Raises:
48 - TypeError: If the model is not of the specified type.
53 + TypeError: If the configuration is not of the specified type.
54 """
55 hf_config = self.model_config.hf_config
51 - if not isinstance(hf_config, hf_config_type):
56 + if not isinstance(hf_config, typ):
57 raise TypeError("Invalid type of HuggingFace config. "
53 - f"Expected type: {hf_config_type}, but "
58 + f"Expected type: {typ}, but "
59 f"found type: {type(hf_config)}")
60
61 return hf_config
62
58 - def get_hf_image_processor_config(self) -> Dict[str, Any]:
63 + def get_hf_image_processor_config(self) -> dict[str, Any]:
64 """
65 Get the HuggingFace image processor configuration of the model.
66 """
@@ -74,18 +79,37 @@ class InputContext:
79
80 return mm_config
81
77 - def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
82 + def get_hf_processor(
83 + self,
84 + typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
85 + /,
86 + **kwargs: object,
87 + ) -> P:
88 + """
89 + Get the HuggingFace processor
90 + (:class:`transformers.ProcessorMixin`) of the model,
91 + additionally checking its type.
92 +
93 + Raises:
94 + TypeError: If the processor is not of the specified type.
95 + """
96 base_kwargs = self.model_config.mm_processor_kwargs
97 if base_kwargs is None:
98 base_kwargs = {}
99
100 merged_kwargs = {**base_kwargs, **kwargs}
101
84 - return cached_get_processor(
102 + hf_processor = cached_get_processor(
103 self.model_config.model,
104 trust_remote_code=self.model_config.trust_remote_code,
105 **merged_kwargs,
106 )
107 + if not isinstance(hf_processor, typ):
108 + raise TypeError("Invalid type of HuggingFace processor. "
109 + f"Expected type: {typ}, but "
110 + f"found type: {type(hf_processor)}")
111 +
112 + return hf_processor
113
114
115 @dataclass(frozen=True)
@@ -93,39 +117,55 @@ class InputProcessingContext(InputContext):
117 tokenizer: AnyTokenizer
118 """The tokenizer used to tokenize the inputs."""
119
96 - def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
97 - base_kwargs = self.model_config.mm_processor_kwargs
98 - if base_kwargs is None:
99 - base_kwargs = {}
100 -
101 - merged_kwargs = {**base_kwargs, **kwargs}
102 -
103 - return cached_get_processor(
104 - self.model_config.model,
105 - tokenizer=self.tokenizer, # Override the tokenizer with ours
106 - trust_remote_code=self.model_config.trust_remote_code,
107 - **merged_kwargs,
120 + def get_hf_processor(
121 + self,
122 + typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
123 + /,
124 + **kwargs: object,
125 + ) -> P:
126 + return super().get_hf_processor(
127 + typ,
128 + tokenizer=self.tokenizer,
129 + **kwargs,
130 )
131
110 - def resolve_hf_processor_call_kwargs(
132 + def call_hf_processor(
133 self,
134 hf_processor: ProcessorMixin,
135 + prompt: str,
136 + processor_data: Mapping[str, object],
137 inference_kwargs: Mapping[str, object],
114 - ) -> Mapping[str, object]:
138 + ) -> BatchFeature:
139 assert callable(hf_processor)
140
141 base_kwargs = self.model_config.mm_processor_kwargs
142 if base_kwargs is None:
143 base_kwargs = {}
144
121 - return resolve_mm_processor_kwargs(
145 + merged_kwargs = resolve_mm_processor_kwargs(
146 base_kwargs,
147 inference_kwargs,
148 hf_processor,
149 + requires_kw_only=False,
150 + allow_var_kwargs=True,
151 )
152
153 + try:
154 + return hf_processor(
155 + text=prompt,
156 + **processor_data,
157 + **merged_kwargs,
158 + return_tensors="pt",
159 + )
160 + except Exception as exc:
161 + data = dict(text=prompt, **processor_data)
162 + msg = (f"Failed to apply {type(hf_processor).__name__} "
163 + f"on data={data} with kwargs={merged_kwargs}")
164 +
165 + raise RuntimeError(msg) from exc
166 +
167
128 -N = TypeVar("N", bound=Type[nn.Module])
168 +N = TypeVar("N", bound=type[nn.Module])
169
170
171 class DummyData(NamedTuple):
@@ -232,7 +272,7 @@ class InputRegistry:
272
273 return wrapper
274
235 - def _get_dummy_data_factory(self, model_cls: Type[nn.Module]):
275 + def _get_dummy_data_factory(self, model_cls: type[nn.Module]):
276 return self._dummy_factories_by_model_type \
277 .get(model_cls, self._default_dummy_data_factory)
278
@@ -257,7 +297,7 @@ class InputRegistry:
297
298 return wrapper
299
260 - def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
300 + def _get_dummy_encoder_data_factory(self, model_cls: type[nn.Module]):
301 return self._dummy_encoder_factories_by_model_type \
302 .get(model_cls, self._default_dummy_data_factory)
303
@@ -368,14 +408,14 @@ class InputRegistry:
408
409 return wrapper
410
371 - def _get_model_input_processor(self, model_cls: Type[nn.Module]):
411 + def _get_model_input_processor(self, model_cls: type[nn.Module]):
412 return self._input_processors_by_model_type \
413 .get(model_cls, self._default_input_processor)
414
415 def _ensure_mm_kwargs(
416 self,
417 inputs: SingletonInputs,
378 - mm_processor_kwargs: Dict[str, Any],
418 + mm_processor_kwargs: dict[str, Any],
419 ):
420 if inputs["type"] == "token":
421 # In case the input processor for that model fails to set it
vllm/model_executor/models/llava.py
+2 -2
@@ -133,8 +133,8 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
133 hf_processor.__is_patched__ = True # type: ignore
134
135 def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136 - hf_processor = self.ctx.get_hf_processor()
137 - assert isinstance(hf_processor, (LlavaProcessor, PixtralProcessor))
136 + hf_processor = self.ctx.get_hf_processor(
137 + (LlavaProcessor, PixtralProcessor))
138
139 if isinstance(hf_processor, PixtralProcessor):
140 self._patch_pixtral_processor(hf_processor)
vllm/model_executor/models/phi3v.py
+11 -5
@@ -34,7 +34,6 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 from vllm.multimodal.inputs import NestedTensors
36 from vllm.multimodal.processing import (BaseMultiModalProcessor,
37 - MultiModalDataDict,
37 MultiModalDataItems, ProcessorInputs,
38 PromptReplacement)
39 from vllm.sequence import IntermediateTensors
@@ -330,20 +329,27 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
329 return self.ctx.get_hf_processor(num_crops=num_crops)
330 return self.ctx.get_hf_processor()
331
333 - def _apply_hf_processor(
332 + def _call_hf_processor(
333 self,
334 + hf_processor: ProcessorMixin,
335 prompt: str,
336 - mm_data: MultiModalDataDict,
336 + processor_data: Mapping[str, object],
337 mm_processor_kwargs: Mapping[str, object],
338 ) -> BatchFeature:
339 - processed_outputs = super()._apply_hf_processor(
340 - prompt, mm_data, mm_processor_kwargs)
339 + processed_outputs = super()._call_hf_processor(
340 + hf_processor,
341 + prompt=prompt,
342 + processor_data=processor_data,
343 + mm_processor_kwargs=mm_processor_kwargs,
344 + )
345 +
346 # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
347 # which will cause OverflowError when decoding the prompt_ids.
348 # Therefore, we need to do an early replacement here
349 token_ids = processed_outputs['input_ids']
350 token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
351 processed_outputs['input_ids'] = token_ids
352 +
353 return processed_outputs
354
355 def _get_prompt_replacements(
vllm/model_executor/models/qwen2_audio.py
+113 -190
@@ -19,45 +19,43 @@
19 # See the License for the specific language governing permissions and
20 # limitations under the License.
21 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22 -from functools import cached_property, lru_cache
23 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 - Union)
22 +from functools import cached_property
23 +from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24 + TypedDict, Union)
25
26 -import librosa
26 import numpy as np
27 import torch
28 import torch.nn as nn
30 -from transformers import Qwen2AudioEncoder
29 +from transformers import BatchFeature, ProcessorMixin
30 +from transformers.models.qwen2_audio import (Qwen2AudioConfig,
31 + Qwen2AudioEncoder,
32 + Qwen2AudioProcessor)
33 +from transformers.models.whisper import WhisperFeatureExtractor
34
35 from vllm.attention import AttentionMetadata
36 from vllm.config import VllmConfig
34 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35 - InputContext, token_inputs)
36 -from vllm.logger import init_logger
37 +from vllm.inputs import InputContext
38 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40 +from vllm.multimodal import MULTIMODAL_REGISTRY
41 from vllm.multimodal.inputs import NestedTensors
41 -from vllm.multimodal.utils import consecutive_placeholder_ranges
42 -from vllm.sequence import IntermediateTensors, SequenceData
42 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 + MultiModalDataItems, ProcessorInputs,
44 + PromptReplacement)
45 +from vllm.sequence import IntermediateTensors
46
47 from .interfaces import SupportsMultiModal, SupportsPP
48 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
49 maybe_prefix, merge_multimodal_embeddings)
50
48 -logger = init_logger(__name__)
49 -
51
52 # # === Audio Inputs === #
53 class Qwen2AudioInputs(TypedDict):
54 input_features: torch.Tensor
54 - """Shape:
55 - `(num_audios, num_mel_bins, 3000)`
56 - """
55 + """Shape: `(num_audios, num_mel_bins, 3000)`"""
56
57 feature_attention_mask: torch.Tensor
59 - """Shape: `(num_audios, 3000)`
60 - """
58 + """Shape: `(num_audios, 3000)`"""
59
60
61 # === Audio Encoder === #
@@ -74,187 +72,114 @@ class Qwen2AudioMultiModalProjector(nn.Module):
72 return hidden_states
73
74
77 -def dummy_data_for_qwen2_audio(ctx: InputContext, seq_len: int,
78 - mm_counts: Mapping[str, int]):
79 - num_audios = mm_counts["audio"]
80 - max_tokens_per_audio = get_max_qwen2_audio_audio_tokens(ctx)
81 - max_llm_audio_tokens = max_tokens_per_audio * num_audios
82 - if seq_len - max_llm_audio_tokens - 2 < 0:
83 - raise RuntimeError(
84 - f"Qwen2-Audio cannot process {num_audios} audios in a prompt, "
85 - "please increase max_model_len or reduce audio limit by "
86 - "--limit-mm-per-prompt.")
87 -
88 - audio_token_index = ctx.model_config.hf_config.audio_token_index
89 -
90 - dummy_seqdata = SequenceData.from_prompt_token_counts(
91 - (audio_token_index, max_llm_audio_tokens),
92 - (0, seq_len - max_llm_audio_tokens),
93 - )
94 - dummy_audio = np.full((max_llm_audio_tokens * 2 * 2 * 160, ), 0.)
95 - return DummyData(
96 - dummy_seqdata, {"audio": [(dummy_audio, 16000)] * num_audios}, {
97 - "audio":
98 - consecutive_placeholder_ranges(num_items=num_audios,
99 - item_size=max_tokens_per_audio)
100 - })
101 -
102 -
103 -def get_processor(
104 - processor_name: str,
105 - *args,
106 - trust_remote_code: bool = False,
107 - **kwargs,
108 -):
109 - """Gets a processor for the given model name via HuggingFace.
110 -
111 - Derived from `vllm.transformers_utils.image_processor.get_image_processor`.
112 - """
113 - # don't put this import at the top level
114 - # it will call torch.cuda.device_count()
115 - from transformers import AutoProcessor
116 -
117 - try:
118 - processor = AutoProcessor.from_pretrained(
119 - processor_name,
120 - *args,
121 - trust_remote_code=trust_remote_code,
122 - **kwargs)
123 - except ValueError as e:
124 - # If the error pertains to the processor class not existing or not
125 - # currently being imported, suggest using the --trust-remote-code flag.
126 - # Unlike AutoTokenizer, AutoProcessor does not separate such errors
127 - if not trust_remote_code:
128 - err_msg = (
129 - "Failed to load the processor. If the processor is "
130 - "a custom processor not yet available in the HuggingFace "
131 - "transformers library, consider setting "
132 - "`trust_remote_code=True` in LLM or using the "
133 - "`--trust-remote-code` flag in the CLI.")
134 - raise RuntimeError(err_msg) from e
135 - else:
136 - raise e
75 +# From Qwen2AudioEncoder._get_feat_extract_output_lengths
76 +def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
77 + feat_lengths = (input_lengths - 1) // 2 + 1
78 + output_lengths = (feat_lengths - 2) // 2 + 1
79 + return feat_lengths, output_lengths
80
138 - return processor
81
82 +def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
83 + hf_config = ctx.get_hf_config(Qwen2AudioConfig)
84 + max_source_position = hf_config.audio_config.max_source_positions
85 + output_lengths = (max_source_position - 2) // 2 + 1
86 + return output_lengths
87
141 -cached_get_processor = lru_cache(get_processor)
88
89 +class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
90
144 -def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
145 - """
146 - Computes the output length of the convolutional layers
147 - and the output length of the audio encoder
148 - """
149 - input_lengths = (input_lengths - 1) // 2 + 1
150 - output_lengths = (input_lengths - 2) // 2 + 1
151 - return input_lengths, output_lengths
91 + def _get_hf_processor(self) -> Qwen2AudioProcessor:
92 + return self.ctx.get_hf_processor(Qwen2AudioProcessor)
93
94 + def _get_feature_extractor(self) -> WhisperFeatureExtractor:
95 + return self._get_hf_processor().feature_extractor # type: ignore
96
154 -def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
155 - max_source_position = (
156 - ctx.model_config.hf_config.audio_config.max_source_positions)
157 - output_lengths = (max_source_position - 2) // 2 + 1
158 - return output_lengths
97 + def _get_processor_data(
98 + self,
99 + mm_items: MultiModalDataItems,
100 + ) -> tuple[dict[str, Any], dict[str, Any]]:
101 + # resample audio to the model's sampling rate
102 + feature_extractor = self._get_feature_extractor()
103 + mm_items.resample_audios(feature_extractor.sampling_rate)
104
105 + return super()._get_processor_data(mm_items)
106
161 -def input_processor_for_qwen2_audio(
162 - ctx: InputContext, inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
163 - multi_modal_data = inputs.get("multi_modal_data")
164 - if multi_modal_data is None or "audio" not in multi_modal_data:
165 - return inputs
166 -
167 - audios = multi_modal_data["audio"]
168 - if not isinstance(audios, list):
169 - audios = [audios]
170 -
171 - if len(audios) == 0:
172 - return inputs
173 -
174 - processor = cached_get_processor(ctx.model_config.model)
175 - resampled_audios = [
176 - librosa.resample(audio,
177 - orig_sr=sampling_rate,
178 - target_sr=processor.feature_extractor.sampling_rate)
179 - for audio, sampling_rate in audios
180 - ]
181 - audio_input_lengths = np.array(
182 - [min(3000, _.shape[0] // 160 + 1) for _ in resampled_audios])
183 -
184 - audio_feat_lengths, audio_output_lengths = _get_feat_extract_output_lengths(
185 - audio_input_lengths)
186 -
187 - audio_token_index = ctx.model_config.hf_config.audio_token_index
188 -
189 - input_ids = inputs['prompt_token_ids']
190 -
191 - new_input_ids = []
192 - audio_num = input_ids.count(audio_token_index)
193 - assert len(audio_input_lengths) == audio_num, \
194 - (f'The text input contains {audio_num} audio tokens, '
195 - f'but {len(audio_input_lengths)} audios provided')
196 - start = 0
197 - for audio_idx in range(audio_num):
198 - end = input_ids.index(audio_token_index, start)
199 - new_input_ids.extend(input_ids[start:end]) # text part
200 -
201 - new_input_ids.extend([audio_token_index] *
202 - audio_output_lengths[audio_idx])
203 - start = end + 1
204 - new_input_ids.extend(input_ids[start:])
205 -
206 - return token_inputs(
207 - prompt_token_ids=new_input_ids,
208 - prompt=inputs.get("prompt"),
209 - multi_modal_data=multi_modal_data,
210 - )
211 -
212 -
213 -def input_mapper_for_qwen2_audio(
214 - ctx: InputContext,
215 - multi_modal_data: Union[np.ndarray, List[np.ndarray]],
216 -) -> MultiModalKwargs:
217 - """Input mapper for Qwen2-Audio."""
218 - if not isinstance(multi_modal_data, list):
219 - multi_modal_data = [multi_modal_data]
220 -
221 - if len(multi_modal_data) == 0:
222 - return MultiModalKwargs()
223 -
224 - processor = cached_get_processor(ctx.model_config.model)
225 - audio_feature_extractor = processor.feature_extractor
226 - if audio_feature_extractor is None:
227 - raise RuntimeError(
228 - "No HuggingFace audio_feature_extractor is available "
229 - "to process the audio object")
230 -
231 - try:
232 - resampled_audios = [
233 - librosa.resample(
234 - audio,
235 - orig_sr=sampling_rate,
236 - target_sr=processor.feature_extractor.sampling_rate)
237 - for audio, sampling_rate in multi_modal_data
107 + def _call_hf_processor(
108 + self,
109 + hf_processor: ProcessorMixin,
110 + prompt: str,
111 + processor_data: Mapping[str, object],
112 + mm_processor_kwargs: Mapping[str, object],
113 + ) -> BatchFeature:
114 + processor_data = dict(processor_data)
115 + audios = processor_data.pop("audios", [])
116 +
117 + if audios:
118 + processor_data["audios"] = audios
119 +
120 + feature_extractor = self._get_feature_extractor()
121 + mm_processor_kwargs = dict(
122 + **mm_processor_kwargs,
123 + sampling_rate=feature_extractor.sampling_rate,
124 + )
125 + else:
126 + # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
127 + pass
128 +
129 + return super()._call_hf_processor(
130 + hf_processor,
131 + prompt=prompt,
132 + processor_data=processor_data,
133 + mm_processor_kwargs=mm_processor_kwargs,
134 + )
135 +
136 + def _get_prompt_replacements(
137 + self,
138 + mm_items: MultiModalDataItems,
139 + hf_inputs: BatchFeature,
140 + mm_processor_kwargs: Mapping[str, object],
141 + ) -> list[PromptReplacement]:
142 + hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
143 + placeholder = hf_config.audio_token_index
144 +
145 + feature_attention_mask = hf_inputs.get("feature_attention_mask")
146 + if feature_attention_mask is None:
147 + audio_output_lengths = []
148 + else:
149 + _, audio_output_lengths = _get_feat_extract_output_lengths(
150 + feature_attention_mask.sum(-1))
151 +
152 + def get_replacement_qwen2_audio(item_idx: int):
153 + return [placeholder] * audio_output_lengths[item_idx]
154 +
155 + return [
156 + PromptReplacement(
157 + modality="audio",
158 + target=[placeholder],
159 + replacement=get_replacement_qwen2_audio,
160 + )
161 ]
239 - batch_data = audio_feature_extractor(resampled_audios,
240 - sampling_rate=16000,
241 - return_attention_mask=True,
242 - padding="max_length",
243 - return_tensors="pt").data
244 - batch_data["feature_attention_mask"] = batch_data.pop("attention_mask")
245 - except Exception:
246 - logger.error("Failed to process audio (%s)", multi_modal_data)
247 - raise
248 -
249 - return MultiModalKwargs(batch_data)
250 -
251 -
252 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_audio)
253 -@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_audio)
254 -@MULTIMODAL_REGISTRY.register_input_mapper("audio",
255 - input_mapper_for_qwen2_audio)
162 +
163 + def _get_dummy_mm_inputs(
164 + self,
165 + mm_counts: Mapping[str, int],
166 + ) -> ProcessorInputs:
167 + audio_len = get_max_qwen2_audio_audio_tokens(self.ctx)
168 +
169 + audio_count = mm_counts["audio"]
170 + audio = np.zeros(audio_len)
171 + data = {"audio": [audio] * audio_count}
172 +
173 + return ProcessorInputs(
174 + prompt_text="<|AUDIO|>" * audio_count,
175 + mm_data=data,
176 + mm_processor_kwargs={},
177 + )
178 +
179 +
180 @MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
181 "audio", get_max_qwen2_audio_audio_tokens)
182 +@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
183 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
184 SupportsPP):
185
@@ -289,9 +214,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
214
215 return get_sampler()
216
292 - def _validate_and_reshape_mm_tensor(self,
293 - mm_input: Union[torch.Tensor,
294 - List[torch.Tensor]],
217 + def _validate_and_reshape_mm_tensor(self, mm_input: object,
218 name: str) -> torch.Tensor:
219 if not isinstance(mm_input, (torch.Tensor, list)):
220 raise ValueError(f"Incorrect type of {name}. "
vllm/model_executor/models/ultravox.py
+56 -57
@@ -3,7 +3,7 @@
3
4 import math
5 from functools import cached_property, lru_cache
6 -from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
6 +from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7 Tuple, TypedDict, Union)
8
9 import numpy as np
@@ -11,7 +11,7 @@ import torch
11 import torch.utils.checkpoint
12 from torch import nn
13 from torch.nn import functional as F
14 -from transformers import BatchFeature
14 +from transformers import BatchFeature, ProcessorMixin
15 from transformers.models.whisper import WhisperFeatureExtractor
16 from transformers.models.whisper.modeling_whisper import WhisperEncoder
17
@@ -25,11 +25,11 @@ from vllm.model_executor.model_loader.loader import DefaultModelLoader
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 - MultiModalDataDict,
28 MultiModalDataItems, ProcessorInputs,
29 PromptReplacement)
30 from vllm.sequence import IntermediateTensors
31 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
32 +from vllm.utils import is_list_of
33
34 from .interfaces import SupportsMultiModal, SupportsPP
35 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -61,8 +61,8 @@ def cached_feature_extractor(model_id: str) -> WhisperFeatureExtractor:
61
62
63 def whisper_feature_extractor(ctx: InputContext) -> WhisperFeatureExtractor:
64 - return cached_feature_extractor(
65 - ctx.get_hf_config(UltravoxConfig).audio_model_id)
64 + hf_config = ctx.get_hf_config(UltravoxConfig)
65 + return cached_feature_extractor(hf_config.audio_model_id)
66
67
68 def get_ultravox_max_audio_tokens(ctx: InputContext):
@@ -73,72 +73,71 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
73 class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
74
75 def _get_feature_extractor(self) -> WhisperFeatureExtractor:
76 - return self._get_hf_processor().audio_processor.feature_extractor
76 + hf_processor = self._get_hf_processor()
77 + return hf_processor.audio_processor.feature_extractor # type: ignore
78
78 - def _resample_audio(
79 + def _get_processor_data(
80 self,
80 - audio: np.ndarray,
81 - sr: int,
82 - ) -> Dict[str, Union[np.ndarray, int]]:
81 + mm_items: MultiModalDataItems,
82 + ) -> tuple[dict[str, Any], dict[str, Any]]:
83 # resample audio to the model's sampling rate
84 feature_extractor = self._get_feature_extractor()
85 - if sr != feature_extractor.sampling_rate:
86 - try:
87 - import librosa
88 - except ImportError as exc:
89 - raise ImportError(
90 - "Please install vllm[audio] for audio support.") from exc
91 - audio = librosa.resample(audio,
92 - orig_sr=sr,
93 - target_sr=feature_extractor.sampling_rate)
94 - sr = feature_extractor.sampling_rate
95 - return {"audio": audio, "sampling_rate": sr}
96 -
97 - def _apply_hf_processor(
85 + mm_items.resample_audios(feature_extractor.sampling_rate)
86 +
87 + return super()._get_processor_data(mm_items)
88 +
89 + def _call_hf_processor(
90 self,
91 + hf_processor: ProcessorMixin,
92 prompt: str,
100 - mm_data: MultiModalDataDict,
93 + processor_data: Mapping[str, object],
94 mm_processor_kwargs: Mapping[str, object],
95 ) -> BatchFeature:
103 - if not mm_data or not mm_data.get("audio", None):
104 - return super()._apply_hf_processor(prompt, mm_data,
105 - mm_processor_kwargs)
96 + processor_data = dict(processor_data)
97 + audios = processor_data.pop("audios", [])
98 +
99 + if not audios:
100 + return super()._call_hf_processor(
101 + hf_processor,
102 + prompt=prompt,
103 + processor_data=processor_data,
104 + mm_processor_kwargs=mm_processor_kwargs,
105 + )
106
107 - audio_data = mm_data["audio"]
108 - if not isinstance(audio_data, list):
109 - audio_data = [audio_data]
107 + feature_extractor = self._get_feature_extractor()
108 + mm_processor_kwargs = dict(
109 + **mm_processor_kwargs,
110 + sampling_rate=feature_extractor.sampling_rate,
111 + )
112 +
113 + # Already resampled by _get_processor_data
114 + assert is_list_of(audios, np.ndarray)
115
116 # Ultravox processor doesn't support multiple inputs,
117 # therefore we need to input text and audio one by one
113 - tokenizer = self._get_tokenizer()
118 audio_features, audio_token_len = [], []
115 - processed_inputs = {}
116 - for audio, sr in audio_data:
117 - data = self._resample_audio(audio, sr)
118 - processed_inputs = super()._apply_hf_processor(
119 - prompt, data, mm_processor_kwargs)
120 - prompt = tokenizer.decode(processed_inputs["input_ids"][0],
121 - skip_special_tokens=False)
122 - audio_features.append(
123 - processed_inputs.pop("audio_values").squeeze(0))
124 - audio_token_len.append(
125 - processed_inputs.pop("audio_token_len").item())
126 -
127 - return dict(
128 - **processed_inputs,
119 + shared_outputs = {}
120 + for audio in audios:
121 + # NOTE: Ultravox processor accepts "audio" instead of "audios"
122 + item_processor_data = dict(**processor_data, audio=audio)
123 +
124 + item_outputs = super()._call_hf_processor(
125 + hf_processor,
126 + prompt=prompt,
127 + processor_data=item_processor_data,
128 + mm_processor_kwargs=mm_processor_kwargs,
129 + )
130 +
131 + audio_features.append(item_outputs.pop("audio_values")[0])
132 + audio_token_len.append(item_outputs.pop("audio_token_len").item())
133 + shared_outputs = item_outputs
134 +
135 + combined_outputs = dict(
136 + **shared_outputs,
137 audio_features=audio_features,
138 audio_token_len=audio_token_len,
139 )
132 -
133 - def _get_processor_data(
134 - self,
135 - mm_data: MultiModalDataDict,
136 - ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
137 - # Ultravox uses "audio" instead of "audios" as calling keyword
138 - processor_data, passthrough_data = super()._get_processor_data(mm_data)
139 - if "audios" in processor_data:
140 - processor_data["audio"] = processor_data.pop("audios")
141 - return processor_data, passthrough_data
140 + return BatchFeature(combined_outputs)
141
142 def _get_prompt_replacements(
143 self,
@@ -147,7 +146,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
146 mm_processor_kwargs: Mapping[str, object],
147 ) -> list[PromptReplacement]:
148 hf_processor = self._get_hf_processor()
150 - placeholder = hf_processor.audio_token_replacement
149 + placeholder = hf_processor.audio_token_replacement # type: ignore
150
151 def get_replacement_ultravox(item_idx: int):
152 audio_token_len = hf_inputs["audio_token_len"][item_idx]
@@ -171,7 +170,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
170
171 audio_count = mm_counts["audio"]
172 audio = np.zeros(audio_len)
174 - data = {"audio": [(audio, sampling_rate)] * audio_count}
173 + data = {"audio": [audio] * audio_count}
174
175 return ProcessorInputs(
176 prompt_text="<|audio|>" * audio_count,
vllm/multimodal/audio.py
+18
@@ -1,3 +1,6 @@
1 +import numpy as np
2 +import numpy.typing as npt
3 +
4 from vllm.inputs.registry import InputContext
5
6 from .base import MultiModalPlugin
@@ -21,3 +24,18 @@ class AudioPlugin(MultiModalPlugin):
24 def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
25 raise NotImplementedError(
26 "There is no default maximum multimodal tokens")
27 +
28 +
29 +def resample_audio(
30 + audio: npt.NDArray[np.floating],
31 + *,
32 + orig_sr: float,
33 + target_sr: float,
34 +) -> npt.NDArray[np.floating]:
35 + try:
36 + import librosa
37 + except ImportError as exc:
38 + msg = "Please install vllm[audio] for audio support."
39 + raise ImportError(msg) from exc
40 +
41 + return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
vllm/multimodal/inputs.py
+13 -12
@@ -15,31 +15,32 @@ _T = TypeVar("_T")
15 # yapf: disable
16 ImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
17 """
18 -A :class:`transformers.image_utils.ImageInput` representing a single image,
19 -which can be passed to a HuggingFace :code:`ImageProcessor`.
18 +A :class:`transformers.image_utils.ImageInput` representing a single image
19 +item, which can be passed to a HuggingFace :code:`ImageProcessor`.
20 """
21
22 VideoItem: TypeAlias = Union[
23 - List[Image],
23 + list[Image],
24 np.ndarray,
25 torch.Tensor,
26 - List[np.ndarray],
27 - List[torch.Tensor],
26 + list[np.ndarray],
27 + list[torch.Tensor],
28 ]
29 """
30 -
31 -A :class:`transformers.image_utils.VideoInput` representing a single video,
32 -which can be passed to a HuggingFace :code:`VideoProcessor`.
30 +A :class:`transformers.image_utils.VideoInput` representing a single video
31 +item, which can be passed to a HuggingFace :code:`VideoProcessor`.
32 """
33
34 AudioItem: TypeAlias = Union[
35 np.ndarray,
37 - List[float],
38 - Tuple[np.ndarray, float], # DEPRECATED: Use mm_processor_kwargs instead
36 + list[float],
37 + # `(audio, sampling_rate)`: If the audio's sampling rate is different
38 + # from that expected by the model, we need to resample it.
39 + tuple[np.ndarray, float],
40 ]
41 """
41 -Represents a single audio that can be inputted to a HuggingFace
42 -:code:`AudioProcessor`.
42 +Represents a single audio
43 +item, which can be passed to a HuggingFace :code:`AudioProcessor`.
44 """
45 # yapf: enable
46
vllm/multimodal/processing.py
+100 -52
@@ -17,6 +17,7 @@ from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18 from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
19
20 +from .audio import resample_audio
21 from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
22 MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
23 VideoItem)
@@ -30,7 +31,7 @@ _PromptSeq = Union[str, list[int]]
31 @dataclass
32 class PromptReplacement:
33 modality: str
33 - """The modality for which the replacement is made"""
34 + """The modality for which the replacement is made."""
35
36 target: _PromptSeq
37 """The text or token sequence to find and replace."""
@@ -211,20 +212,48 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
212 corresponds to a list.
213 """
214
215 + @staticmethod
216 + def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
217 + """
218 + Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
219 + """
220 + multi_data = MultiModalDataItems()
221 +
222 + for k, v in data.items():
223 + # yapf: disable
224 + if k == "video":
225 + # Special case since even a single item can be a list
226 + multi_data[k] = ( # type: ignore[index]
227 + v if is_list_of(v, (list, torch.Tensor)) else [v]
228 + )
229 + elif k in ("image", "audio"):
230 + multi_data[k] = ( # type: ignore[index]
231 + v if isinstance(v, (list, torch.Tensor)) else [v]
232 + )
233 + else:
234 + multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
235 + # yapf: enable
236 +
237 + return multi_data
238 +
239 + # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
240 + # `self.images` doesn't update this dictionary, which may be confusing
241 + # We annotate the getter methods as `Sequence` to prevent others from
242 + # trying to update the list in this way
243 @property
215 - def image(self) -> list[ImageItem]:
216 - return self["image"]
244 + def images(self) -> Sequence[ImageItem]:
245 + return self.get("image", [])
246
247 @property
219 - def video(self) -> list[VideoItem]:
220 - return self["video"]
248 + def videos(self) -> Sequence[VideoItem]:
249 + return self.get("video", [])
250
251 @property
223 - def audio(self) -> list[AudioItem]:
224 - return self["audio"]
252 + def audios(self) -> Sequence[AudioItem]:
253 + return self.get("audio", [])
254
255 def get_image_size(self, item_idx: int) -> ImageSize:
227 - image = self.image[item_idx]
256 + image = self.images[item_idx]
257
258 if isinstance(image, Image):
259 return ImageSize(*image.size)
@@ -234,25 +263,41 @@ class MultiModalDataItems(UserDict[str, list[Any]]):
263
264 assert_never(image)
265
266 + def get_audio_with_sr(
267 + self,
268 + item_idx: int,
269 + *,
270 + default_sr: float,
271 + ) -> tuple[np.ndarray, float]:
272 + audio = self.audios[item_idx]
273 +
274 + if isinstance(audio, tuple):
275 + return audio
276 + if isinstance(audio, list):
277 + return np.array(audio), default_sr
278 + if isinstance(audio, np.ndarray):
279 + return audio, default_sr
280 +
281 + assert_never(audio)
282 +
283 + def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
284 + """
285 + If :code:`drop_sr=True`, the audio items in this dictionary are updated
286 + to be NumPy arrays which implicitly means that their sampling rate is
287 + the same as the model's expected sampling rate; otherwise, they remain
288 + as :code:`(audio, new_sr)` tuples.
289 + """
290 + if not self.audios:
291 + return
292 +
293 + new_audios = []
294 + for item_idx in range(len(self.audios)):
295 + audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
296 + audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
297
238 -def to_multi_format(data: MultiModalDataDict) -> MultiModalDataItems:
239 - """
240 - Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
241 - """
242 - multi_data = MultiModalDataItems()
243 -
244 - for k, v in data.items():
245 - # yapf: disable
246 - if k == "video":
247 - # Special case since even a single item can be a list
248 - multi_data[k] = v if is_list_of(v, list) else [v] # type: ignore[index]
249 - elif k in ("image", "audio"):
250 - multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
251 - else:
252 - multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
253 - # yapf: enable
298 + new_audios.append(audio if drop_sr else (audio, new_sr))
299
255 - return multi_data
300 + self["audio"] = new_audios
301
302
303 class _TokenMatch(NamedTuple):
@@ -596,18 +641,20 @@ class BaseMultiModalProcessor(ABC):
641
642 def _get_processor_data(
643 self,
599 - mm_data: MultiModalDataDict,
600 - ) -> BatchFeature:
644 + mm_items: MultiModalDataItems,
645 + ) -> tuple[dict[str, Any], dict[str, Any]]:
646 processor_data = dict[str, Any]()
647 passthrough_data = dict[str, Any]()
603 - for k, v in mm_data.items():
648 +
649 + for k, v in mm_items.items():
650 # TODO: Make a separate modality for embedding inputs
651 # to avoid confusion
652 if k in ("image", "video", "audio"):
653 if isinstance(v, torch.Tensor) and v.ndim == 3:
654 # Pass through embedding inputs (single)
655 passthrough_data[f"{k}_embeds"] = [v]
610 - elif is_list_of(v, torch.Tensor) and v[0].ndim == 2:
656 + elif (is_list_of(v, torch.Tensor) and len(v) > 0
657 + and v[0].ndim == 2):
658 # Pass through embedding inputs (multi)
659 passthrough_data[f"{k}_embeds"] = v
660 else:
@@ -615,40 +662,41 @@ class BaseMultiModalProcessor(ABC):
662 processor_data[f"{k}s"] = v
663 else:
664 processor_data[k] = v
665 +
666 return processor_data, passthrough_data
667
668 + def _call_hf_processor(
669 + self,
670 + hf_processor: ProcessorMixin,
671 + prompt: str,
672 + processor_data: Mapping[str, object],
673 + mm_processor_kwargs: Mapping[str, object],
674 + ) -> BatchFeature:
675 + return self.ctx.call_hf_processor(
676 + hf_processor,
677 + prompt,
678 + processor_data,
679 + mm_processor_kwargs,
680 + )
681 +
682 def _apply_hf_processor(
683 self,
684 prompt: str,
623 - mm_data: MultiModalDataDict,
685 + mm_items: MultiModalDataItems,
686 mm_processor_kwargs: Mapping[str, object],
687 ) -> BatchFeature:
688 # some mm_processor_kwargs may be used in processor initialization
689 # instead of processor call
690 hf_processor = self._get_hf_processor(**mm_processor_kwargs)
691
630 - processor_data, passthrough_data = self._get_processor_data(mm_data)
692 + processor_data, passthrough_data = self._get_processor_data(mm_items)
693
632 - assert callable(hf_processor)
633 - mm_processor_kwargs = self.ctx.resolve_hf_processor_call_kwargs(
694 + hf_inputs = self._call_hf_processor(
695 hf_processor,
635 - mm_processor_kwargs,
696 + prompt=prompt,
697 + processor_data=processor_data,
698 + mm_processor_kwargs=mm_processor_kwargs,
699 )
637 -
638 - try:
639 - hf_inputs = hf_processor(
640 - text=prompt, # type: ignore
641 - **processor_data,
642 - **mm_processor_kwargs,
643 - return_tensors="pt",
644 - )
645 - except Exception as exc:
646 - data = dict(text=prompt, **processor_data)
647 -
648 - raise RuntimeError(
649 - f"Failed to apply {type(hf_processor).__name__} "
650 - f"on data={data} with kwargs={mm_processor_kwargs}") from exc
651 -
700 hf_inputs.update(passthrough_data)
701
702 return hf_inputs
@@ -730,14 +778,13 @@ class BaseMultiModalProcessor(ABC):
778 3. Extract information about the placeholder tokens from the
779 processed token IDs.
780 """
733 - tokenizer = self._get_tokenizer()
781 + mm_items = MultiModalDataItems.from_dict(mm_data)
782
735 - hf_inputs = self._apply_hf_processor(prompt_text, mm_data,
783 + hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
784 mm_processor_kwargs)
785 prompt_ids, = hf_inputs.pop("input_ids").tolist()
786 mm_kwargs = MultiModalKwargs(hf_inputs)
787
740 - mm_items = to_multi_format(mm_data)
788 prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
789 mm_processor_kwargs)
790 all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
@@ -749,6 +796,7 @@ class BaseMultiModalProcessor(ABC):
796 prompt_ids, mm_item_counts)
797
798 if all_placeholders:
799 + tokenizer = self._get_tokenizer()
800 prompt_text = _decode(tokenizer, prompt_ids)
801 else:
802 (
vllm/utils.py
+23 -7
@@ -775,7 +775,7 @@ def get_dtype_size(dtype: torch.dtype) -> int:
775 # `collections` helpers
776 def is_list_of(
777 value: object,
778 - typ: Type[T],
778 + typ: Union[type[T], tuple[type[T], ...]],
779 *,
780 check: Literal["first", "all"] = "first",
781 ) -> TypeIs[List[T]]:
@@ -1282,6 +1282,7 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1282 def supports_kw(
1283 callable: Callable[..., object],
1284 kw_name: str,
1285 + *,
1286 requires_kw_only: bool = False,
1287 allow_var_kwargs: bool = True,
1288 ) -> bool:
@@ -1326,6 +1327,8 @@ def resolve_mm_processor_kwargs(
1327 init_kwargs: Optional[Mapping[str, object]],
1328 inference_kwargs: Optional[Mapping[str, object]],
1329 callable: Callable[..., object],
1330 + *,
1331 + requires_kw_only: bool = True,
1332 allow_var_kwargs: bool = False,
1333 ) -> Dict[str, Any]:
1334 """Applies filtering to eliminate invalid mm_processor_kwargs, i.e.,
@@ -1344,11 +1347,17 @@ def resolve_mm_processor_kwargs(
1347 runtime_mm_kwargs = get_allowed_kwarg_only_overrides(
1348 callable,
1349 overrides=inference_kwargs,
1347 - allow_var_kwargs=allow_var_kwargs)
1350 + requires_kw_only=requires_kw_only,
1351 + allow_var_kwargs=allow_var_kwargs,
1352 + )
1353
1354 # Filter init time multimodal processor kwargs provided
1355 init_mm_kwargs = get_allowed_kwarg_only_overrides(
1351 - callable, overrides=init_kwargs, allow_var_kwargs=allow_var_kwargs)
1356 + callable,
1357 + overrides=init_kwargs,
1358 + requires_kw_only=requires_kw_only,
1359 + allow_var_kwargs=allow_var_kwargs,
1360 + )
1361
1362 # Merge the final processor kwargs, prioritizing inference
1363 # time values over the initialization time values.
@@ -1359,6 +1368,8 @@ def resolve_mm_processor_kwargs(
1368 def get_allowed_kwarg_only_overrides(
1369 callable: Callable[..., object],
1370 overrides: Optional[Mapping[str, object]],
1371 + *,
1372 + requires_kw_only: bool = True,
1373 allow_var_kwargs: bool = False,
1374 ) -> Dict[str, Any]:
1375 """
@@ -1390,16 +1401,21 @@ def get_allowed_kwarg_only_overrides(
1401 for kwarg_name, val in overrides.items()
1402 if supports_kw(callable,
1403 kwarg_name,
1393 - requires_kw_only=True,
1404 + requires_kw_only=requires_kw_only,
1405 allow_var_kwargs=allow_var_kwargs)
1406 }
1407
1408 # If anything is dropped, log a warning
1409 dropped_keys = overrides.keys() - filtered_overrides.keys()
1410 if dropped_keys:
1400 - logger.warning(
1401 - "The following intended overrides are not keyword-only args "
1402 - "and and will be dropped: %s", dropped_keys)
1411 + if requires_kw_only:
1412 + logger.warning(
1413 + "The following intended overrides are not keyword-only args "
1414 + "and and will be dropped: %s", dropped_keys)
1415 + else:
1416 + logger.warning(
1417 + "The following intended overrides are not keyword args "
1418 + "and and will be dropped: %s", dropped_keys)
1419
1420 return filtered_overrides
1421