@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 630eb5b5ce
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix multi-modal processors for transformers 4.48 (#12187)

Cyrus Leung committed Jan 19, 2025 at 11:16 UTC 630eb5b5ce6ea59b6480440b7f6064be5ca71ae1
6 files changed +199 -36
vllm/model_executor/models/llava.py
+24 -1
@@ -5,9 +5,11 @@ from typing import (Final, Iterable, List, Literal, Mapping, Optional,
5
6 import torch
7 import torch.nn as nn
8 +from packaging.version import Version
9 from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
10 PixtralVisionConfig, PretrainedConfig,
11 SiglipVisionConfig)
12 +from transformers import __version__ as TRANSFORMERS_VERSION
13 from transformers.models.llava import LlavaProcessor
14 from transformers.models.pixtral import PixtralProcessor
15
@@ -716,6 +718,27 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
718 return loader.load_weights(weights)
719
720
721 +class MantisProcessingInfo(LlavaProcessingInfo):
722 +
723 + def get_hf_processor(self):
724 + hf_config = self.get_hf_config()
725 + vision_info = self.get_vision_encoder_info()
726 +
727 + if Version(TRANSFORMERS_VERSION) < Version("4.48"):
728 + # BUG: num_additional_image_tokens = 0 but treated as 1,
729 + # so we set vision_feature_select_strategy to None to offset this
730 + vision_feature_select_strategy = None
731 + else:
732 + # FIXED: https://github.com/huggingface/transformers/pull/33424/files#diff-6a37acc21efcadaae622b079b2712a131131448ff64262bd219aa346aeec38faL150
733 + vision_feature_select_strategy = hf_config.vision_feature_select_strategy # noqa: E501
734 +
735 + return self.ctx.get_hf_processor(
736 + LlavaProcessor,
737 + patch_size=vision_info.get_patch_size(),
738 + vision_feature_select_strategy=vision_feature_select_strategy,
739 + )
740 +
741 +
742 class MantisMultiModalProcessor(LlavaMultiModalProcessor):
743
744 def apply(
@@ -794,7 +817,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
817 # To use this model, please use
818 # `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
819 @MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor,
797 - info=LlavaProcessingInfo,
820 + info=MantisProcessingInfo,
821 dummy_inputs=LlavaDummyInputsBuilder)
822 class MantisForConditionalGeneration(LlavaForConditionalGeneration):
823 pass
vllm/model_executor/models/qwen2_audio.py
+49 -23
@@ -36,8 +36,9 @@ from vllm.config import VllmConfig
36 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40 - NestedTensors)
39 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
40 + MultiModalInputsV2, MultiModalKwargs,
41 + NestedTensors, PlaceholderRange)
42 from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
43 MultiModalDataParser)
44 from vllm.multimodal.processing import (BaseMultiModalProcessor,
@@ -153,29 +154,24 @@ class Qwen2AudioMultiModalProcessor(
154 mm_data: Mapping[str, object],
155 mm_kwargs: Mapping[str, Any],
156 ) -> BatchFeature:
156 - mm_data = dict(mm_data)
157 - audios = mm_data.pop("audios", [])
158 -
159 - if audios:
160 - mm_data["audios"] = audios
161 -
162 - feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
163 - mm_kwargs = dict(
164 - **mm_kwargs,
165 - sampling_rate=feature_extractor.sampling_rate,
166 - )
167 - else:
168 - # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
169 - pass
157 + # Text-only input not supported in composite processor
158 + if not mm_data or not mm_data.get("audios", []):
159 + prompt_ids = self.info.get_tokenizer().encode(prompt)
160 + prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
161 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
162 +
163 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
164 + mm_kwargs = dict(
165 + **mm_kwargs,
166 + sampling_rate=feature_extractor.sampling_rate,
167 + )
168
171 - processed_outputs = super()._call_hf_processor(
169 + return super()._call_hf_processor(
170 prompt=prompt,
171 mm_data=mm_data,
172 mm_kwargs=mm_kwargs,
173 )
174
177 - return processed_outputs
178 -
175 def _get_mm_fields_config(
176 self,
177 hf_inputs: BatchFeature,
@@ -192,8 +188,14 @@ class Qwen2AudioMultiModalProcessor(
188 hf_processor_mm_kwargs: Mapping[str, object],
189 out_mm_kwargs: MultiModalKwargs,
190 ) -> list[PromptReplacement]:
195 - hf_config = self.info.get_hf_config()
196 - placeholder = hf_config.audio_token_index
191 + processor = self.info.get_hf_processor()
192 +
193 + # Use getattr with default to be compatible with transformers<4.48
194 + audio_token = getattr(processor, "audio_token", "<|AUDIO|>")
195 + audio_bos_token = getattr(processor, "audio_bos_token",
196 + "<|audio_bos|>")
197 + audio_eos_token = getattr(processor, "audio_eos_token",
198 + "<|audio_eos|>")
199
200 feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
201 if feature_attention_mask is None:
@@ -214,12 +216,16 @@ class Qwen2AudioMultiModalProcessor(
216 f"The audio {audio} (len={len(audio)}) is too short "
217 "to be represented inside the model")
218
217 - return [placeholder] * num_placeholders
219 + return "".join([
220 + audio_bos_token,
221 + audio_token * num_placeholders,
222 + audio_eos_token,
223 + ])
224
225 return [
226 PromptReplacement(
227 modality="audio",
222 - target=[placeholder],
228 + target=audio_token,
229 replacement=get_replacement_qwen2_audio,
230 )
231 ]
@@ -234,6 +240,26 @@ class Qwen2AudioMultiModalProcessor(
240 # tokens than the number of audio items)
241 return not hasattr(self.info.get_hf_processor(), "audio_token")
242
243 + def apply(
244 + self,
245 + prompt: Union[str, list[int]],
246 + mm_data: MultiModalDataDict,
247 + hf_processor_mm_kwargs: Mapping[str, object],
248 + ) -> MultiModalInputsV2:
249 + result = super().apply(prompt, mm_data, hf_processor_mm_kwargs)
250 +
251 + # Only <|AUDIO|> tokens should be considered as placeholders,
252 + # so we ignore the audio_bos_token and audio_eos_token
253 + result["mm_placeholders"] = {
254 + modality: [
255 + PlaceholderRange(offset=p["offset"] + 1,
256 + length=p["length"] - 2) for p in ps
257 + ]
258 + for modality, ps in result["mm_placeholders"].items()
259 + }
260 +
261 + return result
262 +
263
264 @MULTIMODAL_REGISTRY.register_processor(
265 Qwen2AudioMultiModalProcessor,
vllm/model_executor/models/ultravox.py
+1 -8
@@ -137,7 +137,7 @@ class UltravoxMultiModalProcessor(
137 mm_kwargs: Mapping[str, object],
138 ) -> BatchFeature:
139 # Text-only input not supported in composite processor
140 - if not mm_data:
140 + if not mm_data or not mm_data.get("audios", []):
141 prompt_ids = self.info.get_tokenizer().encode(prompt)
142 prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
143 return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -146,13 +146,6 @@ class UltravoxMultiModalProcessor(
146 audios = mm_data.pop("audios", [])
147 assert isinstance(audios, list)
148
149 - if not audios:
150 - return super()._call_hf_processor(
151 - prompt=prompt,
152 - mm_data=mm_data,
153 - mm_kwargs=mm_kwargs,
154 - )
155 -
149 feature_extractor = self.info.get_feature_extractor()
150 mm_kwargs = dict(
151 **mm_kwargs,
vllm/transformers_utils/config.py
+5 -4
@@ -22,10 +22,10 @@ from vllm.envs import VLLM_USE_MODELSCOPE
22 from vllm.logger import init_logger
23 # yapf conflicts with isort for this block
24 # yapf: disable
25 -from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26 - DbrxConfig, DeepseekVLV2Config,
27 - EAGLEConfig, ExaoneConfig,
28 - H2OVLChatConfig,
25 +from vllm.transformers_utils.configs import (AriaConfig, ChatGLMConfig,
26 + Cohere2Config, DbrxConfig,
27 + DeepseekVLV2Config, EAGLEConfig,
28 + ExaoneConfig, H2OVLChatConfig,
29 InternVLChatConfig, JAISConfig,
30 MedusaConfig, MllamaConfig,
31 MLPSpeculatorConfig, MPTConfig,
@@ -52,6 +52,7 @@ _CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
52 }
53
54 _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
55 + "aria": AriaConfig,
56 "chatglm": ChatGLMConfig,
57 "cohere2": Cohere2Config,
58 "dbrx": DbrxConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,3 +1,4 @@
1 +from vllm.transformers_utils.configs.aria import AriaConfig
2 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3 from vllm.transformers_utils.configs.cohere2 import Cohere2Config
4 from vllm.transformers_utils.configs.dbrx import DbrxConfig
@@ -23,6 +24,7 @@ from vllm.transformers_utils.configs.telechat2 import Telechat2Config
24 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
25
26 __all__ = [
27 + "AriaConfig",
28 "ChatGLMConfig",
29 "Cohere2Config",
30 "DbrxConfig",
vllm/transformers_utils/configs/aria.py
+118
@@ -1,7 +1,32 @@
1 +# Copyright 2024 Rhymes AI. All rights reserved.
2 +#
3 +# Licensed to the Apache Software Foundation (ASF) under one
4 +# or more contributor license agreements. See the NOTICE file
5 +# distributed with this work for additional information
6 +# regarding copyright ownership. The ASF licenses this file
7 +# to you under the Apache License, Version 2.0 (the
8 +# "License"); you may not use this file except in compliance
9 +# with the License. You may obtain a copy of the License at
10 +#
11 +# http://www.apache.org/licenses/LICENSE-2.0
12 +#
13 +# Unless required by applicable law or agreed to in writing,
14 +# software distributed under the License is distributed on an
15 +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 +# KIND, either express or implied. See the License for the
17 +# specific language governing permissions and limitations
18 +# under the License.
19 +from typing import Mapping
20 +
21 +from transformers import PretrainedConfig
22 from transformers.models.idefics2.configuration_idefics2 import (
23 Idefics2VisionConfig)
24 from transformers.models.llama.configuration_llama import LlamaConfig
25
26 +from vllm.logger import init_logger
27 +
28 +logger = init_logger(__name__)
29 +
30
31 class AriaVisionConfig(Idefics2VisionConfig):
32 model_type = "aria_vision_model"
@@ -45,3 +70,96 @@ class AriaMoELMConfig(LlamaConfig):
70 self.moe_num_experts = moe_num_experts
71 self.moe_topk = moe_topk
72 self.moe_num_shared_experts = moe_num_shared_experts
73 +
74 +
75 +class AriaConfig(PretrainedConfig):
76 + """
77 + Configuration class for Aria model.
78 + This class handles the configuration for both vision and text components of
79 + the Aria model,
80 + as well as additional parameters for image token handling and projector
81 + mapping.
82 +
83 + Args:
84 + vision_config (AriaVisionConfig or dict): Configuration for the vision
85 + component.
86 + text_config (AriaMoELMConfig or dict): Configuration for the text
87 + component.
88 + projector_patch_to_query_dict (dict): Mapping of patch sizes to query
89 + dimensions.
90 + ignore_index (int): Index to ignore in loss calculation.
91 + image_token_index (int): Index used to represent image tokens.
92 + **kwargs: Additional keyword arguments passed to the parent class.
93 + Attributes:
94 + model_type (str): Type of the model, set to "aria".
95 + is_composition (bool): Whether the model is a composition of multiple
96 + components.
97 + ignore_index (int): Index to ignore in loss calculation.
98 + image_token_index (int): Index used to represent image tokens.
99 + projector_patch_to_query_dict (dict): Mapping of patch sizes to query
100 + dimensions.
101 + vision_config (AriaVisionConfig): Configuration for the vision
102 + component.
103 + text_config (AriaMoELMConfig): Configuration for the text component.
104 + """
105 +
106 + model_type = "aria"
107 + is_composition = False
108 +
109 + def __init__(
110 + self,
111 + vision_config: AriaVisionConfig = AriaVisionConfig(), # noqa: B008
112 + text_config: AriaMoELMConfig = AriaMoELMConfig(), # noqa: B008
113 + projector_patch_to_query_dict: Mapping[int, int] = {
114 + 1225: 128,
115 + 4900: 256,
116 + },
117 + ignore_index=-100,
118 + image_token_index=32000,
119 + tie_word_embeddings=False,
120 + **kwargs,
121 + ):
122 + super().__init__(**kwargs)
123 + self.ignore_index = ignore_index
124 + self.image_token_index = image_token_index
125 + self.tie_word_embeddings = tie_word_embeddings
126 + attn_implementation = kwargs.pop("attn_implementation", None)
127 +
128 + # Set the default attention implementation to flash_attention_2 if not
129 + # specified
130 + self._attn_implementation = ("flash_attention_2"
131 + if attn_implementation is None else
132 + attn_implementation)
133 +
134 + # Convert the keys and values of projector_patch_to_query_dict to
135 + # integers
136 + # This ensures consistency even if they were provided as strings
137 + self.projector_patch_to_query_dict = {
138 + int(k): int(v)
139 + for k, v in projector_patch_to_query_dict.items()
140 + }
141 +
142 + if isinstance(vision_config, dict) and "model_type" in vision_config:
143 + vision_config = AriaVisionConfig(**vision_config)
144 + if attn_implementation is None:
145 + vision_attn_implementation = "flash_attention_2"
146 + elif attn_implementation == "sdpa":
147 + logger.warning("SDPA is not supported for vit, using "
148 + "flash_attention_2 instead")
149 + vision_attn_implementation = "flash_attention_2"
150 + else:
151 + vision_attn_implementation = attn_implementation
152 + vision_config._attn_implementation = vision_attn_implementation
153 +
154 + self.vision_config = vision_config
155 +
156 + if isinstance(text_config, dict) and "model_type" in text_config:
157 + text_attn_implementation = ("sdpa" if attn_implementation is None
158 + else attn_implementation)
159 + text_config = AriaMoELMConfig(**text_config)
160 + text_config._attn_implementation = text_attn_implementation
161 +
162 + self.text_config = text_config
163 +
164 + # This is needed for the static kv cache
165 + self.num_hidden_layers = self.text_config.num_hidden_layers