@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b37d82791e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Upgrade Aria to transformers 4.48 (#12203)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 20, 2025 at 17:58 UTC b37d82791e3c9f7d492db81493d920004de59a26
10 files changed +178 -379
examples/offline_inference/vision_language.py
-3
@@ -26,11 +26,8 @@ def run_aria(question: str, modality: str):
26
27 # NOTE: Need L40 (or equivalent) to avoid OOM
28 llm = LLM(model=model_name,
29 - tokenizer_mode="slow",
30 - dtype="bfloat16",
29 max_model_len=4096,
30 max_num_seqs=2,
33 - trust_remote_code=True,
31 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
32
33 prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
tests/models/decoder_only/vision_language/test_models.py
+2 -5
@@ -10,7 +10,6 @@ from typing import Type
10 import pytest
11 from transformers import AutoModelForVision2Seq
12 from transformers import __version__ as TRANSFORMERS_VERSION
13 -from transformers.utils import is_flash_attn_2_available
13
14 from vllm.platforms import current_platform
15 from vllm.utils import identity
@@ -140,9 +139,7 @@ VLM_TEST_SETTINGS = {
139 #### Extended model tests
140 "aria": VLMTestInfo(
141 models=["rhymes-ai/Aria"],
143 - tokenizer_mode="slow",
142 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
145 - dtype="bfloat16",
143 prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
144 img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
145 max_model_len=4096,
@@ -158,8 +155,8 @@ VLM_TEST_SETTINGS = {
155 max_tokens=64,
156 marks=[
157 pytest.mark.skipif(
161 - not is_flash_attn_2_available(),
162 - reason="Model needs flash-attn for numeric convergence.",
158 + TRANSFORMERS_VERSION < "4.48.0",
159 + reason="HF model requires transformers>=4.48.0",
160 ),
161 large_gpu_mark(min_gb=64),
162 ],
tests/models/multimodal/processing/test_common.py
+5 -7
@@ -11,6 +11,7 @@ from vllm.multimodal.processing import ProcessingCache
11 from vllm.multimodal.utils import cached_get_tokenizer
12
13 from ....multimodal.utils import random_audio, random_image, random_video
14 +from ...registry import HF_EXAMPLE_MODELS
15
16
17 def _test_processing_correctness(
@@ -20,12 +21,9 @@ def _test_processing_correctness(
21 num_batches: int,
22 simplify_rate: float,
23 ):
23 - if model_id == "TIGER-Lab/Mantis-8B-siglip-llama3":
24 - hf_overrides = {"architectures": ["MantisForConditionalGeneration"]}
25 - elif model_id == "deepseek-ai/deepseek-vl2-tiny":
26 - hf_overrides = {"architectures": ["DeepseekVLV2ForCausalLM"]}
27 - else:
28 - hf_overrides = {}
24 + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
25 + model_info.check_available_online(on_fail="skip")
26 + model_info.check_transformers_version(on_fail="skip")
27
28 limit_mm_per_prompt = {
29 modality: 3 if supports_multi else 1
@@ -41,7 +39,7 @@ def _test_processing_correctness(
39 seed=0,
40 dtype="float16",
41 revision=None,
44 - hf_overrides=hf_overrides,
42 + hf_overrides=model_info.hf_overrides,
43 limit_mm_per_prompt=limit_mm_per_prompt,
44 )
45
tests/models/registry.py
+62 -5
@@ -1,5 +1,9 @@
1 from dataclasses import dataclass, field
2 -from typing import AbstractSet, Mapping, Optional
2 +from typing import AbstractSet, Any, Literal, Mapping, Optional
3 +
4 +import pytest
5 +from packaging.version import Version
6 +from transformers import __version__ as TRANSFORMERS_VERSION
7
8
9 @dataclass(frozen=True)
@@ -38,6 +42,50 @@ class _HfExamplesInfo:
42 trust_remote_code: bool = False
43 """The ``trust_remote_code`` level required to load the model."""
44
45 + hf_overrides: dict[str, Any] = field(default_factory=dict)
46 + """The ``hf_overrides`` required to load the model."""
47 +
48 + def check_transformers_version(
49 + self,
50 + *,
51 + on_fail: Literal["error", "skip"],
52 + ) -> None:
53 + """
54 + If the installed transformers version does not meet the requirements,
55 + perform the given action.
56 + """
57 + if self.min_transformers_version is None:
58 + return
59 +
60 + current_version = TRANSFORMERS_VERSION
61 + required_version = self.min_transformers_version
62 + if Version(current_version) < Version(required_version):
63 + msg = (
64 + f"You have `transformers=={current_version}` installed, but "
65 + f"`transformers>={required_version}` is required to run this "
66 + "model")
67 +
68 + if on_fail == "error":
69 + raise RuntimeError(msg)
70 + else:
71 + pytest.skip(msg)
72 +
73 + def check_available_online(
74 + self,
75 + *,
76 + on_fail: Literal["error", "skip"],
77 + ) -> None:
78 + """
79 + If the model is not available online, perform the given action.
80 + """
81 + if not self.is_available_online:
82 + msg = "Model is not available online"
83 +
84 + if on_fail == "error":
85 + raise RuntimeError(msg)
86 + else:
87 + pytest.skip(msg)
88 +
89
90 # yapf: disable
91 _TEXT_GENERATION_EXAMPLE_MODELS = {
@@ -48,8 +96,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
96 trust_remote_code=True),
97 "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
98 trust_remote_code=True),
51 - "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
52 - trust_remote_code=True),
99 "BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
100 trust_remote_code=True),
101 "BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
@@ -176,6 +222,8 @@ _CROSS_ENCODER_EXAMPLE_MODELS = {
222
223 _MULTIMODAL_EXAMPLE_MODELS = {
224 # [Decoder-only]
225 + "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
226 + min_transformers_version="4.48"),
227 "Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
228 "ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
229 "ChatGLMModel": _HfExamplesInfo("THUDM/glm-4v-9b",
@@ -183,7 +231,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
231 trust_remote_code=True),
232 "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
233 is_available_online=False),
186 - "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny"), # noqa: E501
234 + "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
235 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
236 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
237 "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
238 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
@@ -194,7 +243,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
243 "LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
244 "LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
245 "LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
197 - "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3"), # noqa: E501
246 + "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3", # noqa: E501
247 + hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
248 "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
249 trust_remote_code=True),
250 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
@@ -247,5 +297,12 @@ class HfExampleModels:
297 def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
298 return self.hf_models[model_arch]
299
300 + def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
301 + for info in self.hf_models.values():
302 + if info.default == model_id:
303 + return info
304 +
305 + raise ValueError(f"No example model defined for {model_id}")
306 +
307
308 HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
tests/models/test_initialization.py
+2 -12
@@ -1,9 +1,7 @@
1 from unittest.mock import patch
2
3 import pytest
4 -from packaging.version import Version
4 from transformers import PretrainedConfig
6 -from transformers import __version__ as TRANSFORMERS_VERSION
5
6 from vllm import LLM
7
@@ -13,16 +11,8 @@ from .registry import HF_EXAMPLE_MODELS
11 @pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
12 def test_can_initialize(model_arch):
13 model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
16 - if not model_info.is_available_online:
17 - pytest.skip("Model is not available online")
18 - if model_info.min_transformers_version is not None:
19 - current_version = TRANSFORMERS_VERSION
20 - required_version = model_info.min_transformers_version
21 - if Version(current_version) < Version(required_version):
22 - pytest.skip(
23 - f"You have `transformers=={current_version}` installed, but "
24 - f"`transformers>={required_version}` is required to run this "
25 - "model")
14 + model_info.check_available_online(on_fail="skip")
15 + model_info.check_transformers_version(on_fail="skip")
16
17 # Avoid OOM
18 def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
tests/models/test_registry.py
+3
@@ -21,6 +21,9 @@ from .registry import HF_EXAMPLE_MODELS
21
22 @pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
23 def test_registry_imports(model_arch):
24 + model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
25 + model_info.check_transformers_version(on_fail="skip")
26 +
27 # Ensure all model classes can be imported successfully
28 model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
29
vllm/model_executor/models/aria.py
+100 -175
@@ -1,9 +1,11 @@
1 -from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
2 - TypedDict, Union)
1 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2 + Union)
3
4 import torch
5 import torch.nn as nn
6 -from transformers import BatchFeature, PretrainedConfig
6 +from transformers import AriaConfig, AriaTextConfig, BatchFeature
7 +from transformers.models.aria.modeling_aria import AriaCrossAttention
8 +from transformers.models.aria.processing_aria import AriaProcessor
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
@@ -26,10 +28,11 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 BaseProcessingInfo, PromptReplacement)
29 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
29 -from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
30 - AriaVisionConfig)
31
32 -from .idefics2_vision_model import Idefics2VisionTransformer
32 +# yapf: disable
33 +from .idefics2_vision_model import (
34 + Idefics2VisionTransformer as Idefics3VisionTransformer)
35 +# yapf: enable
36 from .interfaces import SupportsMultiModal
37 from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
38 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -47,87 +50,22 @@ class AriaImagePixelInputs(TypedDict):
50 """
51
52
50 -class AriaVisionTransformer(Idefics2VisionTransformer):
51 - """
52 - AriaVisionTransformer is a modified version of Idefics2VisionTransformer
53 - that replaces the post-layernorm with an identity layer.
54 - """
55 -
56 - def __init__(
57 - self,
58 - config: AriaVisionConfig,
59 - quant_config: Optional[QuantizationConfig] = None,
60 - prefix: str = "",
61 - ) -> None:
62 - super().__init__(config, quant_config, prefix)
63 - self.post_layernorm = nn.Identity()
64 -
65 -
66 -class AriaVisionModel(nn.Module):
67 - config_class = AriaVisionConfig
53 +class AriaProjectorMLP(nn.Module):
54
55 def __init__(
56 self,
71 - config: AriaVisionConfig,
72 - quant_config: Optional[QuantizationConfig] = None,
73 - *,
74 - prefix: str = "",
57 + in_features: int,
58 + hidden_features: int,
59 + output_dim: int,
60 ) -> None:
61 super().__init__()
62
78 - self.vision_model = AriaVisionTransformer(
79 - config,
80 - quant_config,
81 - prefix=f"{prefix}.vision_model",
82 - )
83 -
84 - def forward(
85 - self,
86 - pixel_values: torch.Tensor,
87 - pixel_mask: Optional[torch.Tensor] = None,
88 - ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
89 - patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
90 -
91 - vit_oup = self.vision_model(
92 - pixel_values=pixel_values,
93 - patch_attention_mask=patch_attention_mask,
94 - )
95 -
96 - image_atts = self._create_image_attention_mask(patch_attention_mask)
97 -
98 - return vit_oup, image_atts
99 -
100 - def _create_patch_attention_mask(
101 - self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
102 - if pixel_mask is None:
103 - return None
104 -
105 - patches_subgrid = pixel_mask.unfold(
106 - dimension=1,
107 - size=self.vision_model.config.patch_size,
108 - step=self.vision_model.config.patch_size,
109 - ).unfold(
110 - dimension=2,
111 - size=self.vision_model.config.patch_size,
112 - step=self.vision_model.config.patch_size,
113 - )
114 - return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
115 -
116 - def _create_image_attention_mask(
117 - self, patch_attention_mask: torch.Tensor) -> torch.Tensor:
118 - if patch_attention_mask is None:
119 - return None
120 -
121 - flattened_mask = patch_attention_mask.flatten(1)
122 - return torch.logical_not(flattened_mask)
123 -
124 -
125 -class FFN(nn.Module):
126 -
127 - def __init__(self, embed_dim: int, ff_dim: int, output_dim: int) -> None:
128 - super().__init__()
129 - self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
130 - self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
63 + self.linear_in = ColumnParallelLinear(in_features,
64 + hidden_features,
65 + bias=False)
66 + self.linear_out = RowParallelLinear(hidden_features,
67 + output_dim,
68 + bias=False)
69 self.act = get_act_fn("gelu_new")
70
71 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -137,46 +75,6 @@ class FFN(nn.Module):
75 return hidden_states
76
77
140 -class CrossAttention(nn.Module):
141 -
142 - def __init__(self, kv_dim: int, embed_dim: int, num_heads: int) -> None:
143 - super().__init__()
144 - self.num_heads = num_heads
145 - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
146 - self.k_proj = nn.Linear(kv_dim, embed_dim, bias=False)
147 - self.v_proj = nn.Linear(kv_dim, embed_dim, bias=False)
148 -
149 - self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
150 - self.linear = nn.Linear(embed_dim, embed_dim)
151 -
152 - self.layer_norm = nn.LayerNorm(embed_dim)
153 - self.ln_kv = nn.LayerNorm(kv_dim)
154 -
155 - def forward(
156 - self,
157 - x: torch.Tensor,
158 - hidden_states: torch.Tensor,
159 - attn_mask: Optional[torch.Tensor] = None,
160 - ) -> torch.Tensor:
161 - normed_hidden_states = self.layer_norm(hidden_states)
162 - query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
163 -
164 - x = self.ln_kv(x)
165 - key = self.k_proj(x).permute(1, 0, 2)
166 - value = self.v_proj(x).permute(1, 0, 2)
167 -
168 - attn_output, _ = self.multihead_attn(query,
169 - key,
170 - value,
171 - attn_mask=attn_mask)
172 -
173 - attn_output = attn_output.permute(1, 0, 2)
174 -
175 - attn_output = self.linear(attn_output)
176 -
177 - return attn_output
178 -
179 -
78 class AriaProjector(nn.Module):
79 """
80 A projection module with one cross attention layer and one FFN layer, which
@@ -198,42 +96,42 @@ class AriaProjector(nn.Module):
96 A tensor with the shape of (batch_size, query_number, output_dim)
97 """
98
201 - def __init__(
202 - self,
203 - patch_to_query_dict: dict[int, int],
204 - embed_dim: int,
205 - num_heads: int,
206 - kv_dim: int,
207 - ff_dim: int,
208 - output_dim: int,
209 - norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
210 - ) -> None:
99 + def __init__(self, config: AriaConfig) -> None:
100 super().__init__()
212 - self.patch_to_query_dict = patch_to_query_dict
213 - self.embed_dim = embed_dim
214 - self.num_heads = num_heads
101 +
102 + self.patch_to_query_dict = config.projector_patch_to_query_dict
103 + self.in_features = config.vision_config.hidden_size
104 + self.num_heads = config.vision_config.num_attention_heads
105 + self.kv_dim = config.vision_config.hidden_size
106 + self.hidden_features = config.text_config.hidden_size
107 + self.output_dim = config.text_config.hidden_size
108
109 self.query = nn.Parameter(
217 - torch.empty(max(patch_to_query_dict.values()), self.embed_dim))
110 + torch.empty(config.max_value_projector_patch_to_query_dict,
111 + self.in_features))
112
219 - self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
113 + self.cross_attn = AriaCrossAttention(config)
114
221 - self.ln_ffn = norm_layer(embed_dim)
222 - self.ffn = FFN(embed_dim, ff_dim, output_dim)
115 + self.layer_norm = nn.LayerNorm(self.in_features)
116 + self.feed_forward = AriaProjectorMLP(self.in_features,
117 + self.hidden_features,
118 + self.output_dim)
119
120 def forward(
121 self,
122 x: torch.Tensor,
123 attn_mask: Optional[torch.Tensor] = None,
124 ) -> torch.Tensor:
229 - bs = x.shape[0]
230 - queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
125 + batch_size, num_patches = x.shape[0], x.shape[1]
126
232 - query_num = self.patch_to_query_dict.get(x.shape[1], None)
233 - assert (query_num is not None
234 - ), f"Query number for {x.shape[1]} patches is not provided"
127 + if num_patches not in self.patch_to_query_dict:
128 + raise KeyError(f"Number of patches {num_patches} not found in "
129 + "patch_to_query_dict amongst possible values "
130 + f"{self.patch_to_query_dict.keys()}.")
131
236 - queries = queries[:, :query_num, :]
132 + query_num = self.patch_to_query_dict[num_patches]
133 +
134 + queries = self.query[:query_num].unsqueeze(0).repeat(batch_size, 1, 1)
135
136 if attn_mask is not None:
137 attn_mask = attn_mask.repeat_interleave(self.num_heads, 0)
@@ -241,7 +139,7 @@ class AriaProjector(nn.Module):
139
140 attention_out = self.cross_attn(x, queries, attn_mask=attn_mask)
141
244 - out = self.ffn(self.ln_ffn(attention_out))
142 + out = self.feed_forward(self.layer_norm(attention_out))
143
144 return out
145
@@ -278,7 +176,7 @@ class AriaFusedMoE(FusedMoE):
176 param.data.copy_(loaded_weight.transpose(1, 2))
177
178
281 -class MoELayer(nn.Module):
179 +class AriaTextMoELayer(nn.Module):
180 """
181 Mixture of Experts (MoE) Layer for the AriaMoE model.
182
@@ -289,7 +187,7 @@ class MoELayer(nn.Module):
187
188 def __init__(
189 self,
292 - config: AriaMoELMConfig,
190 + config: AriaTextConfig,
191 quant_config: Optional[QuantizationConfig],
192 ) -> None:
193 super().__init__()
@@ -303,15 +201,16 @@ class MoELayer(nn.Module):
201 num_experts=config.moe_num_experts,
202 top_k=config.moe_topk,
203 hidden_size=config.hidden_size,
306 - intermediate_size=config.moe_intermediate_size,
204 + intermediate_size=config.intermediate_size,
205 quant_config=quant_config,
206 reduce_results=True,
207 )
208 self.shared_experts = LlamaMLP(
209 config.hidden_size,
312 - config.moe_intermediate_size * config.moe_num_shared_experts,
210 + config.intermediate_size * config.moe_num_shared_experts,
211 "silu",
212 quant_config=quant_config,
213 + bias=config.mlp_bias,
214 )
215
216 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -329,13 +228,13 @@ class MoELayer(nn.Module):
228 router_output = torch.nn.functional.linear(hidden_states,
229 self.router_weight)
230
332 - shared_expert_output = self.shared_experts(hidden_states)
231 sparse_expert_output = self.experts(hidden_states, router_output)
232 + shared_expert_output = self.shared_experts(hidden_states)
233
234 return sparse_expert_output + shared_expert_output
235
236
338 -class MoEDecoderLayer(LlamaDecoderLayer):
237 +class AriaTextDecoderLayer(LlamaDecoderLayer):
238 """
239 Custom Decoder Layer for the AriaMoE model which modifies the standard
240 `LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
@@ -344,16 +243,16 @@ class MoEDecoderLayer(LlamaDecoderLayer):
243
244 def __init__(
245 self,
347 - config: AriaMoELMConfig,
246 + config: AriaTextConfig,
247 cache_config: Optional[CacheConfig] = None,
248 quant_config: Optional[QuantizationConfig] = None,
249 prefix: str = "",
250 ) -> None:
251 super().__init__(config, cache_config, quant_config, prefix)
353 - self.mlp = MoELayer(config, quant_config=quant_config)
252 + self.mlp = AriaTextMoELayer(config, quant_config=quant_config)
253
254
356 -class AriaMoELMModel(LlamaModel):
255 +class AriaTextModel(LlamaModel):
256 """
257 Custom LlamaModel for the AriaMoE model which modifies the standard
258 LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
@@ -362,7 +261,7 @@ class AriaMoELMModel(LlamaModel):
261 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262 super().__init__(vllm_config=vllm_config,
263 prefix=prefix,
365 - layer_type=MoEDecoderLayer)
264 + layer_type=AriaTextDecoderLayer)
265
266 # Adapted from LlamaModel.load_weights with the modification of adding
267 # the expert weights mapping to `stacked_params_mapping`
@@ -434,25 +333,23 @@ class AriaMoELMModel(LlamaModel):
333 return loaded_params
334
335
437 -def build_mm_projector(config: PretrainedConfig):
438 - return AriaProjector(
439 - patch_to_query_dict=config.projector_patch_to_query_dict,
440 - embed_dim=config.vision_config.hidden_size,
441 - num_heads=config.vision_config.num_attention_heads,
442 - kv_dim=config.vision_config.hidden_size,
443 - ff_dim=config.text_config.hidden_size,
444 - output_dim=config.text_config.hidden_size,
445 - )
446 -
447 -
336 class AriaProcessingInfo(BaseProcessingInfo):
337
338 def get_hf_config(self):
451 - return self.ctx.get_hf_config()
339 + return self.ctx.get_hf_config(AriaConfig)
340
453 - def get_vision_config(self) -> AriaVisionConfig:
341 + def get_vision_config(self):
342 return self.get_hf_config().vision_config
343
344 + def get_hf_processor(self):
345 + processor = self.ctx.get_hf_processor(AriaProcessor)
346 +
347 + # Patch for https://github.com/huggingface/transformers/issues/35768
348 + processor.tokenizer.image_token = "<|img|>"
349 + processor.image_token = "<|img|>"
350 +
351 + return processor
352 +
353 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
354 return {"image": None}
355
@@ -554,10 +451,14 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
451 quant_config = vllm_config.quant_config
452
453 self.config = config
557 - self.vision_tower = AriaVisionModel(config.vision_config)
558 - self.multi_modal_projector = build_mm_projector(config)
454 + self.vision_tower = Idefics3VisionTransformer(
455 + config.vision_config,
456 + quant_config,
457 + prefix=f"{prefix}.vision_tower",
458 + )
459 + self.multi_modal_projector = AriaProjector(config)
460 self.vocab_size = config.text_config.vocab_size
560 - self.language_model = AriaMoELMModel(
461 + self.language_model = AriaTextModel(
462 vllm_config=vllm_config.with_hf_config(config.text_config),
463 prefix=maybe_prefix(prefix, "language_model.model"),
464 )
@@ -608,6 +509,22 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
509 pixel_mask=pixel_mask,
510 )
511
512 + def _create_patch_attention_mask(
513 + self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
514 + if pixel_mask is None:
515 + return None
516 +
517 + patches_subgrid = pixel_mask.unfold(
518 + dimension=1,
519 + size=self.vision_tower.config.patch_size,
520 + step=self.vision_tower.config.patch_size,
521 + ).unfold(
522 + dimension=2,
523 + size=self.vision_tower.config.patch_size,
524 + step=self.vision_tower.config.patch_size,
525 + )
526 + return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
527 +
528 def _process_image_input(
529 self, image_input: AriaImagePixelInputs
530 ) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -616,9 +533,18 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
533 pixel_values = image_input['pixel_values']
534 pixel_mask = image_input['pixel_mask']
535
619 - image_feature, image_attn_mask = self.vision_tower(
620 - pixel_values, pixel_mask=pixel_mask)
621 - return self.multi_modal_projector(image_feature, image_attn_mask)
536 + patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
537 +
538 + image_outputs = self.vision_tower(
539 + pixel_values=pixel_values,
540 + patch_attention_mask=patch_attention_mask,
541 + )
542 + image_attn_mask = None
543 + if patch_attention_mask is not None:
544 + flattened_mask = patch_attention_mask.flatten(1)
545 + image_attn_mask = torch.logical_not(flattened_mask)
546 +
547 + return self.multi_modal_projector(image_outputs, image_attn_mask)
548
549 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
550 image_input = self._parse_and_validate_image_input(**kwargs)
@@ -683,6 +609,5 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
609 return next_tokens
610
611 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
686 -
612 loader = AutoWeightsLoader(self)
613 loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/transformers_utils/config.py
+4 -5
@@ -22,10 +22,10 @@ from vllm.envs import VLLM_USE_MODELSCOPE
22 from vllm.logger import init_logger
23 # yapf conflicts with isort for this block
24 # yapf: disable
25 -from vllm.transformers_utils.configs import (AriaConfig, ChatGLMConfig,
26 - Cohere2Config, DbrxConfig,
27 - DeepseekVLV2Config, EAGLEConfig,
28 - ExaoneConfig, H2OVLChatConfig,
25 +from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26 + DbrxConfig, DeepseekVLV2Config,
27 + EAGLEConfig, ExaoneConfig,
28 + H2OVLChatConfig,
29 InternVLChatConfig, JAISConfig,
30 MedusaConfig, MllamaConfig,
31 MLPSpeculatorConfig, MPTConfig,
@@ -52,7 +52,6 @@ _CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
52 }
53
54 _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
55 - "aria": AriaConfig,
55 "chatglm": ChatGLMConfig,
56 "cohere2": Cohere2Config,
57 "dbrx": DbrxConfig,
vllm/transformers_utils/configs/__init__.py
-2
@@ -1,4 +1,3 @@
1 -from vllm.transformers_utils.configs.aria import AriaConfig
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 from vllm.transformers_utils.configs.cohere2 import Cohere2Config
3 from vllm.transformers_utils.configs.dbrx import DbrxConfig
@@ -24,7 +23,6 @@ from vllm.transformers_utils.configs.telechat2 import Telechat2Config
23 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
24
25 __all__ = [
27 - "AriaConfig",
26 "ChatGLMConfig",
27 "Cohere2Config",
28 "DbrxConfig",
vllm/transformers_utils/configs/aria.py deleted
-165
@@ -1,165 +0,0 @@
1 -# Copyright 2024 Rhymes AI. All rights reserved.
2 -#
3 -# Licensed to the Apache Software Foundation (ASF) under one
4 -# or more contributor license agreements. See the NOTICE file
5 -# distributed with this work for additional information
6 -# regarding copyright ownership. The ASF licenses this file
7 -# to you under the Apache License, Version 2.0 (the
8 -# "License"); you may not use this file except in compliance
9 -# with the License. You may obtain a copy of the License at
10 -#
11 -# http://www.apache.org/licenses/LICENSE-2.0
12 -#
13 -# Unless required by applicable law or agreed to in writing,
14 -# software distributed under the License is distributed on an
15 -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 -# KIND, either express or implied. See the License for the
17 -# specific language governing permissions and limitations
18 -# under the License.
19 -from typing import Mapping
20 -
21 -from transformers import PretrainedConfig
22 -from transformers.models.idefics2.configuration_idefics2 import (
23 - Idefics2VisionConfig)
24 -from transformers.models.llama.configuration_llama import LlamaConfig
25 -
26 -from vllm.logger import init_logger
27 -
28 -logger = init_logger(__name__)
29 -
30 -
31 -class AriaVisionConfig(Idefics2VisionConfig):
32 - model_type = "aria_vision_model"
33 -
34 -
35 -class AriaMoELMConfig(LlamaConfig):
36 - """
37 - Configuration class for AriaMoE language model.
38 -
39 - This class extends the LlamaConfig to include additional parameters specific
40 - to the Mixture of Experts (MoE) architecture.
41 - """
42 -
43 - model_type = "aria_moe_lm"
44 -
45 - def __init__(
46 - self,
47 - moe_intermediate_size: int = 4096,
48 - moe_num_experts: int = 8,
49 - moe_topk: int = 2,
50 - moe_num_shared_experts: int = 2,
51 - **kwargs,
52 - ):
53 - """
54 - Initialize the AriaMoELMConfig.
55 -
56 - Args:
57 - moe_intermediate_size (int): The intermediate size for MoE layers.
58 - Default is 4096.
59 - moe_num_experts (int): The number of experts in the MoE layer.
60 - Default is 8.
61 - moe_topk (int): The number of top experts to route to for each
62 - token. Default is 2.
63 - moe_num_shared_experts (int): The number of shared experts. Default
64 - is 2.
65 - **kwargs: Additional keyword arguments to be passed to the parent
66 - LlamaConfig.
67 - """
68 - super().__init__(**kwargs)
69 - self.moe_intermediate_size = moe_intermediate_size
70 - self.moe_num_experts = moe_num_experts
71 - self.moe_topk = moe_topk
72 - self.moe_num_shared_experts = moe_num_shared_experts
73 -
74 -
75 -class AriaConfig(PretrainedConfig):
76 - """
77 - Configuration class for Aria model.
78 - This class handles the configuration for both vision and text components of
79 - the Aria model,
80 - as well as additional parameters for image token handling and projector
81 - mapping.
82 -
83 - Args:
84 - vision_config (AriaVisionConfig or dict): Configuration for the vision
85 - component.
86 - text_config (AriaMoELMConfig or dict): Configuration for the text
87 - component.
88 - projector_patch_to_query_dict (dict): Mapping of patch sizes to query
89 - dimensions.
90 - ignore_index (int): Index to ignore in loss calculation.
91 - image_token_index (int): Index used to represent image tokens.
92 - **kwargs: Additional keyword arguments passed to the parent class.
93 - Attributes:
94 - model_type (str): Type of the model, set to "aria".
95 - is_composition (bool): Whether the model is a composition of multiple
96 - components.
97 - ignore_index (int): Index to ignore in loss calculation.
98 - image_token_index (int): Index used to represent image tokens.
99 - projector_patch_to_query_dict (dict): Mapping of patch sizes to query
100 - dimensions.
101 - vision_config (AriaVisionConfig): Configuration for the vision
102 - component.
103 - text_config (AriaMoELMConfig): Configuration for the text component.
104 - """
105 -
106 - model_type = "aria"
107 - is_composition = False
108 -
109 - def __init__(
110 - self,
111 - vision_config: AriaVisionConfig = AriaVisionConfig(), # noqa: B008
112 - text_config: AriaMoELMConfig = AriaMoELMConfig(), # noqa: B008
113 - projector_patch_to_query_dict: Mapping[int, int] = {
114 - 1225: 128,
115 - 4900: 256,
116 - },
117 - ignore_index=-100,
118 - image_token_index=32000,
119 - tie_word_embeddings=False,
120 - **kwargs,
121 - ):
122 - super().__init__(**kwargs)
123 - self.ignore_index = ignore_index
124 - self.image_token_index = image_token_index
125 - self.tie_word_embeddings = tie_word_embeddings
126 - attn_implementation = kwargs.pop("attn_implementation", None)
127 -
128 - # Set the default attention implementation to flash_attention_2 if not
129 - # specified
130 - self._attn_implementation = ("flash_attention_2"
131 - if attn_implementation is None else
132 - attn_implementation)
133 -
134 - # Convert the keys and values of projector_patch_to_query_dict to
135 - # integers
136 - # This ensures consistency even if they were provided as strings
137 - self.projector_patch_to_query_dict = {
138 - int(k): int(v)
139 - for k, v in projector_patch_to_query_dict.items()
140 - }
141 -
142 - if isinstance(vision_config, dict) and "model_type" in vision_config:
143 - vision_config = AriaVisionConfig(**vision_config)
144 - if attn_implementation is None:
145 - vision_attn_implementation = "flash_attention_2"
146 - elif attn_implementation == "sdpa":
147 - logger.warning("SDPA is not supported for vit, using "
148 - "flash_attention_2 instead")
149 - vision_attn_implementation = "flash_attention_2"
150 - else:
151 - vision_attn_implementation = attn_implementation
152 - vision_config._attn_implementation = vision_attn_implementation
153 -
154 - self.vision_config = vision_config
155 -
156 - if isinstance(text_config, dict) and "model_type" in text_config:
157 - text_attn_implementation = ("sdpa" if attn_implementation is None
158 - else attn_implementation)
159 - text_config = AriaMoELMConfig(**text_config)
160 - text_config._attn_implementation = text_attn_implementation
161 -
162 - self.text_config = text_config
163 -
164 - # This is needed for the static kv cache
165 - self.num_hidden_layers = self.text_config.num_hidden_layers