A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model] Upgrade Aria to transformers 4.48 (#12203)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 20, 2025 at 17:58 UTC
b37d82791e3c9f7d492db81493d920004de59a26
10 files changed
+178
-379
examples/offline_inference/vision_language.py
-3
@@ -26,11 +26,8 @@ def run_aria(question: str, modality: str):
26
27
# NOTE: Need L40 (or equivalent) to avoid OOM
28
llm = LLM(model=model_name,
29
- tokenizer_mode="slow",
30
- dtype="bfloat16",
29
max_model_len=4096,
30
max_num_seqs=2,
33
- trust_remote_code=True,
31
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
32
33
prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
tests/models/decoder_only/vision_language/test_models.py
+2
-5
@@ -10,7 +10,6 @@ from typing import Type
10
import pytest
11
from transformers import AutoModelForVision2Seq
12
from transformers import __version__ as TRANSFORMERS_VERSION
13
-from transformers.utils import is_flash_attn_2_available
13
14
from vllm.platforms import current_platform
15
from vllm.utils import identity
@@ -140,9 +139,7 @@ VLM_TEST_SETTINGS = {
139
#### Extended model tests
140
"aria": VLMTestInfo(
141
models=["rhymes-ai/Aria"],
143
- tokenizer_mode="slow",
142
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
145
- dtype="bfloat16",
143
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
144
img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
145
max_model_len=4096,
@@ -158,8 +155,8 @@ VLM_TEST_SETTINGS = {
155
max_tokens=64,
156
marks=[
157
pytest.mark.skipif(
161
- not is_flash_attn_2_available(),
162
- reason="Model needs flash-attn for numeric convergence.",
158
+ TRANSFORMERS_VERSION < "4.48.0",
159
+ reason="HF model requires transformers>=4.48.0",
160
),
161
large_gpu_mark(min_gb=64),
162
],
tests/models/multimodal/processing/test_common.py
+5
-7
@@ -11,6 +11,7 @@ from vllm.multimodal.processing import ProcessingCache
11
from vllm.multimodal.utils import cached_get_tokenizer
12
13
from ....multimodal.utils import random_audio, random_image, random_video
14
+from ...registry import HF_EXAMPLE_MODELS
15
16
17
def _test_processing_correctness(
@@ -20,12 +21,9 @@ def _test_processing_correctness(
21
num_batches: int,
22
simplify_rate: float,
23
):
23
- if model_id == "TIGER-Lab/Mantis-8B-siglip-llama3":
24
- hf_overrides = {"architectures": ["MantisForConditionalGeneration"]}
25
- elif model_id == "deepseek-ai/deepseek-vl2-tiny":
26
- hf_overrides = {"architectures": ["DeepseekVLV2ForCausalLM"]}
27
- else:
28
- hf_overrides = {}
24
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
25
+ model_info.check_available_online(on_fail="skip")
26
+ model_info.check_transformers_version(on_fail="skip")
27
28
limit_mm_per_prompt = {
29
modality: 3 if supports_multi else 1
@@ -41,7 +39,7 @@ def _test_processing_correctness(
39
seed=0,
40
dtype="float16",
41
revision=None,
44
- hf_overrides=hf_overrides,
42
+ hf_overrides=model_info.hf_overrides,
43
limit_mm_per_prompt=limit_mm_per_prompt,
44
)
45
tests/models/registry.py
+62
-5
@@ -1,5 +1,9 @@
1
from dataclasses import dataclass, field
2
-from typing import AbstractSet, Mapping, Optional
2
+from typing import AbstractSet, Any, Literal, Mapping, Optional
3
+
4
+import pytest
5
+from packaging.version import Version
6
+from transformers import __version__ as TRANSFORMERS_VERSION
7
8
9
@dataclass(frozen=True)
@@ -38,6 +42,50 @@ class _HfExamplesInfo:
42
trust_remote_code: bool = False
43
"""The ``trust_remote_code`` level required to load the model."""
44
45
+ hf_overrides: dict[str, Any] = field(default_factory=dict)
46
+ """The ``hf_overrides`` required to load the model."""
47
+
48
+ def check_transformers_version(
49
+ self,
50
+ *,
51
+ on_fail: Literal["error", "skip"],
52
+ ) -> None:
53
+ """
54
+ If the installed transformers version does not meet the requirements,
55
+ perform the given action.
56
+ """
57
+ if self.min_transformers_version is None:
58
+ return
59
+
60
+ current_version = TRANSFORMERS_VERSION
61
+ required_version = self.min_transformers_version
62
+ if Version(current_version) < Version(required_version):
63
+ msg = (
64
+ f"You have `transformers=={current_version}` installed, but "
65
+ f"`transformers>={required_version}` is required to run this "
66
+ "model")
67
+
68
+ if on_fail == "error":
69
+ raise RuntimeError(msg)
70
+ else:
71
+ pytest.skip(msg)
72
+
73
+ def check_available_online(
74
+ self,
75
+ *,
76
+ on_fail: Literal["error", "skip"],
77
+ ) -> None:
78
+ """
79
+ If the model is not available online, perform the given action.
80
+ """
81
+ if not self.is_available_online:
82
+ msg = "Model is not available online"
83
+
84
+ if on_fail == "error":
85
+ raise RuntimeError(msg)
86
+ else:
87
+ pytest.skip(msg)
88
+
89
90
# yapf: disable
91
_TEXT_GENERATION_EXAMPLE_MODELS = {
@@ -48,8 +96,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
96
trust_remote_code=True),
97
"ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
98
trust_remote_code=True),
51
- "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
52
- trust_remote_code=True),
99
"BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
100
trust_remote_code=True),
101
"BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
@@ -176,6 +222,8 @@ _CROSS_ENCODER_EXAMPLE_MODELS = {
222
223
_MULTIMODAL_EXAMPLE_MODELS = {
224
# [Decoder-only]
225
+ "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
226
+ min_transformers_version="4.48"),
227
"Blip2ForConditionalGeneration": _HfExamplesInfo("Salesforce/blip2-opt-2.7b"), # noqa: E501
228
"ChameleonForConditionalGeneration": _HfExamplesInfo("facebook/chameleon-7b"), # noqa: E501
229
"ChatGLMModel": _HfExamplesInfo("THUDM/glm-4v-9b",
@@ -183,7 +231,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
231
trust_remote_code=True),
232
"ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
233
is_available_online=False),
186
- "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny"), # noqa: E501
234
+ "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny", # noqa: E501
235
+ hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]}), # noqa: E501
236
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
237
"H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
238
"InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
@@ -194,7 +243,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
243
"LlavaNextForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-v1.6-mistral-7b-hf"), # noqa: E501
244
"LlavaNextVideoForConditionalGeneration": _HfExamplesInfo("llava-hf/LLaVA-NeXT-Video-7B-hf"), # noqa: E501
245
"LlavaOnevisionForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-onevision-qwen2-0.5b-ov-hf"), # noqa: E501
197
- "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3"), # noqa: E501
246
+ "MantisForConditionalGeneration": _HfExamplesInfo("TIGER-Lab/Mantis-8B-siglip-llama3", # noqa: E501
247
+ hf_overrides={"architectures": ["MantisForConditionalGeneration"]}), # noqa: E501
248
"MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-Llama3-V-2_5",
249
trust_remote_code=True),
250
"MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
@@ -247,5 +297,12 @@ class HfExampleModels:
297
def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
298
return self.hf_models[model_arch]
299
300
+ def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
301
+ for info in self.hf_models.values():
302
+ if info.default == model_id:
303
+ return info
304
+
305
+ raise ValueError(f"No example model defined for {model_id}")
306
+
307
308
HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
tests/models/test_initialization.py
+2
-12
@@ -1,9 +1,7 @@
1
from unittest.mock import patch
2
3
import pytest
4
-from packaging.version import Version
4
from transformers import PretrainedConfig
6
-from transformers import __version__ as TRANSFORMERS_VERSION
5
6
from vllm import LLM
7
@@ -13,16 +11,8 @@ from .registry import HF_EXAMPLE_MODELS
11
@pytest.mark.parametrize("model_arch", HF_EXAMPLE_MODELS.get_supported_archs())
12
def test_can_initialize(model_arch):
13
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
16
- if not model_info.is_available_online:
17
- pytest.skip("Model is not available online")
18
- if model_info.min_transformers_version is not None:
19
- current_version = TRANSFORMERS_VERSION
20
- required_version = model_info.min_transformers_version
21
- if Version(current_version) < Version(required_version):
22
- pytest.skip(
23
- f"You have `transformers=={current_version}` installed, but "
24
- f"`transformers>={required_version}` is required to run this "
25
- "model")
14
+ model_info.check_available_online(on_fail="skip")
15
+ model_info.check_transformers_version(on_fail="skip")
16
17
# Avoid OOM
18
def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
tests/models/test_registry.py
+3
@@ -21,6 +21,9 @@ from .registry import HF_EXAMPLE_MODELS
21
22
@pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
23
def test_registry_imports(model_arch):
24
+ model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
25
+ model_info.check_transformers_version(on_fail="skip")
26
+
27
# Ensure all model classes can be imported successfully
28
model_cls, _ = ModelRegistry.resolve_model_cls(model_arch)
29
vllm/model_executor/models/aria.py
+100
-175
@@ -1,9 +1,11 @@
1
-from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
2
- TypedDict, Union)
1
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2
+ Union)
3
4
import torch
5
import torch.nn as nn
6
-from transformers import BatchFeature, PretrainedConfig
6
+from transformers import AriaConfig, AriaTextConfig, BatchFeature
7
+from transformers.models.aria.modeling_aria import AriaCrossAttention
8
+from transformers.models.aria.processing_aria import AriaProcessor
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
@@ -26,10 +28,11 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
BaseProcessingInfo, PromptReplacement)
29
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30
from vllm.sequence import IntermediateTensors
29
-from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
30
- AriaVisionConfig)
31
32
-from .idefics2_vision_model import Idefics2VisionTransformer
32
+# yapf: disable
33
+from .idefics2_vision_model import (
34
+ Idefics2VisionTransformer as Idefics3VisionTransformer)
35
+# yapf: enable
36
from .interfaces import SupportsMultiModal
37
from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
38
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -47,87 +50,22 @@ class AriaImagePixelInputs(TypedDict):
50
"""
51
52
50
-class AriaVisionTransformer(Idefics2VisionTransformer):
51
- """
52
- AriaVisionTransformer is a modified version of Idefics2VisionTransformer
53
- that replaces the post-layernorm with an identity layer.
54
- """
55
-
56
- def __init__(
57
- self,
58
- config: AriaVisionConfig,
59
- quant_config: Optional[QuantizationConfig] = None,
60
- prefix: str = "",
61
- ) -> None:
62
- super().__init__(config, quant_config, prefix)
63
- self.post_layernorm = nn.Identity()
64
-
65
-
66
-class AriaVisionModel(nn.Module):
67
- config_class = AriaVisionConfig
53
+class AriaProjectorMLP(nn.Module):
54
55
def __init__(
56
self,
71
- config: AriaVisionConfig,
72
- quant_config: Optional[QuantizationConfig] = None,
73
- *,
74
- prefix: str = "",
57
+ in_features: int,
58
+ hidden_features: int,
59
+ output_dim: int,
60
) -> None:
61
super().__init__()
62
78
- self.vision_model = AriaVisionTransformer(
79
- config,
80
- quant_config,
81
- prefix=f"{prefix}.vision_model",
82
- )
83
-
84
- def forward(
85
- self,
86
- pixel_values: torch.Tensor,
87
- pixel_mask: Optional[torch.Tensor] = None,
88
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
89
- patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
90
-
91
- vit_oup = self.vision_model(
92
- pixel_values=pixel_values,
93
- patch_attention_mask=patch_attention_mask,
94
- )
95
-
96
- image_atts = self._create_image_attention_mask(patch_attention_mask)
97
-
98
- return vit_oup, image_atts
99
-
100
- def _create_patch_attention_mask(
101
- self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
102
- if pixel_mask is None:
103
- return None
104
-
105
- patches_subgrid = pixel_mask.unfold(
106
- dimension=1,
107
- size=self.vision_model.config.patch_size,
108
- step=self.vision_model.config.patch_size,
109
- ).unfold(
110
- dimension=2,
111
- size=self.vision_model.config.patch_size,
112
- step=self.vision_model.config.patch_size,
113
- )
114
- return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
115
-
116
- def _create_image_attention_mask(
117
- self, patch_attention_mask: torch.Tensor) -> torch.Tensor:
118
- if patch_attention_mask is None:
119
- return None
120
-
121
- flattened_mask = patch_attention_mask.flatten(1)
122
- return torch.logical_not(flattened_mask)
123
-
124
-
125
-class FFN(nn.Module):
126
-
127
- def __init__(self, embed_dim: int, ff_dim: int, output_dim: int) -> None:
128
- super().__init__()
129
- self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
130
- self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
63
+ self.linear_in = ColumnParallelLinear(in_features,
64
+ hidden_features,
65
+ bias=False)
66
+ self.linear_out = RowParallelLinear(hidden_features,
67
+ output_dim,
68
+ bias=False)
69
self.act = get_act_fn("gelu_new")
70
71
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -137,46 +75,6 @@ class FFN(nn.Module):
75
return hidden_states
76
77
140
-class CrossAttention(nn.Module):
141
-
142
- def __init__(self, kv_dim: int, embed_dim: int, num_heads: int) -> None:
143
- super().__init__()
144
- self.num_heads = num_heads
145
- self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
146
- self.k_proj = nn.Linear(kv_dim, embed_dim, bias=False)
147
- self.v_proj = nn.Linear(kv_dim, embed_dim, bias=False)
148
-
149
- self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
150
- self.linear = nn.Linear(embed_dim, embed_dim)
151
-
152
- self.layer_norm = nn.LayerNorm(embed_dim)
153
- self.ln_kv = nn.LayerNorm(kv_dim)
154
-
155
- def forward(
156
- self,
157
- x: torch.Tensor,
158
- hidden_states: torch.Tensor,
159
- attn_mask: Optional[torch.Tensor] = None,
160
- ) -> torch.Tensor:
161
- normed_hidden_states = self.layer_norm(hidden_states)
162
- query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
163
-
164
- x = self.ln_kv(x)
165
- key = self.k_proj(x).permute(1, 0, 2)
166
- value = self.v_proj(x).permute(1, 0, 2)
167
-
168
- attn_output, _ = self.multihead_attn(query,
169
- key,
170
- value,
171
- attn_mask=attn_mask)
172
-
173
- attn_output = attn_output.permute(1, 0, 2)
174
-
175
- attn_output = self.linear(attn_output)
176
-
177
- return attn_output
178
-
179
-
78
class AriaProjector(nn.Module):
79
"""
80
A projection module with one cross attention layer and one FFN layer, which
@@ -198,42 +96,42 @@ class AriaProjector(nn.Module):
96
A tensor with the shape of (batch_size, query_number, output_dim)
97
"""
98
201
- def __init__(
202
- self,
203
- patch_to_query_dict: dict[int, int],
204
- embed_dim: int,
205
- num_heads: int,
206
- kv_dim: int,
207
- ff_dim: int,
208
- output_dim: int,
209
- norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
210
- ) -> None:
99
+ def __init__(self, config: AriaConfig) -> None:
100
super().__init__()
212
- self.patch_to_query_dict = patch_to_query_dict
213
- self.embed_dim = embed_dim
214
- self.num_heads = num_heads
101
+
102
+ self.patch_to_query_dict = config.projector_patch_to_query_dict
103
+ self.in_features = config.vision_config.hidden_size
104
+ self.num_heads = config.vision_config.num_attention_heads
105
+ self.kv_dim = config.vision_config.hidden_size
106
+ self.hidden_features = config.text_config.hidden_size
107
+ self.output_dim = config.text_config.hidden_size
108
109
self.query = nn.Parameter(
217
- torch.empty(max(patch_to_query_dict.values()), self.embed_dim))
110
+ torch.empty(config.max_value_projector_patch_to_query_dict,
111
+ self.in_features))
112
219
- self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
113
+ self.cross_attn = AriaCrossAttention(config)
114
221
- self.ln_ffn = norm_layer(embed_dim)
222
- self.ffn = FFN(embed_dim, ff_dim, output_dim)
115
+ self.layer_norm = nn.LayerNorm(self.in_features)
116
+ self.feed_forward = AriaProjectorMLP(self.in_features,
117
+ self.hidden_features,
118
+ self.output_dim)
119
120
def forward(
121
self,
122
x: torch.Tensor,
123
attn_mask: Optional[torch.Tensor] = None,
124
) -> torch.Tensor:
229
- bs = x.shape[0]
230
- queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
125
+ batch_size, num_patches = x.shape[0], x.shape[1]
126
232
- query_num = self.patch_to_query_dict.get(x.shape[1], None)
233
- assert (query_num is not None
234
- ), f"Query number for {x.shape[1]} patches is not provided"
127
+ if num_patches not in self.patch_to_query_dict:
128
+ raise KeyError(f"Number of patches {num_patches} not found in "
129
+ "patch_to_query_dict amongst possible values "
130
+ f"{self.patch_to_query_dict.keys()}.")
131
236
- queries = queries[:, :query_num, :]
132
+ query_num = self.patch_to_query_dict[num_patches]
133
+
134
+ queries = self.query[:query_num].unsqueeze(0).repeat(batch_size, 1, 1)
135
136
if attn_mask is not None:
137
attn_mask = attn_mask.repeat_interleave(self.num_heads, 0)
@@ -241,7 +139,7 @@ class AriaProjector(nn.Module):
139
140
attention_out = self.cross_attn(x, queries, attn_mask=attn_mask)
141
244
- out = self.ffn(self.ln_ffn(attention_out))
142
+ out = self.feed_forward(self.layer_norm(attention_out))
143
144
return out
145
@@ -278,7 +176,7 @@ class AriaFusedMoE(FusedMoE):
176
param.data.copy_(loaded_weight.transpose(1, 2))
177
178
281
-class MoELayer(nn.Module):
179
+class AriaTextMoELayer(nn.Module):
180
"""
181
Mixture of Experts (MoE) Layer for the AriaMoE model.
182
@@ -289,7 +187,7 @@ class MoELayer(nn.Module):
187
188
def __init__(
189
self,
292
- config: AriaMoELMConfig,
190
+ config: AriaTextConfig,
191
quant_config: Optional[QuantizationConfig],
192
) -> None:
193
super().__init__()
@@ -303,15 +201,16 @@ class MoELayer(nn.Module):
201
num_experts=config.moe_num_experts,
202
top_k=config.moe_topk,
203
hidden_size=config.hidden_size,
306
- intermediate_size=config.moe_intermediate_size,
204
+ intermediate_size=config.intermediate_size,
205
quant_config=quant_config,
206
reduce_results=True,
207
)
208
self.shared_experts = LlamaMLP(
209
config.hidden_size,
312
- config.moe_intermediate_size * config.moe_num_shared_experts,
210
+ config.intermediate_size * config.moe_num_shared_experts,
211
"silu",
212
quant_config=quant_config,
213
+ bias=config.mlp_bias,
214
)
215
216
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -329,13 +228,13 @@ class MoELayer(nn.Module):
228
router_output = torch.nn.functional.linear(hidden_states,
229
self.router_weight)
230
332
- shared_expert_output = self.shared_experts(hidden_states)
231
sparse_expert_output = self.experts(hidden_states, router_output)
232
+ shared_expert_output = self.shared_experts(hidden_states)
233
234
return sparse_expert_output + shared_expert_output
235
236
338
-class MoEDecoderLayer(LlamaDecoderLayer):
237
+class AriaTextDecoderLayer(LlamaDecoderLayer):
238
"""
239
Custom Decoder Layer for the AriaMoE model which modifies the standard
240
`LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
@@ -344,16 +243,16 @@ class MoEDecoderLayer(LlamaDecoderLayer):
243
244
def __init__(
245
self,
347
- config: AriaMoELMConfig,
246
+ config: AriaTextConfig,
247
cache_config: Optional[CacheConfig] = None,
248
quant_config: Optional[QuantizationConfig] = None,
249
prefix: str = "",
250
) -> None:
251
super().__init__(config, cache_config, quant_config, prefix)
353
- self.mlp = MoELayer(config, quant_config=quant_config)
252
+ self.mlp = AriaTextMoELayer(config, quant_config=quant_config)
253
254
356
-class AriaMoELMModel(LlamaModel):
255
+class AriaTextModel(LlamaModel):
256
"""
257
Custom LlamaModel for the AriaMoE model which modifies the standard
258
LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
@@ -362,7 +261,7 @@ class AriaMoELMModel(LlamaModel):
261
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
262
super().__init__(vllm_config=vllm_config,
263
prefix=prefix,
365
- layer_type=MoEDecoderLayer)
264
+ layer_type=AriaTextDecoderLayer)
265
266
# Adapted from LlamaModel.load_weights with the modification of adding
267
# the expert weights mapping to `stacked_params_mapping`
@@ -434,25 +333,23 @@ class AriaMoELMModel(LlamaModel):
333
return loaded_params
334
335
437
-def build_mm_projector(config: PretrainedConfig):
438
- return AriaProjector(
439
- patch_to_query_dict=config.projector_patch_to_query_dict,
440
- embed_dim=config.vision_config.hidden_size,
441
- num_heads=config.vision_config.num_attention_heads,
442
- kv_dim=config.vision_config.hidden_size,
443
- ff_dim=config.text_config.hidden_size,
444
- output_dim=config.text_config.hidden_size,
445
- )
446
-
447
-
336
class AriaProcessingInfo(BaseProcessingInfo):
337
338
def get_hf_config(self):
451
- return self.ctx.get_hf_config()
339
+ return self.ctx.get_hf_config(AriaConfig)
340
453
- def get_vision_config(self) -> AriaVisionConfig:
341
+ def get_vision_config(self):
342
return self.get_hf_config().vision_config
343
344
+ def get_hf_processor(self):
345
+ processor = self.ctx.get_hf_processor(AriaProcessor)
346
+
347
+ # Patch for https://github.com/huggingface/transformers/issues/35768
348
+ processor.tokenizer.image_token = "<|img|>"
349
+ processor.image_token = "<|img|>"
350
+
351
+ return processor
352
+
353
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
354
return {"image": None}
355
@@ -554,10 +451,14 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
451
quant_config = vllm_config.quant_config
452
453
self.config = config
557
- self.vision_tower = AriaVisionModel(config.vision_config)
558
- self.multi_modal_projector = build_mm_projector(config)
454
+ self.vision_tower = Idefics3VisionTransformer(
455
+ config.vision_config,
456
+ quant_config,
457
+ prefix=f"{prefix}.vision_tower",
458
+ )
459
+ self.multi_modal_projector = AriaProjector(config)
460
self.vocab_size = config.text_config.vocab_size
560
- self.language_model = AriaMoELMModel(
461
+ self.language_model = AriaTextModel(
462
vllm_config=vllm_config.with_hf_config(config.text_config),
463
prefix=maybe_prefix(prefix, "language_model.model"),
464
)
@@ -608,6 +509,22 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
509
pixel_mask=pixel_mask,
510
)
511
512
+ def _create_patch_attention_mask(
513
+ self, pixel_mask: Optional[torch.Tensor]) -> torch.Tensor:
514
+ if pixel_mask is None:
515
+ return None
516
+
517
+ patches_subgrid = pixel_mask.unfold(
518
+ dimension=1,
519
+ size=self.vision_tower.config.patch_size,
520
+ step=self.vision_tower.config.patch_size,
521
+ ).unfold(
522
+ dimension=2,
523
+ size=self.vision_tower.config.patch_size,
524
+ step=self.vision_tower.config.patch_size,
525
+ )
526
+ return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
527
+
528
def _process_image_input(
529
self, image_input: AriaImagePixelInputs
530
) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -616,9 +533,18 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
533
pixel_values = image_input['pixel_values']
534
pixel_mask = image_input['pixel_mask']
535
619
- image_feature, image_attn_mask = self.vision_tower(
620
- pixel_values, pixel_mask=pixel_mask)
621
- return self.multi_modal_projector(image_feature, image_attn_mask)
536
+ patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
537
+
538
+ image_outputs = self.vision_tower(
539
+ pixel_values=pixel_values,
540
+ patch_attention_mask=patch_attention_mask,
541
+ )
542
+ image_attn_mask = None
543
+ if patch_attention_mask is not None:
544
+ flattened_mask = patch_attention_mask.flatten(1)
545
+ image_attn_mask = torch.logical_not(flattened_mask)
546
+
547
+ return self.multi_modal_projector(image_outputs, image_attn_mask)
548
549
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
550
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -683,6 +609,5 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
609
return next_tokens
610
611
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
686
-
612
loader = AutoWeightsLoader(self)
613
loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
vllm/transformers_utils/config.py
+4
-5
@@ -22,10 +22,10 @@ from vllm.envs import VLLM_USE_MODELSCOPE
22
from vllm.logger import init_logger
23
# yapf conflicts with isort for this block
24
# yapf: disable
25
-from vllm.transformers_utils.configs import (AriaConfig, ChatGLMConfig,
26
- Cohere2Config, DbrxConfig,
27
- DeepseekVLV2Config, EAGLEConfig,
28
- ExaoneConfig, H2OVLChatConfig,
25
+from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26
+ DbrxConfig, DeepseekVLV2Config,
27
+ EAGLEConfig, ExaoneConfig,
28
+ H2OVLChatConfig,
29
InternVLChatConfig, JAISConfig,
30
MedusaConfig, MllamaConfig,
31
MLPSpeculatorConfig, MPTConfig,
@@ -52,7 +52,6 @@ _CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
52
}
53
54
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
55
- "aria": AriaConfig,
55
"chatglm": ChatGLMConfig,
56
"cohere2": Cohere2Config,
57
"dbrx": DbrxConfig,
vllm/transformers_utils/configs/__init__.py
-2
@@ -1,4 +1,3 @@
1
-from vllm.transformers_utils.configs.aria import AriaConfig
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.cohere2 import Cohere2Config
3
from vllm.transformers_utils.configs.dbrx import DbrxConfig
@@ -24,7 +23,6 @@ from vllm.transformers_utils.configs.telechat2 import Telechat2Config
23
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
24
25
__all__ = [
27
- "AriaConfig",
26
"ChatGLMConfig",
27
"Cohere2Config",
28
"DbrxConfig",
vllm/transformers_utils/configs/aria.py
deleted
-165
@@ -1,165 +0,0 @@
1
-# Copyright 2024 Rhymes AI. All rights reserved.
2
-#
3
-# Licensed to the Apache Software Foundation (ASF) under one
4
-# or more contributor license agreements. See the NOTICE file
5
-# distributed with this work for additional information
6
-# regarding copyright ownership. The ASF licenses this file
7
-# to you under the Apache License, Version 2.0 (the
8
-# "License"); you may not use this file except in compliance
9
-# with the License. You may obtain a copy of the License at
10
-#
11
-# http://www.apache.org/licenses/LICENSE-2.0
12
-#
13
-# Unless required by applicable law or agreed to in writing,
14
-# software distributed under the License is distributed on an
15
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
-# KIND, either express or implied. See the License for the
17
-# specific language governing permissions and limitations
18
-# under the License.
19
-from typing import Mapping
20
-
21
-from transformers import PretrainedConfig
22
-from transformers.models.idefics2.configuration_idefics2 import (
23
- Idefics2VisionConfig)
24
-from transformers.models.llama.configuration_llama import LlamaConfig
25
-
26
-from vllm.logger import init_logger
27
-
28
-logger = init_logger(__name__)
29
-
30
-
31
-class AriaVisionConfig(Idefics2VisionConfig):
32
- model_type = "aria_vision_model"
33
-
34
-
35
-class AriaMoELMConfig(LlamaConfig):
36
- """
37
- Configuration class for AriaMoE language model.
38
-
39
- This class extends the LlamaConfig to include additional parameters specific
40
- to the Mixture of Experts (MoE) architecture.
41
- """
42
-
43
- model_type = "aria_moe_lm"
44
-
45
- def __init__(
46
- self,
47
- moe_intermediate_size: int = 4096,
48
- moe_num_experts: int = 8,
49
- moe_topk: int = 2,
50
- moe_num_shared_experts: int = 2,
51
- **kwargs,
52
- ):
53
- """
54
- Initialize the AriaMoELMConfig.
55
-
56
- Args:
57
- moe_intermediate_size (int): The intermediate size for MoE layers.
58
- Default is 4096.
59
- moe_num_experts (int): The number of experts in the MoE layer.
60
- Default is 8.
61
- moe_topk (int): The number of top experts to route to for each
62
- token. Default is 2.
63
- moe_num_shared_experts (int): The number of shared experts. Default
64
- is 2.
65
- **kwargs: Additional keyword arguments to be passed to the parent
66
- LlamaConfig.
67
- """
68
- super().__init__(**kwargs)
69
- self.moe_intermediate_size = moe_intermediate_size
70
- self.moe_num_experts = moe_num_experts
71
- self.moe_topk = moe_topk
72
- self.moe_num_shared_experts = moe_num_shared_experts
73
-
74
-
75
-class AriaConfig(PretrainedConfig):
76
- """
77
- Configuration class for Aria model.
78
- This class handles the configuration for both vision and text components of
79
- the Aria model,
80
- as well as additional parameters for image token handling and projector
81
- mapping.
82
-
83
- Args:
84
- vision_config (AriaVisionConfig or dict): Configuration for the vision
85
- component.
86
- text_config (AriaMoELMConfig or dict): Configuration for the text
87
- component.
88
- projector_patch_to_query_dict (dict): Mapping of patch sizes to query
89
- dimensions.
90
- ignore_index (int): Index to ignore in loss calculation.
91
- image_token_index (int): Index used to represent image tokens.
92
- **kwargs: Additional keyword arguments passed to the parent class.
93
- Attributes:
94
- model_type (str): Type of the model, set to "aria".
95
- is_composition (bool): Whether the model is a composition of multiple
96
- components.
97
- ignore_index (int): Index to ignore in loss calculation.
98
- image_token_index (int): Index used to represent image tokens.
99
- projector_patch_to_query_dict (dict): Mapping of patch sizes to query
100
- dimensions.
101
- vision_config (AriaVisionConfig): Configuration for the vision
102
- component.
103
- text_config (AriaMoELMConfig): Configuration for the text component.
104
- """
105
-
106
- model_type = "aria"
107
- is_composition = False
108
-
109
- def __init__(
110
- self,
111
- vision_config: AriaVisionConfig = AriaVisionConfig(), # noqa: B008
112
- text_config: AriaMoELMConfig = AriaMoELMConfig(), # noqa: B008
113
- projector_patch_to_query_dict: Mapping[int, int] = {
114
- 1225: 128,
115
- 4900: 256,
116
- },
117
- ignore_index=-100,
118
- image_token_index=32000,
119
- tie_word_embeddings=False,
120
- **kwargs,
121
- ):
122
- super().__init__(**kwargs)
123
- self.ignore_index = ignore_index
124
- self.image_token_index = image_token_index
125
- self.tie_word_embeddings = tie_word_embeddings
126
- attn_implementation = kwargs.pop("attn_implementation", None)
127
-
128
- # Set the default attention implementation to flash_attention_2 if not
129
- # specified
130
- self._attn_implementation = ("flash_attention_2"
131
- if attn_implementation is None else
132
- attn_implementation)
133
-
134
- # Convert the keys and values of projector_patch_to_query_dict to
135
- # integers
136
- # This ensures consistency even if they were provided as strings
137
- self.projector_patch_to_query_dict = {
138
- int(k): int(v)
139
- for k, v in projector_patch_to_query_dict.items()
140
- }
141
-
142
- if isinstance(vision_config, dict) and "model_type" in vision_config:
143
- vision_config = AriaVisionConfig(**vision_config)
144
- if attn_implementation is None:
145
- vision_attn_implementation = "flash_attention_2"
146
- elif attn_implementation == "sdpa":
147
- logger.warning("SDPA is not supported for vit, using "
148
- "flash_attention_2 instead")
149
- vision_attn_implementation = "flash_attention_2"
150
- else:
151
- vision_attn_implementation = attn_implementation
152
- vision_config._attn_implementation = vision_attn_implementation
153
-
154
- self.vision_config = vision_config
155
-
156
- if isinstance(text_config, dict) and "model_type" in text_config:
157
- text_attn_implementation = ("sdpa" if attn_implementation is None
158
- else attn_implementation)
159
- text_config = AriaMoELMConfig(**text_config)
160
- text_config._attn_implementation = text_attn_implementation
161
-
162
- self.text_config = text_config
163
-
164
- # This is needed for the static kv cache
165
- self.num_hidden_layers = self.text_config.num_hidden_layers