A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model]: Add support for Aria model (#10514)
Signed-off-by: xffxff <1247714429@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
zhou fan committed
Nov 26, 2024 at 02:10 UTC
b1d920531f6d5fd6c020096499c91a8f26620cd6
8 files changed
+791
docs/source/models/supported_models.rst
+6
@@ -476,6 +476,12 @@ Text Generation
476
- Example HF Models
477
- :ref:`LoRA <lora>`
478
- :ref:`PP <distributed_serving>`
479
+ * - :code:`AriaForConditionalGeneration`
480
+ - Aria
481
+ - T + I
482
+ - :code:`rhymes-ai/Aria`
483
+ -
484
+ - ✅︎
485
* - :code:`Blip2ForConditionalGeneration`
486
- BLIP-2
487
- T + I\ :sup:`E`
examples/offline_inference_vision_language.py
+18
@@ -402,6 +402,23 @@ def run_idefics3(question: str, modality: str):
402
return llm, prompt, stop_token_ids
403
404
405
+# Aria
406
+def run_aria(question: str, modality: str):
407
+ assert modality == "image"
408
+ model_name = "rhymes-ai/Aria"
409
+
410
+ llm = LLM(model=model_name,
411
+ tokenizer_mode="slow",
412
+ trust_remote_code=True,
413
+ dtype="bfloat16")
414
+
415
+ prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
416
+ "<|im_end|>\n<|im_start|>assistant\n")
417
+
418
+ stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
419
+ return llm, prompt, stop_token_ids
420
+
421
+
422
model_example_map = {
423
"llava": run_llava,
424
"llava-next": run_llava_next,
@@ -423,6 +440,7 @@ model_example_map = {
440
"molmo": run_molmo,
441
"glm4v": run_glm4v,
442
"idefics3": run_idefics3,
443
+ "aria": run_aria,
444
}
445
446
examples/offline_inference_vision_language_multi_image.py
+20
@@ -321,6 +321,25 @@ def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
321
)
322
323
324
+def load_aria(question, image_urls: List[str]) -> ModelRequestData:
325
+ model_name = "rhymes-ai/Aria"
326
+ llm = LLM(model=model_name,
327
+ tokenizer_mode="slow",
328
+ trust_remote_code=True,
329
+ dtype="bfloat16",
330
+ limit_mm_per_prompt={"image": len(image_urls)})
331
+ placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
332
+ prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
333
+ "<|im_start|>assistant\n")
334
+ stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
335
+ return ModelRequestData(
336
+ llm=llm,
337
+ prompt=prompt,
338
+ stop_token_ids=stop_token_ids,
339
+ image_data=[fetch_image(url) for url in image_urls],
340
+ chat_template=None)
341
+
342
+
343
model_example_map = {
344
"phi3_v": load_phi3v,
345
"h2ovl_chat": load_h2onvl,
@@ -330,6 +349,7 @@ model_example_map = {
349
"qwen_vl_chat": load_qwenvl_chat,
350
"mllama": load_mllama,
351
"idefics3": load_idefics3,
352
+ "aria": load_aria,
353
}
354
355
tests/models/registry.py
+2
@@ -43,6 +43,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
43
trust_remote_code=True),
44
"ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
45
trust_remote_code=True),
46
+ "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
47
+ trust_remote_code=True),
48
"BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
49
trust_remote_code=True),
50
"BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
vllm/entrypoints/chat_utils.py
+2
@@ -412,6 +412,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
412
return ""
413
if model_type == "idefics3":
414
return "<image>"
415
+ if model_type == "aria":
416
+ return "<|fim_prefix|><|img|><|fim_suffix|>"
417
418
raise TypeError(f"Unknown {modality} model type: {model_type}")
419
elif modality == "audio":
vllm/model_executor/models/aria.py
new
+695
@@ -0,0 +1,695 @@
1
+import math
2
+from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
3
+
4
+import torch
5
+import torch.nn as nn
6
+from torch.nn.init import trunc_normal_
7
+from transformers import LlamaConfig
8
+
9
+from vllm.attention import AttentionMetadata
10
+from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11
+from vllm.distributed import get_tensor_model_parallel_rank
12
+from vllm.inputs import INPUT_REGISTRY, token_inputs
13
+from vllm.model_executor.layers.activation import get_act_fn
14
+from vllm.model_executor.layers.fused_moe import FusedMoE
15
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
+ RowParallelLinear)
17
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19
+ get_compressed_tensors_cache_scale)
20
+from vllm.model_executor.layers.sampler import (Sampler, SamplerOutput,
21
+ SamplingMetadata)
22
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23
+from vllm.model_executor.model_loader.weight_utils import (
24
+ default_weight_loader, maybe_remap_kv_scale_name)
25
+from vllm.model_executor.models.idefics2_vision_model import (
26
+ Idefics2VisionTransformer)
27
+from vllm.model_executor.models.interfaces import SupportsMultiModal
28
+from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29
+ LlamaModel)
30
+from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31
+ is_pp_missing_parameter,
32
+ make_layers, maybe_prefix,
33
+ merge_multimodal_embeddings)
34
+from vllm.multimodal import MULTIMODAL_REGISTRY
35
+from vllm.multimodal.base import MultiModalInputs
36
+from vllm.multimodal.image import cached_get_image_processor
37
+from vllm.multimodal.inputs import NestedTensors
38
+from vllm.multimodal.utils import (cached_get_tokenizer,
39
+ repeat_and_pad_placeholder_tokens)
40
+from vllm.sequence import IntermediateTensors
41
+from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
42
+ AriaVisionConfig)
43
+
44
+from .utils import flatten_bn
45
+
46
+
47
+class AriaImagePixelInputs(TypedDict):
48
+ pixel_values: torch.Tensor
49
+ pixel_mask: Optional[torch.Tensor]
50
+ """
51
+ Shape:
52
+ pixel_values: `(batch_size * num_images, num_channels, height, width)`
53
+ pixel_mask: `(batch_size * num_images, height, width)`
54
+ """
55
+
56
+
57
+class AriaVisionTransformer(Idefics2VisionTransformer):
58
+ """
59
+ AriaVisionTransformer is a modified version of Idefics2VisionTransformer
60
+ that replaces the post-layernorm with an identity layer.
61
+ """
62
+
63
+ def __init__(
64
+ self,
65
+ config: AriaVisionConfig,
66
+ quant_config: Optional[QuantizationConfig] = None,
67
+ prefix: str = "",
68
+ ) -> None:
69
+ super().__init__(config, quant_config, prefix)
70
+ self.post_layernorm = nn.Identity()
71
+
72
+
73
+class AriaVisionModel(nn.Module):
74
+ config_class = AriaVisionConfig
75
+
76
+ def __init__(
77
+ self,
78
+ config: AriaVisionConfig,
79
+ quant_config: Optional[QuantizationConfig] = None,
80
+ *,
81
+ prefix: str = "",
82
+ ) -> None:
83
+ super().__init__()
84
+
85
+ self.vision_model = AriaVisionTransformer(
86
+ config,
87
+ quant_config,
88
+ prefix=f"{prefix}.vision_model",
89
+ )
90
+
91
+ def forward(
92
+ self,
93
+ pixel_values: torch.Tensor,
94
+ pixel_mask: Optional[torch.BoolTensor] = None,
95
+ ) -> Tuple[torch.Tensor, Optional[torch.BoolTensor]]:
96
+ patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
97
+
98
+ vit_oup = self.vision_model(
99
+ pixel_values=pixel_values,
100
+ patch_attention_mask=patch_attention_mask,
101
+ )
102
+
103
+ image_atts = self._create_image_attention_mask(patch_attention_mask)
104
+
105
+ return vit_oup, image_atts
106
+
107
+ def _create_patch_attention_mask(self, pixel_mask):
108
+ if pixel_mask is None:
109
+ return None
110
+
111
+ patches_subgrid = pixel_mask.unfold(
112
+ dimension=1,
113
+ size=self.vision_model.config.patch_size,
114
+ step=self.vision_model.config.patch_size,
115
+ ).unfold(
116
+ dimension=2,
117
+ size=self.vision_model.config.patch_size,
118
+ step=self.vision_model.config.patch_size,
119
+ )
120
+ return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
121
+
122
+ def _create_image_attention_mask(self, patch_attention_mask):
123
+ if patch_attention_mask is None:
124
+ return None
125
+
126
+ flattened_mask = patch_attention_mask.flatten(1)
127
+ return torch.logical_not(flattened_mask)
128
+
129
+
130
+class FFN(nn.Module):
131
+
132
+ def __init__(self, embed_dim, ff_dim, output_dim):
133
+ super().__init__()
134
+ self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
135
+ self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
136
+ self.act = get_act_fn("gelu_new")
137
+
138
+ def forward(self, hidden_states):
139
+ hidden_states, _ = self.linear_in(hidden_states)
140
+ hidden_states = self.act(hidden_states)
141
+ hidden_states, _ = self.linear_out(hidden_states)
142
+ return hidden_states
143
+
144
+
145
+class CrossAttention(nn.Module):
146
+
147
+ def __init__(self, kv_dim, embed_dim, num_heads, drop_out_rate=0):
148
+ super().__init__()
149
+ self.num_heads = num_heads
150
+ self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
151
+ self.k_proj = nn.Linear(kv_dim, embed_dim, bias=False)
152
+ self.v_proj = nn.Linear(kv_dim, embed_dim, bias=False)
153
+
154
+ self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
155
+ self.linear = nn.Linear(embed_dim, embed_dim)
156
+ self.dropout = nn.Dropout(drop_out_rate)
157
+
158
+ self.layer_norm = nn.LayerNorm(embed_dim)
159
+ self.ln_kv = nn.LayerNorm(kv_dim)
160
+
161
+ def forward(self, x, hidden_states, attn_mask=None, add_residual=False):
162
+ normed_hidden_states = self.layer_norm(hidden_states)
163
+ query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
164
+
165
+ x = self.ln_kv(x)
166
+ key = self.k_proj(x).permute(1, 0, 2)
167
+ value = self.v_proj(x).permute(1, 0, 2)
168
+
169
+ attn_output, _ = self.multihead_attn(query,
170
+ key,
171
+ value,
172
+ attn_mask=attn_mask)
173
+
174
+ attn_output = attn_output.permute(1, 0, 2)
175
+
176
+ if add_residual:
177
+ attn_output = hidden_states + self.dropout(
178
+ self.linear(attn_output))
179
+ else:
180
+ attn_output = self.dropout(self.linear(attn_output))
181
+
182
+ return attn_output
183
+
184
+
185
+class AriaProjector(nn.Module):
186
+ """
187
+ A projection module with one cross attention layer and one FFN layer, which
188
+ projects ViT's outputs into MoE's inputs.
189
+
190
+ Args:
191
+ patch_to_query_dict (dict): Maps patch numbers to their corresponding
192
+ query numbers,
193
+ e.g., {1225: 128, 4900: 256}. This allows for different query sizes
194
+ based on image resolution.
195
+ embed_dim (int): Embedding dimension.
196
+ num_heads (int): Number of attention heads.
197
+ kv_dim (int): Dimension of key and value.
198
+ ff_dim (int): Hidden dimension of the feed-forward network.
199
+ output_dim (int): Output dimension.
200
+ norm_layer (nn.Module): Normalization layer. Default is nn.LayerNorm.
201
+
202
+ Outputs:
203
+ A tensor with the shape of (batch_size, query_number, output_dim)
204
+ """
205
+
206
+ def __init__(
207
+ self,
208
+ patch_to_query_dict,
209
+ embed_dim,
210
+ num_heads,
211
+ kv_dim,
212
+ ff_dim,
213
+ output_dim,
214
+ norm_layer=nn.LayerNorm,
215
+ ):
216
+ super().__init__()
217
+ self.patch_to_query_dict = patch_to_query_dict
218
+ self.embed_dim = embed_dim
219
+ self.num_heads = num_heads
220
+
221
+ self.query = nn.Parameter(
222
+ torch.zeros(max(patch_to_query_dict.values()), self.embed_dim))
223
+
224
+ trunc_normal_(self.query, std=0.02)
225
+
226
+ self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
227
+
228
+ self.ln_ffn = norm_layer(embed_dim)
229
+ self.ffn = FFN(embed_dim, ff_dim, output_dim)
230
+
231
+ def forward(self, x, attn_mask=None):
232
+ bs = x.shape[0]
233
+ queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
234
+
235
+ query_num = self.patch_to_query_dict.get(x.shape[1], None)
236
+ assert (query_num is not None
237
+ ), f"Query number for {x.shape[1]} patches is not provided"
238
+
239
+ queries = queries[:, :query_num, :]
240
+
241
+ if attn_mask is not None:
242
+ attn_mask = attn_mask.repeat_interleave(self.num_heads, 0)
243
+ attn_mask = attn_mask.unsqueeze(1).expand(-1, queries.size(1), -1)
244
+
245
+ attention_out = self.cross_attn(x, queries, attn_mask=attn_mask)
246
+
247
+ out = self.ffn(self.ln_ffn(attention_out))
248
+
249
+ return out
250
+
251
+
252
+class AriaFusedMoE(FusedMoE):
253
+
254
+ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
255
+ shard_id: str) -> Set[str]:
256
+ # Override the weight_loader to handle the expert weights in the Aria
257
+ # model, which are already packed with experts, and merge the gate and
258
+ # up weights for each expert.
259
+ # Note: Loading expert weights with quantization is not supported
260
+ tp_rank = get_tensor_model_parallel_rank()
261
+ if shard_id == 'w13':
262
+ # the shape of loaded_weight is
263
+ # (num_experts, hidden_size, 2 * moe_intermediate_size)
264
+ if self.tp_size > 1:
265
+ up, gate = loaded_weight.chunk(2, dim=-1)
266
+ up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
267
+ gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
268
+ up_and_gate = torch.cat([up_current_rank, gate_current_rank],
269
+ dim=-1).transpose(1, 2)
270
+ param.data.copy_(up_and_gate)
271
+ else:
272
+ param.data.copy_(loaded_weight.transpose(1, 2))
273
+ elif shard_id == 'w2':
274
+ # the shape of loaded_weight is
275
+ # (num_experts, moe_intermediate_size, hidden_size)
276
+ if self.tp_size > 1:
277
+ down_current_rank = loaded_weight.chunk(self.tp_size,
278
+ dim=1)[tp_rank]
279
+ param.data.copy_(down_current_rank.transpose(1, 2))
280
+ else:
281
+ param.data.copy_(loaded_weight.transpose(1, 2))
282
+
283
+
284
+class MoELayer(nn.Module):
285
+ """
286
+ Mixture of Experts (MoE) Layer for the AriaMoE model.
287
+
288
+ This layer implements the MoE mechanism, which routes input tokens to
289
+ different experts based on a routing algorithm, processes them through the
290
+ experts, and then combines the outputs.
291
+ """
292
+
293
+ def __init__(
294
+ self,
295
+ config: AriaMoELMConfig,
296
+ quant_config: Optional[QuantizationConfig],
297
+ ) -> None:
298
+ super().__init__()
299
+ self.config = config
300
+
301
+ self.router_weight = nn.Parameter(
302
+ torch.empty(
303
+ (self.config.moe_num_experts, self.config.hidden_size)))
304
+
305
+ self.experts = AriaFusedMoE(
306
+ num_experts=config.moe_num_experts,
307
+ top_k=config.moe_topk,
308
+ hidden_size=config.hidden_size,
309
+ intermediate_size=config.moe_intermediate_size,
310
+ quant_config=quant_config,
311
+ reduce_results=True,
312
+ )
313
+ self.shared_experts = LlamaMLP(
314
+ config.hidden_size,
315
+ config.moe_intermediate_size * config.moe_num_shared_experts,
316
+ "silu",
317
+ quant_config=quant_config,
318
+ )
319
+
320
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
321
+ """
322
+ Forward pass of the MoE Layer.
323
+
324
+ Args:
325
+ hidden_states (torch.Tensor): Input tensor of shape (batch_size,
326
+ sequence_length, hidden_size).
327
+
328
+ Returns:
329
+ torch.Tensor: Output tensor after passing through the MoE layer.
330
+ """
331
+
332
+ router_output = torch.nn.functional.linear(hidden_states,
333
+ self.router_weight)
334
+
335
+ shared_expert_output = self.shared_experts(hidden_states)
336
+ sparse_expert_output = self.experts(hidden_states, router_output)
337
+
338
+ return sparse_expert_output + shared_expert_output
339
+
340
+
341
+class MoEDecoderLayer(LlamaDecoderLayer):
342
+ """
343
+ Custom Decoder Layer for the AriaMoE model which modifies the standard
344
+ `LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
345
+ Experts (MoE) Layer.
346
+ """
347
+
348
+ def __init__(
349
+ self,
350
+ config: LlamaConfig,
351
+ cache_config: Optional[CacheConfig] = None,
352
+ quant_config: Optional[QuantizationConfig] = None,
353
+ prefix: str = "",
354
+ ) -> None:
355
+ super().__init__(config, cache_config, quant_config, prefix)
356
+ self.mlp = MoELayer(config, quant_config=quant_config)
357
+
358
+
359
+class AriaMoELMModel(LlamaModel):
360
+ """
361
+ Custom LlamaModel for the AriaMoE model which modifies the standard
362
+ LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
363
+ """
364
+
365
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
366
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
367
+
368
+ config = vllm_config.model_config.hf_config
369
+ cache_config = vllm_config.cache_config
370
+ quant_config = vllm_config.quant_config
371
+
372
+ # FIXME: this is a hack to disable the compilation of the model
373
+ self.do_not_compile = True
374
+
375
+ self.layers = None
376
+
377
+ self.start_layer, self.end_layer, self.layers = make_layers(
378
+ config.num_hidden_layers,
379
+ lambda prefix: MoEDecoderLayer(
380
+ config=config,
381
+ cache_config=cache_config,
382
+ quant_config=quant_config,
383
+ prefix=prefix,
384
+ ),
385
+ prefix=f"{prefix}.layers",
386
+ )
387
+
388
+ # Adapted from LlamaModel.load_weights with the modification of adding
389
+ # the expert weights mapping to `stacked_params_mapping`
390
+ def load_weights(self, weights: Iterable[Tuple[str,
391
+ torch.Tensor]]) -> Set[str]:
392
+ stacked_params_mapping = [
393
+ # (param_name, shard_name, shard_id)
394
+ (".qkv_proj", ".q_proj", "q"),
395
+ (".qkv_proj", ".k_proj", "k"),
396
+ (".qkv_proj", ".v_proj", "v"),
397
+ (".gate_up_proj", ".gate_proj", 0),
398
+ (".gate_up_proj", ".up_proj", 1),
399
+ ("experts.w13_weight", "experts.fc1.weight", 'w13'),
400
+ ("experts.w2_weight", "experts.fc2.weight", 'w2'),
401
+ ]
402
+ params_dict = dict(self.named_parameters())
403
+ loaded_params: Set[str] = set()
404
+ for name, loaded_weight in weights:
405
+ if "rotary_emb.inv_freq" in name:
406
+ continue
407
+ if ("rotary_emb.cos_cached" in name
408
+ or "rotary_emb.sin_cached" in name):
409
+ # Models trained using ColossalAI may include these tensors in
410
+ # the checkpoint. Skip them.
411
+ continue
412
+ if scale_name := get_compressed_tensors_cache_scale(name):
413
+ # Loading kv cache scales for compressed-tensors quantization
414
+ param = params_dict[scale_name]
415
+ weight_loader = getattr(param, "weight_loader",
416
+ default_weight_loader)
417
+ loaded_weight = loaded_weight[0]
418
+ weight_loader(param, loaded_weight)
419
+ loaded_params.add(scale_name)
420
+ continue
421
+ for param_name, weight_name, shard_id in stacked_params_mapping:
422
+ if weight_name not in name:
423
+ continue
424
+ name = name.replace(weight_name, param_name)
425
+ # Skip loading extra bias for GPTQ models.
426
+ if name.endswith(".bias") and name not in params_dict:
427
+ continue
428
+
429
+ if is_pp_missing_parameter(name, self):
430
+ continue
431
+
432
+ param = params_dict[name]
433
+ weight_loader = param.weight_loader
434
+ weight_loader(param, loaded_weight, shard_id)
435
+ break
436
+ else:
437
+ # Skip loading extra bias for GPTQ models.
438
+ if name.endswith(".bias") and name not in params_dict:
439
+ continue
440
+ # Remapping the name of FP8 kv-scale.
441
+ name = maybe_remap_kv_scale_name(name, params_dict)
442
+ if name is None:
443
+ continue
444
+
445
+ if is_pp_missing_parameter(name, self):
446
+ continue
447
+
448
+ param = params_dict[name]
449
+ weight_loader = getattr(param, "weight_loader",
450
+ default_weight_loader)
451
+ weight_loader(param, loaded_weight)
452
+ loaded_params.add(name)
453
+ return loaded_params
454
+
455
+
456
+def build_mm_projector(config):
457
+ return AriaProjector(
458
+ patch_to_query_dict=config.projector_patch_to_query_dict,
459
+ embed_dim=config.vision_config.hidden_size,
460
+ num_heads=config.vision_config.num_attention_heads,
461
+ kv_dim=config.vision_config.hidden_size,
462
+ ff_dim=config.text_config.hidden_size,
463
+ output_dim=config.text_config.hidden_size,
464
+ )
465
+
466
+
467
+def get_max_multimodal_tokens(ctx):
468
+ return max(ctx.model_config.hf_config.image_size2tokens.values())
469
+
470
+
471
+def input_mapper_for_aria(ctx, data):
472
+ return MultiModalInputs(data)
473
+
474
+
475
+def input_processor(ctx, llm_inputs):
476
+ multi_modal_data = llm_inputs.get("multi_modal_data")
477
+ # if it is pure text input, use it as is
478
+ if multi_modal_data is None or "image" not in multi_modal_data:
479
+ return llm_inputs
480
+
481
+ model_config = ctx.model_config
482
+
483
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
484
+ image_processor = cached_get_image_processor(
485
+ model_config.model, trust_remote_code=model_config.trust_remote_code)
486
+ hf_config = model_config.hf_config
487
+
488
+ # prepare image tokens, the max_image_size is used to determine the number
489
+ # of patch_size for every image
490
+ max_image_size = multi_modal_data.pop("max_image_size", 980)
491
+ _split_image = multi_modal_data.pop("split_image", False)
492
+
493
+ assert isinstance(max_image_size,
494
+ (int, float)), "max_image_size should be float or int"
495
+ images = (multi_modal_data["image"] if isinstance(
496
+ multi_modal_data["image"], list) else [multi_modal_data["image"]])
497
+
498
+ image_inputs = image_processor.preprocess(images,
499
+ max_image_size=max_image_size,
500
+ split_image=_split_image,
501
+ return_tensors="pt").data
502
+ image_inputs['pixel_values'] = image_inputs['pixel_values'].to(
503
+ ctx.model_config.dtype)
504
+ num_crops = image_inputs.pop("num_crops")
505
+
506
+ prompt_token_ids = llm_inputs["prompt_token_ids"]
507
+ if num_crops.sum().item() > 0:
508
+ _, prompt_token_ids, _ = repeat_and_pad_placeholder_tokens(
509
+ tokenizer,
510
+ None,
511
+ prompt_token_ids,
512
+ placeholder_token_id=hf_config.image_token_index,
513
+ repeat_count=num_crops,
514
+ )
515
+
516
+ repeat_count = [hf_config.image_size2tokens[max_image_size]
517
+ ] * sum(num_crops).item()
518
+ new_prompt, new_token_ids, _ = repeat_and_pad_placeholder_tokens(
519
+ tokenizer,
520
+ None,
521
+ prompt_token_ids,
522
+ placeholder_token_id=hf_config.image_token_index,
523
+ repeat_count=repeat_count,
524
+ )
525
+
526
+ return token_inputs(
527
+ prompt_token_ids=new_token_ids,
528
+ prompt=new_prompt,
529
+ multi_modal_data={"image": image_inputs},
530
+ )
531
+
532
+
533
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_multimodal_tokens)
534
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_aria)
535
+@INPUT_REGISTRY.register_input_processor(input_processor)
536
+class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
537
+ """
538
+ Aria model for conditional generation tasks.
539
+
540
+ This model combines a vision tower, a multi-modal projector, and a language
541
+ model to perform tasks that involve both image and text inputs.
542
+ """
543
+
544
+ def __init__(
545
+ self,
546
+ vllm_config: VllmConfig,
547
+ prefix: str = "",
548
+ ):
549
+ super().__init__()
550
+ config = vllm_config.model_config.hf_config
551
+ quant_config = vllm_config.quant_config
552
+
553
+ # prepare the image_size to tokens mapping for the image preprocess, see
554
+ # input_processor
555
+ config.image_size2tokens = {
556
+ int(math.sqrt(k) * config.vision_config.patch_size): v
557
+ for k, v in config.projector_patch_to_query_dict.items()
558
+ }
559
+ self.config = config
560
+ self.vision_tower = AriaVisionModel(config.vision_config)
561
+ self.multi_modal_projector = build_mm_projector(config)
562
+ self.vocab_size = config.text_config.vocab_size
563
+ self.language_model = AriaMoELMModel(
564
+ vllm_config=vllm_config.with_hf_config(config.text_config),
565
+ prefix=maybe_prefix(prefix, "language_model.model"),
566
+ )
567
+ self.pad_token_id = (self.config.pad_token_id
568
+ if self.config.pad_token_id is not None else -1)
569
+ self.unpadded_vocab_size = config.text_config.vocab_size
570
+ self.lm_head = ParallelLMHead(
571
+ self.unpadded_vocab_size,
572
+ config.text_config.hidden_size,
573
+ org_num_embeddings=self.language_model.org_vocab_size,
574
+ quant_config=quant_config,
575
+ )
576
+ logit_scale = getattr(config, "logit_scale", 1.0)
577
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
578
+ self.vocab_size, logit_scale)
579
+ self.sampler = Sampler()
580
+
581
+ def _validate_image_sizes(
582
+ self, images: List[torch.Tensor]) -> List[torch.Tensor]:
583
+ if not all(img.shape == images[0].shape for img in images):
584
+ raise ValueError("All images must be the same size")
585
+ return images
586
+
587
+ def _parse_and_validate_image_input(
588
+ self, **kwargs: object) -> Optional[AriaImagePixelInputs]:
589
+ pixel_values = kwargs.pop("pixel_values", None)
590
+ pixel_mask = kwargs.pop("pixel_mask", None)
591
+
592
+ if pixel_values is None:
593
+ return None
594
+
595
+ if not isinstance(pixel_values, (torch.Tensor, list)):
596
+ raise ValueError("Incorrect type of pixel values. "
597
+ f"Got type: {type(pixel_values)}")
598
+
599
+ pixel_values = self._validate_image_sizes(pixel_values)
600
+ pixel_values = flatten_bn(pixel_values, concat=True)
601
+ if pixel_mask is not None:
602
+ pixel_mask = flatten_bn(pixel_mask, concat=True)
603
+
604
+ return AriaImagePixelInputs(
605
+ pixel_values=pixel_values,
606
+ pixel_mask=pixel_mask,
607
+ )
608
+
609
+ def _process_image_input(
610
+ self, image_input: AriaImagePixelInputs
611
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
612
+ assert self.vision_tower is not None
613
+
614
+ pixel_values = image_input['pixel_values']
615
+ pixel_mask = image_input['pixel_mask']
616
+
617
+ image_feature, image_attn_mask = self.vision_tower(
618
+ pixel_values, pixel_mask=pixel_mask)
619
+ return self.multi_modal_projector(image_feature, image_attn_mask)
620
+
621
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
622
+ image_input = self._parse_and_validate_image_input(**kwargs)
623
+ if image_input is None:
624
+ return None
625
+ multimodal_embeddings = self._process_image_input(image_input)
626
+ return multimodal_embeddings
627
+
628
+ def get_input_embeddings(
629
+ self,
630
+ input_ids: torch.Tensor,
631
+ multimodal_embeddings: Optional[NestedTensors] = None,
632
+ ) -> torch.Tensor:
633
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
634
+ if multimodal_embeddings is not None:
635
+ inputs_embeds = merge_multimodal_embeddings(
636
+ input_ids, inputs_embeds, multimodal_embeddings,
637
+ self.config.image_token_index)
638
+ return inputs_embeds
639
+
640
+ def forward(
641
+ self,
642
+ input_ids: torch.Tensor,
643
+ positions: torch.Tensor,
644
+ kv_caches: List[torch.Tensor],
645
+ attn_metadata: AttentionMetadata,
646
+ intermediate_tensors: Optional[IntermediateTensors] = None,
647
+ inputs_embeds: Optional[torch.Tensor] = None,
648
+ **kwargs: object,
649
+ ) -> Union[torch.Tensor, IntermediateTensors]:
650
+ if inputs_embeds is None:
651
+ multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
652
+ # always pass the input via `inputs_embeds`
653
+ # to make sure the computation graph is consistent
654
+ inputs_embeds = self.get_input_embeddings(input_ids,
655
+ multimodal_embeddings)
656
+ input_ids = None
657
+
658
+ hidden_states = self.language_model(
659
+ input_ids,
660
+ positions,
661
+ kv_caches,
662
+ attn_metadata,
663
+ intermediate_tensors,
664
+ inputs_embeds=inputs_embeds,
665
+ )
666
+
667
+ return hidden_states
668
+
669
+ def compute_logits(self, hidden_states: torch.Tensor,
670
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
671
+ logits = self.logits_processor(self.lm_head, hidden_states,
672
+ sampling_metadata)
673
+ return logits
674
+
675
+ def sample(
676
+ self,
677
+ logits: torch.Tensor,
678
+ sampling_metadata: SamplingMetadata,
679
+ ) -> Optional[SamplerOutput]:
680
+ next_tokens = self.sampler(logits, sampling_metadata)
681
+ return next_tokens
682
+
683
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
684
+ hf_to_vllm_mapper = WeightsMapper(
685
+ orig_to_new_prefix={
686
+ "language_model.model": "language_model",
687
+ "language_model.lm_head": "lm_head",
688
+ },
689
+ orig_to_new_suffix={
690
+ "router.weight": "router_weight",
691
+ },
692
+ )
693
+
694
+ loader = AutoWeightsLoader(self)
695
+ loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+1
@@ -133,6 +133,7 @@ _CROSS_ENCODER_MODELS = {
133
134
_MULTIMODAL_MODELS = {
135
# [Decoder-only]
136
+ "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
137
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
138
"ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
139
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
vllm/transformers_utils/configs/aria.py
new
+47
@@ -0,0 +1,47 @@
1
+from transformers.models.idefics2.configuration_idefics2 import (
2
+ Idefics2VisionConfig)
3
+from transformers.models.llama.configuration_llama import LlamaConfig
4
+
5
+
6
+class AriaVisionConfig(Idefics2VisionConfig):
7
+ model_type = "aria_vision_model"
8
+
9
+
10
+class AriaMoELMConfig(LlamaConfig):
11
+ """
12
+ Configuration class for AriaMoE language model.
13
+
14
+ This class extends the LlamaConfig to include additional parameters specific
15
+ to the Mixture of Experts (MoE) architecture.
16
+ """
17
+
18
+ model_type = "aria_moe_lm"
19
+
20
+ def __init__(
21
+ self,
22
+ moe_intermediate_size: int = 4096,
23
+ moe_num_experts: int = 8,
24
+ moe_topk: int = 2,
25
+ moe_num_shared_experts: int = 2,
26
+ **kwargs,
27
+ ):
28
+ """
29
+ Initialize the AriaMoELMConfig.
30
+
31
+ Args:
32
+ moe_intermediate_size (int): The intermediate size for MoE layers.
33
+ Default is 4096.
34
+ moe_num_experts (int): The number of experts in the MoE layer.
35
+ Default is 8.
36
+ moe_topk (int): The number of top experts to route to for each
37
+ token. Default is 2.
38
+ moe_num_shared_experts (int): The number of shared experts. Default
39
+ is 2.
40
+ **kwargs: Additional keyword arguments to be passed to the parent
41
+ LlamaConfig.
42
+ """
43
+ super().__init__(**kwargs)
44
+ self.moe_intermediate_size = moe_intermediate_size
45
+ self.moe_num_experts = moe_num_experts
46
+ self.moe_topk = moe_topk
47
+ self.moe_num_shared_experts = moe_num_shared_experts