@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1d920531f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model]: Add support for Aria model (#10514)

Signed-off-by: xffxff <1247714429@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

zhou fan committed Nov 26, 2024 at 02:10 UTC b1d920531f6d5fd6c020096499c91a8f26620cd6
8 files changed +791
docs/source/models/supported_models.rst
+6
@@ -476,6 +476,12 @@ Text Generation
476 - Example HF Models
477 - :ref:`LoRA <lora>`
478 - :ref:`PP <distributed_serving>`
479 + * - :code:`AriaForConditionalGeneration`
480 + - Aria
481 + - T + I
482 + - :code:`rhymes-ai/Aria`
483 + -
484 + - ✅︎
485 * - :code:`Blip2ForConditionalGeneration`
486 - BLIP-2
487 - T + I\ :sup:`E`
examples/offline_inference_vision_language.py
+18
@@ -402,6 +402,23 @@ def run_idefics3(question: str, modality: str):
402 return llm, prompt, stop_token_ids
403
404
405 +# Aria
406 +def run_aria(question: str, modality: str):
407 + assert modality == "image"
408 + model_name = "rhymes-ai/Aria"
409 +
410 + llm = LLM(model=model_name,
411 + tokenizer_mode="slow",
412 + trust_remote_code=True,
413 + dtype="bfloat16")
414 +
415 + prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
416 + "<|im_end|>\n<|im_start|>assistant\n")
417 +
418 + stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
419 + return llm, prompt, stop_token_ids
420 +
421 +
422 model_example_map = {
423 "llava": run_llava,
424 "llava-next": run_llava_next,
@@ -423,6 +440,7 @@ model_example_map = {
440 "molmo": run_molmo,
441 "glm4v": run_glm4v,
442 "idefics3": run_idefics3,
443 + "aria": run_aria,
444 }
445
446
examples/offline_inference_vision_language_multi_image.py
+20
@@ -321,6 +321,25 @@ def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
321 )
322
323
324 +def load_aria(question, image_urls: List[str]) -> ModelRequestData:
325 + model_name = "rhymes-ai/Aria"
326 + llm = LLM(model=model_name,
327 + tokenizer_mode="slow",
328 + trust_remote_code=True,
329 + dtype="bfloat16",
330 + limit_mm_per_prompt={"image": len(image_urls)})
331 + placeholders = "<fim_prefix><|img|><fim_suffix>\n" * len(image_urls)
332 + prompt = (f"<|im_start|>user\n{placeholders}{question}<|im_end|>\n"
333 + "<|im_start|>assistant\n")
334 + stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
335 + return ModelRequestData(
336 + llm=llm,
337 + prompt=prompt,
338 + stop_token_ids=stop_token_ids,
339 + image_data=[fetch_image(url) for url in image_urls],
340 + chat_template=None)
341 +
342 +
343 model_example_map = {
344 "phi3_v": load_phi3v,
345 "h2ovl_chat": load_h2onvl,
@@ -330,6 +349,7 @@ model_example_map = {
349 "qwen_vl_chat": load_qwenvl_chat,
350 "mllama": load_mllama,
351 "idefics3": load_idefics3,
352 + "aria": load_aria,
353 }
354
355
tests/models/registry.py
+2
@@ -43,6 +43,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
43 trust_remote_code=True),
44 "ArcticForCausalLM": _HfExamplesInfo("Snowflake/snowflake-arctic-instruct",
45 trust_remote_code=True),
46 + "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria",
47 + trust_remote_code=True),
48 "BaiChuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan-7B",
49 trust_remote_code=True),
50 "BaichuanForCausalLM": _HfExamplesInfo("baichuan-inc/Baichuan2-7B-chat",
vllm/entrypoints/chat_utils.py
+2
@@ -412,6 +412,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
412 return ""
413 if model_type == "idefics3":
414 return "<image>"
415 + if model_type == "aria":
416 + return "<|fim_prefix|><|img|><|fim_suffix|>"
417
418 raise TypeError(f"Unknown {modality} model type: {model_type}")
419 elif modality == "audio":
vllm/model_executor/models/aria.py new
+695
@@ -0,0 +1,695 @@
1 +import math
2 +from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
3 +
4 +import torch
5 +import torch.nn as nn
6 +from torch.nn.init import trunc_normal_
7 +from transformers import LlamaConfig
8 +
9 +from vllm.attention import AttentionMetadata
10 +from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11 +from vllm.distributed import get_tensor_model_parallel_rank
12 +from vllm.inputs import INPUT_REGISTRY, token_inputs
13 +from vllm.model_executor.layers.activation import get_act_fn
14 +from vllm.model_executor.layers.fused_moe import FusedMoE
15 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 + RowParallelLinear)
17 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19 + get_compressed_tensors_cache_scale)
20 +from vllm.model_executor.layers.sampler import (Sampler, SamplerOutput,
21 + SamplingMetadata)
22 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23 +from vllm.model_executor.model_loader.weight_utils import (
24 + default_weight_loader, maybe_remap_kv_scale_name)
25 +from vllm.model_executor.models.idefics2_vision_model import (
26 + Idefics2VisionTransformer)
27 +from vllm.model_executor.models.interfaces import SupportsMultiModal
28 +from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29 + LlamaModel)
30 +from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31 + is_pp_missing_parameter,
32 + make_layers, maybe_prefix,
33 + merge_multimodal_embeddings)
34 +from vllm.multimodal import MULTIMODAL_REGISTRY
35 +from vllm.multimodal.base import MultiModalInputs
36 +from vllm.multimodal.image import cached_get_image_processor
37 +from vllm.multimodal.inputs import NestedTensors
38 +from vllm.multimodal.utils import (cached_get_tokenizer,
39 + repeat_and_pad_placeholder_tokens)
40 +from vllm.sequence import IntermediateTensors
41 +from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
42 + AriaVisionConfig)
43 +
44 +from .utils import flatten_bn
45 +
46 +
47 +class AriaImagePixelInputs(TypedDict):
48 + pixel_values: torch.Tensor
49 + pixel_mask: Optional[torch.Tensor]
50 + """
51 + Shape:
52 + pixel_values: `(batch_size * num_images, num_channels, height, width)`
53 + pixel_mask: `(batch_size * num_images, height, width)`
54 + """
55 +
56 +
57 +class AriaVisionTransformer(Idefics2VisionTransformer):
58 + """
59 + AriaVisionTransformer is a modified version of Idefics2VisionTransformer
60 + that replaces the post-layernorm with an identity layer.
61 + """
62 +
63 + def __init__(
64 + self,
65 + config: AriaVisionConfig,
66 + quant_config: Optional[QuantizationConfig] = None,
67 + prefix: str = "",
68 + ) -> None:
69 + super().__init__(config, quant_config, prefix)
70 + self.post_layernorm = nn.Identity()
71 +
72 +
73 +class AriaVisionModel(nn.Module):
74 + config_class = AriaVisionConfig
75 +
76 + def __init__(
77 + self,
78 + config: AriaVisionConfig,
79 + quant_config: Optional[QuantizationConfig] = None,
80 + *,
81 + prefix: str = "",
82 + ) -> None:
83 + super().__init__()
84 +
85 + self.vision_model = AriaVisionTransformer(
86 + config,
87 + quant_config,
88 + prefix=f"{prefix}.vision_model",
89 + )
90 +
91 + def forward(
92 + self,
93 + pixel_values: torch.Tensor,
94 + pixel_mask: Optional[torch.BoolTensor] = None,
95 + ) -> Tuple[torch.Tensor, Optional[torch.BoolTensor]]:
96 + patch_attention_mask = self._create_patch_attention_mask(pixel_mask)
97 +
98 + vit_oup = self.vision_model(
99 + pixel_values=pixel_values,
100 + patch_attention_mask=patch_attention_mask,
101 + )
102 +
103 + image_atts = self._create_image_attention_mask(patch_attention_mask)
104 +
105 + return vit_oup, image_atts
106 +
107 + def _create_patch_attention_mask(self, pixel_mask):
108 + if pixel_mask is None:
109 + return None
110 +
111 + patches_subgrid = pixel_mask.unfold(
112 + dimension=1,
113 + size=self.vision_model.config.patch_size,
114 + step=self.vision_model.config.patch_size,
115 + ).unfold(
116 + dimension=2,
117 + size=self.vision_model.config.patch_size,
118 + step=self.vision_model.config.patch_size,
119 + )
120 + return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
121 +
122 + def _create_image_attention_mask(self, patch_attention_mask):
123 + if patch_attention_mask is None:
124 + return None
125 +
126 + flattened_mask = patch_attention_mask.flatten(1)
127 + return torch.logical_not(flattened_mask)
128 +
129 +
130 +class FFN(nn.Module):
131 +
132 + def __init__(self, embed_dim, ff_dim, output_dim):
133 + super().__init__()
134 + self.linear_in = ColumnParallelLinear(embed_dim, ff_dim, bias=False)
135 + self.linear_out = RowParallelLinear(ff_dim, output_dim, bias=False)
136 + self.act = get_act_fn("gelu_new")
137 +
138 + def forward(self, hidden_states):
139 + hidden_states, _ = self.linear_in(hidden_states)
140 + hidden_states = self.act(hidden_states)
141 + hidden_states, _ = self.linear_out(hidden_states)
142 + return hidden_states
143 +
144 +
145 +class CrossAttention(nn.Module):
146 +
147 + def __init__(self, kv_dim, embed_dim, num_heads, drop_out_rate=0):
148 + super().__init__()
149 + self.num_heads = num_heads
150 + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
151 + self.k_proj = nn.Linear(kv_dim, embed_dim, bias=False)
152 + self.v_proj = nn.Linear(kv_dim, embed_dim, bias=False)
153 +
154 + self.multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
155 + self.linear = nn.Linear(embed_dim, embed_dim)
156 + self.dropout = nn.Dropout(drop_out_rate)
157 +
158 + self.layer_norm = nn.LayerNorm(embed_dim)
159 + self.ln_kv = nn.LayerNorm(kv_dim)
160 +
161 + def forward(self, x, hidden_states, attn_mask=None, add_residual=False):
162 + normed_hidden_states = self.layer_norm(hidden_states)
163 + query = self.q_proj(normed_hidden_states).permute(1, 0, 2)
164 +
165 + x = self.ln_kv(x)
166 + key = self.k_proj(x).permute(1, 0, 2)
167 + value = self.v_proj(x).permute(1, 0, 2)
168 +
169 + attn_output, _ = self.multihead_attn(query,
170 + key,
171 + value,
172 + attn_mask=attn_mask)
173 +
174 + attn_output = attn_output.permute(1, 0, 2)
175 +
176 + if add_residual:
177 + attn_output = hidden_states + self.dropout(
178 + self.linear(attn_output))
179 + else:
180 + attn_output = self.dropout(self.linear(attn_output))
181 +
182 + return attn_output
183 +
184 +
185 +class AriaProjector(nn.Module):
186 + """
187 + A projection module with one cross attention layer and one FFN layer, which
188 + projects ViT's outputs into MoE's inputs.
189 +
190 + Args:
191 + patch_to_query_dict (dict): Maps patch numbers to their corresponding
192 + query numbers,
193 + e.g., {1225: 128, 4900: 256}. This allows for different query sizes
194 + based on image resolution.
195 + embed_dim (int): Embedding dimension.
196 + num_heads (int): Number of attention heads.
197 + kv_dim (int): Dimension of key and value.
198 + ff_dim (int): Hidden dimension of the feed-forward network.
199 + output_dim (int): Output dimension.
200 + norm_layer (nn.Module): Normalization layer. Default is nn.LayerNorm.
201 +
202 + Outputs:
203 + A tensor with the shape of (batch_size, query_number, output_dim)
204 + """
205 +
206 + def __init__(
207 + self,
208 + patch_to_query_dict,
209 + embed_dim,
210 + num_heads,
211 + kv_dim,
212 + ff_dim,
213 + output_dim,
214 + norm_layer=nn.LayerNorm,
215 + ):
216 + super().__init__()
217 + self.patch_to_query_dict = patch_to_query_dict
218 + self.embed_dim = embed_dim
219 + self.num_heads = num_heads
220 +
221 + self.query = nn.Parameter(
222 + torch.zeros(max(patch_to_query_dict.values()), self.embed_dim))
223 +
224 + trunc_normal_(self.query, std=0.02)
225 +
226 + self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
227 +
228 + self.ln_ffn = norm_layer(embed_dim)
229 + self.ffn = FFN(embed_dim, ff_dim, output_dim)
230 +
231 + def forward(self, x, attn_mask=None):
232 + bs = x.shape[0]
233 + queries = self.query.unsqueeze(0).repeat(bs, 1, 1)
234 +
235 + query_num = self.patch_to_query_dict.get(x.shape[1], None)
236 + assert (query_num is not None
237 + ), f"Query number for {x.shape[1]} patches is not provided"
238 +
239 + queries = queries[:, :query_num, :]
240 +
241 + if attn_mask is not None:
242 + attn_mask = attn_mask.repeat_interleave(self.num_heads, 0)
243 + attn_mask = attn_mask.unsqueeze(1).expand(-1, queries.size(1), -1)
244 +
245 + attention_out = self.cross_attn(x, queries, attn_mask=attn_mask)
246 +
247 + out = self.ffn(self.ln_ffn(attention_out))
248 +
249 + return out
250 +
251 +
252 +class AriaFusedMoE(FusedMoE):
253 +
254 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
255 + shard_id: str) -> Set[str]:
256 + # Override the weight_loader to handle the expert weights in the Aria
257 + # model, which are already packed with experts, and merge the gate and
258 + # up weights for each expert.
259 + # Note: Loading expert weights with quantization is not supported
260 + tp_rank = get_tensor_model_parallel_rank()
261 + if shard_id == 'w13':
262 + # the shape of loaded_weight is
263 + # (num_experts, hidden_size, 2 * moe_intermediate_size)
264 + if self.tp_size > 1:
265 + up, gate = loaded_weight.chunk(2, dim=-1)
266 + up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
267 + gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
268 + up_and_gate = torch.cat([up_current_rank, gate_current_rank],
269 + dim=-1).transpose(1, 2)
270 + param.data.copy_(up_and_gate)
271 + else:
272 + param.data.copy_(loaded_weight.transpose(1, 2))
273 + elif shard_id == 'w2':
274 + # the shape of loaded_weight is
275 + # (num_experts, moe_intermediate_size, hidden_size)
276 + if self.tp_size > 1:
277 + down_current_rank = loaded_weight.chunk(self.tp_size,
278 + dim=1)[tp_rank]
279 + param.data.copy_(down_current_rank.transpose(1, 2))
280 + else:
281 + param.data.copy_(loaded_weight.transpose(1, 2))
282 +
283 +
284 +class MoELayer(nn.Module):
285 + """
286 + Mixture of Experts (MoE) Layer for the AriaMoE model.
287 +
288 + This layer implements the MoE mechanism, which routes input tokens to
289 + different experts based on a routing algorithm, processes them through the
290 + experts, and then combines the outputs.
291 + """
292 +
293 + def __init__(
294 + self,
295 + config: AriaMoELMConfig,
296 + quant_config: Optional[QuantizationConfig],
297 + ) -> None:
298 + super().__init__()
299 + self.config = config
300 +
301 + self.router_weight = nn.Parameter(
302 + torch.empty(
303 + (self.config.moe_num_experts, self.config.hidden_size)))
304 +
305 + self.experts = AriaFusedMoE(
306 + num_experts=config.moe_num_experts,
307 + top_k=config.moe_topk,
308 + hidden_size=config.hidden_size,
309 + intermediate_size=config.moe_intermediate_size,
310 + quant_config=quant_config,
311 + reduce_results=True,
312 + )
313 + self.shared_experts = LlamaMLP(
314 + config.hidden_size,
315 + config.moe_intermediate_size * config.moe_num_shared_experts,
316 + "silu",
317 + quant_config=quant_config,
318 + )
319 +
320 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
321 + """
322 + Forward pass of the MoE Layer.
323 +
324 + Args:
325 + hidden_states (torch.Tensor): Input tensor of shape (batch_size,
326 + sequence_length, hidden_size).
327 +
328 + Returns:
329 + torch.Tensor: Output tensor after passing through the MoE layer.
330 + """
331 +
332 + router_output = torch.nn.functional.linear(hidden_states,
333 + self.router_weight)
334 +
335 + shared_expert_output = self.shared_experts(hidden_states)
336 + sparse_expert_output = self.experts(hidden_states, router_output)
337 +
338 + return sparse_expert_output + shared_expert_output
339 +
340 +
341 +class MoEDecoderLayer(LlamaDecoderLayer):
342 + """
343 + Custom Decoder Layer for the AriaMoE model which modifies the standard
344 + `LlamaDecoderLayer` by replacing the traditional MLP with a Mixture of
345 + Experts (MoE) Layer.
346 + """
347 +
348 + def __init__(
349 + self,
350 + config: LlamaConfig,
351 + cache_config: Optional[CacheConfig] = None,
352 + quant_config: Optional[QuantizationConfig] = None,
353 + prefix: str = "",
354 + ) -> None:
355 + super().__init__(config, cache_config, quant_config, prefix)
356 + self.mlp = MoELayer(config, quant_config=quant_config)
357 +
358 +
359 +class AriaMoELMModel(LlamaModel):
360 + """
361 + Custom LlamaModel for the AriaMoE model which modifies the standard
362 + LlamaModel by replacing the `LlamaDecoderLayer` with `MoEDecoderLayer`.
363 + """
364 +
365 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
366 + super().__init__(vllm_config=vllm_config, prefix=prefix)
367 +
368 + config = vllm_config.model_config.hf_config
369 + cache_config = vllm_config.cache_config
370 + quant_config = vllm_config.quant_config
371 +
372 + # FIXME: this is a hack to disable the compilation of the model
373 + self.do_not_compile = True
374 +
375 + self.layers = None
376 +
377 + self.start_layer, self.end_layer, self.layers = make_layers(
378 + config.num_hidden_layers,
379 + lambda prefix: MoEDecoderLayer(
380 + config=config,
381 + cache_config=cache_config,
382 + quant_config=quant_config,
383 + prefix=prefix,
384 + ),
385 + prefix=f"{prefix}.layers",
386 + )
387 +
388 + # Adapted from LlamaModel.load_weights with the modification of adding
389 + # the expert weights mapping to `stacked_params_mapping`
390 + def load_weights(self, weights: Iterable[Tuple[str,
391 + torch.Tensor]]) -> Set[str]:
392 + stacked_params_mapping = [
393 + # (param_name, shard_name, shard_id)
394 + (".qkv_proj", ".q_proj", "q"),
395 + (".qkv_proj", ".k_proj", "k"),
396 + (".qkv_proj", ".v_proj", "v"),
397 + (".gate_up_proj", ".gate_proj", 0),
398 + (".gate_up_proj", ".up_proj", 1),
399 + ("experts.w13_weight", "experts.fc1.weight", 'w13'),
400 + ("experts.w2_weight", "experts.fc2.weight", 'w2'),
401 + ]
402 + params_dict = dict(self.named_parameters())
403 + loaded_params: Set[str] = set()
404 + for name, loaded_weight in weights:
405 + if "rotary_emb.inv_freq" in name:
406 + continue
407 + if ("rotary_emb.cos_cached" in name
408 + or "rotary_emb.sin_cached" in name):
409 + # Models trained using ColossalAI may include these tensors in
410 + # the checkpoint. Skip them.
411 + continue
412 + if scale_name := get_compressed_tensors_cache_scale(name):
413 + # Loading kv cache scales for compressed-tensors quantization
414 + param = params_dict[scale_name]
415 + weight_loader = getattr(param, "weight_loader",
416 + default_weight_loader)
417 + loaded_weight = loaded_weight[0]
418 + weight_loader(param, loaded_weight)
419 + loaded_params.add(scale_name)
420 + continue
421 + for param_name, weight_name, shard_id in stacked_params_mapping:
422 + if weight_name not in name:
423 + continue
424 + name = name.replace(weight_name, param_name)
425 + # Skip loading extra bias for GPTQ models.
426 + if name.endswith(".bias") and name not in params_dict:
427 + continue
428 +
429 + if is_pp_missing_parameter(name, self):
430 + continue
431 +
432 + param = params_dict[name]
433 + weight_loader = param.weight_loader
434 + weight_loader(param, loaded_weight, shard_id)
435 + break
436 + else:
437 + # Skip loading extra bias for GPTQ models.
438 + if name.endswith(".bias") and name not in params_dict:
439 + continue
440 + # Remapping the name of FP8 kv-scale.
441 + name = maybe_remap_kv_scale_name(name, params_dict)
442 + if name is None:
443 + continue
444 +
445 + if is_pp_missing_parameter(name, self):
446 + continue
447 +
448 + param = params_dict[name]
449 + weight_loader = getattr(param, "weight_loader",
450 + default_weight_loader)
451 + weight_loader(param, loaded_weight)
452 + loaded_params.add(name)
453 + return loaded_params
454 +
455 +
456 +def build_mm_projector(config):
457 + return AriaProjector(
458 + patch_to_query_dict=config.projector_patch_to_query_dict,
459 + embed_dim=config.vision_config.hidden_size,
460 + num_heads=config.vision_config.num_attention_heads,
461 + kv_dim=config.vision_config.hidden_size,
462 + ff_dim=config.text_config.hidden_size,
463 + output_dim=config.text_config.hidden_size,
464 + )
465 +
466 +
467 +def get_max_multimodal_tokens(ctx):
468 + return max(ctx.model_config.hf_config.image_size2tokens.values())
469 +
470 +
471 +def input_mapper_for_aria(ctx, data):
472 + return MultiModalInputs(data)
473 +
474 +
475 +def input_processor(ctx, llm_inputs):
476 + multi_modal_data = llm_inputs.get("multi_modal_data")
477 + # if it is pure text input, use it as is
478 + if multi_modal_data is None or "image" not in multi_modal_data:
479 + return llm_inputs
480 +
481 + model_config = ctx.model_config
482 +
483 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
484 + image_processor = cached_get_image_processor(
485 + model_config.model, trust_remote_code=model_config.trust_remote_code)
486 + hf_config = model_config.hf_config
487 +
488 + # prepare image tokens, the max_image_size is used to determine the number
489 + # of patch_size for every image
490 + max_image_size = multi_modal_data.pop("max_image_size", 980)
491 + _split_image = multi_modal_data.pop("split_image", False)
492 +
493 + assert isinstance(max_image_size,
494 + (int, float)), "max_image_size should be float or int"
495 + images = (multi_modal_data["image"] if isinstance(
496 + multi_modal_data["image"], list) else [multi_modal_data["image"]])
497 +
498 + image_inputs = image_processor.preprocess(images,
499 + max_image_size=max_image_size,
500 + split_image=_split_image,
501 + return_tensors="pt").data
502 + image_inputs['pixel_values'] = image_inputs['pixel_values'].to(
503 + ctx.model_config.dtype)
504 + num_crops = image_inputs.pop("num_crops")
505 +
506 + prompt_token_ids = llm_inputs["prompt_token_ids"]
507 + if num_crops.sum().item() > 0:
508 + _, prompt_token_ids, _ = repeat_and_pad_placeholder_tokens(
509 + tokenizer,
510 + None,
511 + prompt_token_ids,
512 + placeholder_token_id=hf_config.image_token_index,
513 + repeat_count=num_crops,
514 + )
515 +
516 + repeat_count = [hf_config.image_size2tokens[max_image_size]
517 + ] * sum(num_crops).item()
518 + new_prompt, new_token_ids, _ = repeat_and_pad_placeholder_tokens(
519 + tokenizer,
520 + None,
521 + prompt_token_ids,
522 + placeholder_token_id=hf_config.image_token_index,
523 + repeat_count=repeat_count,
524 + )
525 +
526 + return token_inputs(
527 + prompt_token_ids=new_token_ids,
528 + prompt=new_prompt,
529 + multi_modal_data={"image": image_inputs},
530 + )
531 +
532 +
533 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_multimodal_tokens)
534 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_aria)
535 +@INPUT_REGISTRY.register_input_processor(input_processor)
536 +class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
537 + """
538 + Aria model for conditional generation tasks.
539 +
540 + This model combines a vision tower, a multi-modal projector, and a language
541 + model to perform tasks that involve both image and text inputs.
542 + """
543 +
544 + def __init__(
545 + self,
546 + vllm_config: VllmConfig,
547 + prefix: str = "",
548 + ):
549 + super().__init__()
550 + config = vllm_config.model_config.hf_config
551 + quant_config = vllm_config.quant_config
552 +
553 + # prepare the image_size to tokens mapping for the image preprocess, see
554 + # input_processor
555 + config.image_size2tokens = {
556 + int(math.sqrt(k) * config.vision_config.patch_size): v
557 + for k, v in config.projector_patch_to_query_dict.items()
558 + }
559 + self.config = config
560 + self.vision_tower = AriaVisionModel(config.vision_config)
561 + self.multi_modal_projector = build_mm_projector(config)
562 + self.vocab_size = config.text_config.vocab_size
563 + self.language_model = AriaMoELMModel(
564 + vllm_config=vllm_config.with_hf_config(config.text_config),
565 + prefix=maybe_prefix(prefix, "language_model.model"),
566 + )
567 + self.pad_token_id = (self.config.pad_token_id
568 + if self.config.pad_token_id is not None else -1)
569 + self.unpadded_vocab_size = config.text_config.vocab_size
570 + self.lm_head = ParallelLMHead(
571 + self.unpadded_vocab_size,
572 + config.text_config.hidden_size,
573 + org_num_embeddings=self.language_model.org_vocab_size,
574 + quant_config=quant_config,
575 + )
576 + logit_scale = getattr(config, "logit_scale", 1.0)
577 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
578 + self.vocab_size, logit_scale)
579 + self.sampler = Sampler()
580 +
581 + def _validate_image_sizes(
582 + self, images: List[torch.Tensor]) -> List[torch.Tensor]:
583 + if not all(img.shape == images[0].shape for img in images):
584 + raise ValueError("All images must be the same size")
585 + return images
586 +
587 + def _parse_and_validate_image_input(
588 + self, **kwargs: object) -> Optional[AriaImagePixelInputs]:
589 + pixel_values = kwargs.pop("pixel_values", None)
590 + pixel_mask = kwargs.pop("pixel_mask", None)
591 +
592 + if pixel_values is None:
593 + return None
594 +
595 + if not isinstance(pixel_values, (torch.Tensor, list)):
596 + raise ValueError("Incorrect type of pixel values. "
597 + f"Got type: {type(pixel_values)}")
598 +
599 + pixel_values = self._validate_image_sizes(pixel_values)
600 + pixel_values = flatten_bn(pixel_values, concat=True)
601 + if pixel_mask is not None:
602 + pixel_mask = flatten_bn(pixel_mask, concat=True)
603 +
604 + return AriaImagePixelInputs(
605 + pixel_values=pixel_values,
606 + pixel_mask=pixel_mask,
607 + )
608 +
609 + def _process_image_input(
610 + self, image_input: AriaImagePixelInputs
611 + ) -> Tuple[torch.Tensor, torch.Tensor]:
612 + assert self.vision_tower is not None
613 +
614 + pixel_values = image_input['pixel_values']
615 + pixel_mask = image_input['pixel_mask']
616 +
617 + image_feature, image_attn_mask = self.vision_tower(
618 + pixel_values, pixel_mask=pixel_mask)
619 + return self.multi_modal_projector(image_feature, image_attn_mask)
620 +
621 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
622 + image_input = self._parse_and_validate_image_input(**kwargs)
623 + if image_input is None:
624 + return None
625 + multimodal_embeddings = self._process_image_input(image_input)
626 + return multimodal_embeddings
627 +
628 + def get_input_embeddings(
629 + self,
630 + input_ids: torch.Tensor,
631 + multimodal_embeddings: Optional[NestedTensors] = None,
632 + ) -> torch.Tensor:
633 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
634 + if multimodal_embeddings is not None:
635 + inputs_embeds = merge_multimodal_embeddings(
636 + input_ids, inputs_embeds, multimodal_embeddings,
637 + self.config.image_token_index)
638 + return inputs_embeds
639 +
640 + def forward(
641 + self,
642 + input_ids: torch.Tensor,
643 + positions: torch.Tensor,
644 + kv_caches: List[torch.Tensor],
645 + attn_metadata: AttentionMetadata,
646 + intermediate_tensors: Optional[IntermediateTensors] = None,
647 + inputs_embeds: Optional[torch.Tensor] = None,
648 + **kwargs: object,
649 + ) -> Union[torch.Tensor, IntermediateTensors]:
650 + if inputs_embeds is None:
651 + multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
652 + # always pass the input via `inputs_embeds`
653 + # to make sure the computation graph is consistent
654 + inputs_embeds = self.get_input_embeddings(input_ids,
655 + multimodal_embeddings)
656 + input_ids = None
657 +
658 + hidden_states = self.language_model(
659 + input_ids,
660 + positions,
661 + kv_caches,
662 + attn_metadata,
663 + intermediate_tensors,
664 + inputs_embeds=inputs_embeds,
665 + )
666 +
667 + return hidden_states
668 +
669 + def compute_logits(self, hidden_states: torch.Tensor,
670 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
671 + logits = self.logits_processor(self.lm_head, hidden_states,
672 + sampling_metadata)
673 + return logits
674 +
675 + def sample(
676 + self,
677 + logits: torch.Tensor,
678 + sampling_metadata: SamplingMetadata,
679 + ) -> Optional[SamplerOutput]:
680 + next_tokens = self.sampler(logits, sampling_metadata)
681 + return next_tokens
682 +
683 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
684 + hf_to_vllm_mapper = WeightsMapper(
685 + orig_to_new_prefix={
686 + "language_model.model": "language_model",
687 + "language_model.lm_head": "lm_head",
688 + },
689 + orig_to_new_suffix={
690 + "router.weight": "router_weight",
691 + },
692 + )
693 +
694 + loader = AutoWeightsLoader(self)
695 + loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/registry.py
+1
@@ -133,6 +133,7 @@ _CROSS_ENCODER_MODELS = {
133
134 _MULTIMODAL_MODELS = {
135 # [Decoder-only]
136 + "AriaForConditionalGeneration": ("aria", "AriaForConditionalGeneration"),
137 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
138 "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
139 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
vllm/transformers_utils/configs/aria.py new
+47
@@ -0,0 +1,47 @@
1 +from transformers.models.idefics2.configuration_idefics2 import (
2 + Idefics2VisionConfig)
3 +from transformers.models.llama.configuration_llama import LlamaConfig
4 +
5 +
6 +class AriaVisionConfig(Idefics2VisionConfig):
7 + model_type = "aria_vision_model"
8 +
9 +
10 +class AriaMoELMConfig(LlamaConfig):
11 + """
12 + Configuration class for AriaMoE language model.
13 +
14 + This class extends the LlamaConfig to include additional parameters specific
15 + to the Mixture of Experts (MoE) architecture.
16 + """
17 +
18 + model_type = "aria_moe_lm"
19 +
20 + def __init__(
21 + self,
22 + moe_intermediate_size: int = 4096,
23 + moe_num_experts: int = 8,
24 + moe_topk: int = 2,
25 + moe_num_shared_experts: int = 2,
26 + **kwargs,
27 + ):
28 + """
29 + Initialize the AriaMoELMConfig.
30 +
31 + Args:
32 + moe_intermediate_size (int): The intermediate size for MoE layers.
33 + Default is 4096.
34 + moe_num_experts (int): The number of experts in the MoE layer.
35 + Default is 8.
36 + moe_topk (int): The number of top experts to route to for each
37 + token. Default is 2.
38 + moe_num_shared_experts (int): The number of shared experts. Default
39 + is 2.
40 + **kwargs: Additional keyword arguments to be passed to the parent
41 + LlamaConfig.
42 + """
43 + super().__init__(**kwargs)
44 + self.moe_intermediate_size = moe_intermediate_size
45 + self.moe_num_experts = moe_num_experts
46 + self.moe_topk = moe_topk
47 + self.moe_num_shared_experts = moe_num_shared_experts