@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 528dbcac7d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model][Bugfix]: correct Aria model output (#12309)

Signed-off-by: xffxff <1247714429@qq.com>

zhou fan committed Jan 22, 2025 at 19:39 UTC 528dbcac7d042d84b5541a1fba968087b7dc2866
2 files changed +54 -3
examples/offline_inference/vision_language.py
+2 -1
@@ -28,9 +28,10 @@ def run_aria(question: str, modality: str):
28 llm = LLM(model=model_name,
29 max_model_len=4096,
30 max_num_seqs=2,
31 + dtype="bfloat16",
32 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
33
33 - prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
34 + prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>{question}"
35 "<|im_end|>\n<|im_start|>assistant\n")
36
37 stop_token_ids = [93532, 93653, 944, 93421, 1019, 93653, 93519]
vllm/model_executor/models/aria.py
+52 -2
@@ -30,6 +30,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
31
32 # yapf: disable
33 +from .idefics2_vision_model import Idefics2VisionConfig
34 from .idefics2_vision_model import (
35 Idefics2VisionTransformer as Idefics3VisionTransformer)
36 # yapf: enable
@@ -50,6 +51,53 @@ class AriaImagePixelInputs(TypedDict):
51 """
52
53
54 +class AriaVisionTransformer(Idefics3VisionTransformer):
55 +
56 + def __init__(
57 + self,
58 + config: Idefics2VisionConfig,
59 + quant_config: Optional[QuantizationConfig] = None,
60 + prefix: str = "",
61 + ) -> None:
62 + super().__init__(config, quant_config, prefix)
63 + # Unlike Idefics3VisionTransformer which uses LayerNorm after the
64 + # final layer, Aria omits this normalization, so we replace it with an
65 + # Identity layer
66 + self.post_layernorm = nn.Identity()
67 +
68 + def load_weights(self, weights: Iterable[Tuple[str,
69 + torch.Tensor]]) -> Set[str]:
70 + stacked_params_mapping = [
71 + # (param_name, shard_name, shard_id)
72 + ("qkv_proj", "q_proj", "q"),
73 + ("qkv_proj", "k_proj", "k"),
74 + ("qkv_proj", "v_proj", "v"),
75 + ]
76 + params_dict = dict(self.named_parameters())
77 + loaded_params: Set[str] = set()
78 + for name, loaded_weight in weights:
79 +
80 + # NOTE: post_layernorm is not used in Aria
81 + if "post_layernorm" in name:
82 + continue
83 +
84 + for param_name, weight_name, shard_id in stacked_params_mapping:
85 + if weight_name not in name:
86 + continue
87 + name = name.replace(weight_name, param_name)
88 + param = params_dict[name]
89 + weight_loader = param.weight_loader
90 + weight_loader(param, loaded_weight, shard_id)
91 + break
92 + else:
93 + param = params_dict[name]
94 + weight_loader = getattr(param, "weight_loader",
95 + default_weight_loader)
96 + weight_loader(param, loaded_weight)
97 + loaded_params.add(name)
98 + return loaded_params
99 +
100 +
101 class AriaProjectorMLP(nn.Module):
102
103 def __init__(
@@ -228,8 +276,10 @@ class AriaTextMoELayer(nn.Module):
276 router_output = torch.nn.functional.linear(hidden_states,
277 self.router_weight)
278
279 + hidden_states_copy = hidden_states.clone()
280 + # NOTE: hidden_states will be modified inplace by `FusedMoE`
281 sparse_expert_output = self.experts(hidden_states, router_output)
232 - shared_expert_output = self.shared_experts(hidden_states)
282 + shared_expert_output = self.shared_experts(hidden_states_copy)
283
284 return sparse_expert_output + shared_expert_output
285
@@ -445,7 +495,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
495 quant_config = vllm_config.quant_config
496
497 self.config = config
448 - self.vision_tower = Idefics3VisionTransformer(
498 + self.vision_tower = AriaVisionTransformer(
499 config.vision_config,
500 quant_config,
501 prefix=f"{prefix}.vision_tower",