@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e303dcf523
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Model] Add Ernie4.5 and Ernie4.5MoE Model Support (#20220)

Signed-off-by: wangyafeng <wangyafeng@baidu.com>

CSWYF3634076 committed Jul 2, 2025 at 18:37 UTC e303dcf523344a982c5e6937ddc8946b2b9860cc
5 files changed +634
docs/models/supported_models.md
+2
@@ -330,6 +330,8 @@ Specified using `--task generate`.
330 | `DeepseekV2ForCausalLM` | DeepSeek-V2 | `deepseek-ai/DeepSeek-V2`, `deepseek-ai/DeepSeek-V2-Chat` etc. | | ✅︎ | ✅︎ |
331 | `DeepseekV3ForCausalLM` | DeepSeek-V3 | `deepseek-ai/DeepSeek-V3-Base`, `deepseek-ai/DeepSeek-V3` etc. | | ✅︎ | ✅︎ |
332 | `Dots1ForCausalLM` | dots.llm1 | `rednote-hilab/dots.llm1.base`, `rednote-hilab/dots.llm1.inst` etc. | | ✅︎ | ✅︎ |
333 +| `Ernie4_5_ForCausalLM` | Ernie4.5 | `baidu/ERNIE-4.5-0.3B-PT`,etc. | | ✅︎ | ✅︎ |
334 +| `Ernie4_5_MoeForCausalLM` | Ernie4.5MoE | `baidu/ERNIE-4.5-21B-A3B-PT`, `baidu/ERNIE-4.5-300B-A47B-PT`, etc. | | ✅︎ | ✅︎ |
335 | `ExaoneForCausalLM` | EXAONE-3 | `LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
336 | `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ | ✅︎ |
337 | `FalconMambaForCausalLM` | FalconMamba | `tiiuae/falcon-mamba-7b`, `tiiuae/falcon-mamba-7b-instruct`, etc. | | ✅︎ | ✅︎ |
tests/models/registry.py
+4
@@ -162,6 +162,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
162 trust_remote_code=True),
163 "DeepseekV3ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3", # noqa: E501
164 trust_remote_code=True),
165 + "Ernie4_5_ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT",
166 + trust_remote_code=True),
167 + "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT",
168 + trust_remote_code=True),
169 "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
170 "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
171 "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
vllm/model_executor/models/ernie45.py new
+43
@@ -0,0 +1,43 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# Copyright 2025 The Baidu team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only Erine model compatible with HuggingFace weights."""
25 +from vllm.config import VllmConfig
26 +from vllm.model_executor.models.llama import LlamaForCausalLM
27 +
28 +from .utils import PPMissingLayer
29 +
30 +
31 +class Ernie4_5_ForCausalLM(LlamaForCausalLM):
32 +
33 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
34 + super().__init__(vllm_config=vllm_config, prefix=prefix)
35 + # Hack Llama model to fit HF format Ernie4.5 dense implementation
36 + # Attention difference between Ernie and Llama:
37 + # 1. rotary_dim and no Neox style.
38 + # 2. There is no bias for o_proj in attention
39 + for layer in self.model.layers:
40 + if not isinstance(layer, PPMissingLayer):
41 + layer.self_attn.rotary_emb.is_neox_style = False
42 + layer.self_attn.o_proj.bias = None
43 + layer.self_attn.o_proj.skip_bias_add = True
vllm/model_executor/models/ernie45_moe.py new
+583
@@ -0,0 +1,583 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# Copyright 2025 The Baidu team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only ErineMoE model compatible with HuggingFace weights."""
25 +from collections.abc import Iterable
26 +from typing import Any, Optional, Union
27 +
28 +import torch
29 +from torch import nn
30 +from transformers import PretrainedConfig
31 +
32 +from vllm.attention import Attention
33 +from vllm.compilation.decorators import support_torch_compile
34 +from vllm.config import CacheConfig, VllmConfig
35 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
36 +from vllm.logger import init_logger
37 +from vllm.model_executor.layers.activation import SiluAndMul
38 +from vllm.model_executor.layers.fused_moe import FusedMoE
39 +from vllm.model_executor.layers.layernorm import RMSNorm
40 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
41 + QKVParallelLinear,
42 + ReplicatedLinear,
43 + RowParallelLinear)
44 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 +from vllm.model_executor.layers.quantization import QuantizationConfig
46 +from vllm.model_executor.layers.rotary_embedding import get_rope
47 +from vllm.model_executor.layers.vocab_parallel_embedding import (
48 + ParallelLMHead, VocabParallelEmbedding)
49 +from vllm.model_executor.model_loader.weight_utils import (
50 + default_weight_loader, maybe_remap_kv_scale_name)
51 +from vllm.model_executor.sampling_metadata import SamplingMetadata
52 +from vllm.sequence import IntermediateTensors
53 +
54 +from .interfaces import SupportsPP
55 +from .utils import (PPMissingLayer, extract_layer_index,
56 + is_pp_missing_parameter,
57 + make_empty_intermediate_tensors_factory, make_layers,
58 + maybe_prefix)
59 +
60 +logger = init_logger(__name__)
61 +
62 +
63 +class Ernie4_5_MoeMLP(nn.Module):
64 +
65 + def __init__(
66 + self,
67 + hidden_size: int,
68 + intermediate_size: int,
69 + hidden_act: str,
70 + use_bias: bool = False,
71 + quant_config: Optional[QuantizationConfig] = None,
72 + reduce_results: bool = True,
73 + prefix: str = "",
74 + ) -> None:
75 + super().__init__()
76 + self.gate_up_proj = MergedColumnParallelLinear(
77 + hidden_size, [intermediate_size] * 2,
78 + bias=use_bias,
79 + quant_config=quant_config,
80 + prefix=f"{prefix}.gate_up_proj")
81 + self.down_proj = RowParallelLinear(intermediate_size,
82 + hidden_size,
83 + bias=use_bias,
84 + quant_config=quant_config,
85 + reduce_results=reduce_results,
86 + prefix=f"{prefix}.down_proj")
87 + if hidden_act != "silu":
88 + raise ValueError(f"Unsupported activation: {hidden_act}. "
89 + "Only silu is supported for now.")
90 + self.act_fn = SiluAndMul()
91 +
92 + def forward(self, x):
93 + gate_up, _ = self.gate_up_proj(x)
94 + x = self.act_fn(gate_up)
95 + x, _ = self.down_proj(x)
96 + return x
97 +
98 +
99 +class Ernie4_5_MoeMoE(nn.Module):
100 +
101 + def __init__(
102 + self,
103 + config: PretrainedConfig,
104 + quant_config: Optional[QuantizationConfig] = None,
105 + prefix: str = "",
106 + ):
107 + super().__init__()
108 +
109 + layer_idx = extract_layer_index(prefix)
110 + self.layer_idx = layer_idx
111 + self.tp_size = get_tensor_model_parallel_world_size()
112 + self.moe_num_shared_experts = getattr(config, "moe_num_shared_experts",
113 + None)
114 +
115 + if self.tp_size > config.moe_num_experts:
116 + raise ValueError(
117 + f"Tensor parallel size {self.tp_size} is greater than "
118 + f"the number of experts {config.moe_num_experts}.")
119 +
120 + self.gate = ReplicatedLinear(config.hidden_size,
121 + config.moe_num_experts,
122 + bias=False,
123 + quant_config=None,
124 + prefix=f"{prefix}.gate")
125 +
126 + self.experts = FusedMoE(num_experts=config.moe_num_experts,
127 + top_k=config.moe_k,
128 + hidden_size=config.hidden_size,
129 + intermediate_size=config.moe_intermediate_size,
130 + reduce_results=False,
131 + renormalize=True,
132 + quant_config=quant_config,
133 + prefix=f"{prefix}.experts")
134 +
135 + if self.moe_num_shared_experts is not None:
136 + intermediate_size = (config.moe_intermediate_size *
137 + config.moe_num_shared_experts)
138 + self.shared_experts = Ernie4_5_MoeMLP(
139 + hidden_size=config.hidden_size,
140 + intermediate_size=intermediate_size,
141 + hidden_act=config.hidden_act,
142 + quant_config=quant_config,
143 + prefix=f"{prefix}.shared_experts",
144 + reduce_results=self.experts.must_reduce_shared_expert_outputs(
145 + ))
146 +
147 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
148 + orig_shape = hidden_states.shape
149 + hidden_dim = hidden_states.shape[-1]
150 + hidden_states = hidden_states.view(-1, hidden_dim)
151 + if self.moe_num_shared_experts is not None:
152 + shared_output = self.shared_experts(hidden_states)
153 +
154 + router_logits, _ = self.gate(hidden_states)
155 +
156 + final_hidden_states = self.experts(hidden_states=hidden_states,
157 + router_logits=router_logits)
158 +
159 + if self.moe_num_shared_experts is not None and \
160 + shared_output is not None:
161 + final_hidden_states = final_hidden_states + shared_output
162 +
163 + if self.tp_size > 1:
164 + final_hidden_states = (
165 + self.experts.maybe_all_reduce_tensor_model_parallel(
166 + final_hidden_states))
167 +
168 + return final_hidden_states.view(orig_shape)
169 +
170 +
171 +class Ernie4_5_MoeAttention(nn.Module):
172 +
173 + def __init__(
174 + self,
175 + hidden_size: int,
176 + num_heads: int,
177 + num_kv_heads: int,
178 + head_dim: Optional[int] = None,
179 + rope_theta: float = 500000,
180 + rope_scaling: Optional[dict[str, Any]] = None,
181 + max_position_embeddings: int = 131072,
182 + rms_norm_eps: float = 1e-05,
183 + qkv_bias: bool = False,
184 + cache_config: Optional[CacheConfig] = None,
185 + quant_config: Optional[QuantizationConfig] = None,
186 + prefix: str = "",
187 + ) -> None:
188 + super().__init__()
189 + layer_idx = extract_layer_index(prefix) if len(prefix) > 0 else 0
190 + self.layer_idx = layer_idx
191 + self.hidden_size = hidden_size
192 + tp_size = get_tensor_model_parallel_world_size()
193 + self.total_num_heads = num_heads
194 + assert self.total_num_heads % tp_size == 0
195 + self.num_heads = self.total_num_heads // tp_size
196 +
197 + self.total_num_kv_heads = num_kv_heads
198 + if self.total_num_kv_heads >= tp_size:
199 + # Number of KV heads is greater than TP size, so we partition
200 + # the KV heads across multiple tensor parallel GPUs.
201 + assert self.total_num_kv_heads % tp_size == 0
202 + else:
203 + # Number of KV heads is less than TP size, so we replicate
204 + # the KV heads across multiple tensor parallel GPUs.
205 + assert tp_size % self.total_num_kv_heads == 0
206 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
207 + self.head_dim = head_dim or (hidden_size // self.total_num_heads)
208 +
209 + self.q_size = self.num_heads * self.head_dim
210 + self.kv_size = self.num_kv_heads * self.head_dim
211 + self.scaling = self.head_dim**-0.5
212 + self.rope_theta = rope_theta
213 + self.max_position_embeddings = max_position_embeddings
214 +
215 + self.qkv_proj = QKVParallelLinear(hidden_size,
216 + self.head_dim,
217 + self.total_num_heads,
218 + self.total_num_kv_heads,
219 + bias=qkv_bias,
220 + quant_config=quant_config,
221 + prefix=f"{prefix}.qkv_proj")
222 +
223 + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
224 + hidden_size,
225 + bias=False,
226 + quant_config=quant_config,
227 + prefix=f"{prefix}.o_proj")
228 +
229 + self.rotary_emb = get_rope(
230 + self.head_dim,
231 + rotary_dim=self.head_dim,
232 + max_position=max_position_embeddings,
233 + base=rope_theta,
234 + is_neox_style=False,
235 + rope_scaling=rope_scaling,
236 + )
237 + self.attn = Attention(self.num_heads,
238 + self.head_dim,
239 + self.scaling,
240 + num_kv_heads=self.num_kv_heads,
241 + cache_config=cache_config,
242 + quant_config=quant_config,
243 + prefix=f"{prefix}.attn")
244 +
245 + def forward(
246 + self,
247 + positions: torch.Tensor,
248 + hidden_states: torch.Tensor,
249 + ) -> torch.Tensor:
250 +
251 + qkv, _ = self.qkv_proj(hidden_states)
252 +
253 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
254 + q, k = self.rotary_emb(positions, q, k)
255 +
256 + # Attention
257 + attn_output = self.attn(q, k, v)
258 + # Output projection
259 + output, _ = self.o_proj(attn_output)
260 + return output
261 +
262 +
263 +class Ernie4_5_MoeDecoderLayer(nn.Module):
264 +
265 + def __init__(
266 + self,
267 + config: PretrainedConfig,
268 + cache_config: Optional[CacheConfig] = None,
269 + quant_config: Optional[QuantizationConfig] = None,
270 + prefix: str = "",
271 + ) -> None:
272 + super().__init__()
273 + self.hidden_size = config.hidden_size
274 + rope_theta = getattr(config, "rope_theta", 500000)
275 + rope_scaling = getattr(config, "rope_scaling", None)
276 + max_position_embeddings = getattr(config, "max_position_embeddings",
277 + 131072)
278 + self.self_attn = Ernie4_5_MoeAttention(
279 + hidden_size=self.hidden_size,
280 + num_heads=config.num_attention_heads,
281 + num_kv_heads=config.num_key_value_heads,
282 + head_dim=getattr(config, 'head_dim', None),
283 + rope_theta=rope_theta,
284 + rope_scaling=rope_scaling,
285 + max_position_embeddings=max_position_embeddings,
286 + rms_norm_eps=config.rms_norm_eps,
287 + qkv_bias=getattr(config, 'use_bias', False),
288 + cache_config=cache_config,
289 + quant_config=quant_config,
290 + prefix=f"{prefix}.self_attn",
291 + )
292 +
293 + layer_idx = extract_layer_index(prefix)
294 + self.layer_idx = layer_idx
295 +
296 + # MoE
297 + moe_num_experts = getattr(config, "moe_num_experts", 0)
298 + moe_layer_start_index = getattr(config, "moe_layer_start_index", 0)
299 + moe_layer_end_index = getattr(config, "moe_layer_end_index",
300 + config.num_hidden_layers - 1)
301 + moe_layer_interval = getattr(config, "moe_layer_interval", 1)
302 + use_moe = getattr(config, "use_moe", moe_num_experts > 0)
303 +
304 + if (use_moe and ((layer_idx + 1) % moe_layer_interval == 0)
305 + and layer_idx >= moe_layer_start_index
306 + and layer_idx <= moe_layer_end_index):
307 + self.mlp = Ernie4_5_MoeMoE(config=config,
308 + quant_config=quant_config,
309 + prefix=f"{prefix}.mlp")
310 + else:
311 + self.mlp = Ernie4_5_MoeMLP(
312 + hidden_size=config.hidden_size,
313 + intermediate_size=config.intermediate_size,
314 + hidden_act=config.hidden_act,
315 + use_bias=getattr(config, 'use_bias', False),
316 + quant_config=quant_config,
317 + prefix=f"{prefix}.mlp")
318 +
319 + self.input_layernorm = RMSNorm(config.hidden_size,
320 + eps=config.rms_norm_eps)
321 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
322 + eps=config.rms_norm_eps)
323 +
324 + def forward(
325 + self,
326 + positions: torch.Tensor,
327 + hidden_states: torch.Tensor,
328 + residual: Optional[torch.Tensor],
329 + ) -> torch.Tensor:
330 +
331 + # Self Attention
332 + if residual is None:
333 + residual = hidden_states
334 + hidden_states = self.input_layernorm(hidden_states)
335 + else:
336 + hidden_states, residual = self.input_layernorm(
337 + hidden_states, residual)
338 +
339 + hidden_states = self.self_attn(
340 + positions=positions,
341 + hidden_states=hidden_states,
342 + )
343 +
344 + # Fully Connected
345 + hidden_states, residual = self.post_attention_layernorm(
346 + hidden_states, residual)
347 +
348 + hidden_states = self.mlp(hidden_states)
349 +
350 + return hidden_states, residual
351 +
352 +
353 +@support_torch_compile
354 +class Ernie4_5_MoeModel(nn.Module):
355 +
356 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
357 + super().__init__()
358 +
359 + config = vllm_config.model_config.hf_config
360 + cache_config = vllm_config.cache_config
361 + quant_config = vllm_config.quant_config
362 +
363 + self.padding_idx = config.pad_token_id
364 + self.vocab_size = config.vocab_size
365 + self.config = config
366 +
367 + if get_pp_group().is_first_rank:
368 + self.embed_tokens = VocabParallelEmbedding(
369 + config.vocab_size,
370 + config.hidden_size,
371 + quant_config=quant_config,
372 + prefix=f"{prefix}.embed_tokens")
373 + else:
374 + self.embed_tokens = PPMissingLayer()
375 +
376 + self.start_layer, self.end_layer, self.layers = make_layers(
377 + config.num_hidden_layers,
378 + lambda prefix: Ernie4_5_MoeDecoderLayer(config=config,
379 + cache_config=cache_config,
380 + quant_config=quant_config,
381 + prefix=prefix),
382 + prefix=f"{prefix}.layers",
383 + )
384 +
385 + if get_pp_group().is_last_rank:
386 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
387 + else:
388 + self.norm = PPMissingLayer()
389 +
390 + self.make_empty_intermediate_tensors = (
391 + make_empty_intermediate_tensors_factory(
392 + ["hidden_states", "residual"], config.hidden_size))
393 +
394 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
395 + return self.embed_tokens(input_ids)
396 +
397 + def forward(
398 + self,
399 + input_ids: torch.Tensor,
400 + positions: torch.Tensor,
401 + intermediate_tensors: Optional[IntermediateTensors] = None,
402 + inputs_embeds: Optional[torch.Tensor] = None,
403 + ) -> Union[torch.Tensor, IntermediateTensors]:
404 +
405 + if get_pp_group().is_first_rank:
406 + if inputs_embeds is not None:
407 + hidden_states = inputs_embeds
408 + else:
409 + hidden_states = self.get_input_embeddings(input_ids)
410 + residual = None
411 + else:
412 + assert intermediate_tensors is not None
413 + hidden_states = intermediate_tensors["hidden_states"]
414 + residual = intermediate_tensors["residual"]
415 +
416 + for i in range(self.start_layer, self.end_layer):
417 + layer = self.layers[i]
418 + hidden_states, residual = layer(positions, hidden_states, residual)
419 +
420 + if not get_pp_group().is_last_rank:
421 + return IntermediateTensors({
422 + "hidden_states": hidden_states,
423 + "residual": residual
424 + })
425 +
426 + hidden_states, _ = self.norm(hidden_states, residual)
427 +
428 + return hidden_states
429 +
430 +
431 +class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP):
432 + packed_modules_mapping = {
433 + "qkv_proj": [
434 + "q_proj",
435 + "k_proj",
436 + "v_proj",
437 + ],
438 + "gate_up_proj": [
439 + "gate_proj",
440 + "up_proj",
441 + ],
442 + }
443 +
444 + fall_back_to_pt_during_load = False
445 +
446 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
447 + super().__init__()
448 + config = vllm_config.model_config.hf_config
449 + quant_config = vllm_config.quant_config
450 + self.config = config
451 + self.quant_config = quant_config
452 + self.model = Ernie4_5_MoeModel(vllm_config=vllm_config,
453 + prefix=maybe_prefix(prefix, "model"))
454 +
455 + if get_pp_group().is_last_rank:
456 + self.lm_head = ParallelLMHead(config.vocab_size,
457 + config.hidden_size,
458 + quant_config=quant_config)
459 + else:
460 + self.lm_head = PPMissingLayer()
461 +
462 + if self.config.tie_word_embeddings:
463 + self.lm_head.weight = self.model.embed_tokens.weight
464 + self.logits_processor = LogitsProcessor(config.vocab_size)
465 + self.make_empty_intermediate_tensors = (
466 + self.model.make_empty_intermediate_tensors)
467 +
468 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
469 + return self.model.get_input_embeddings(input_ids)
470 +
471 + def forward(
472 + self,
473 + input_ids: torch.Tensor,
474 + positions: torch.Tensor,
475 + intermediate_tensors: Optional[IntermediateTensors] = None,
476 + inputs_embeds: Optional[torch.Tensor] = None,
477 + ) -> Union[torch.Tensor, IntermediateTensors]:
478 + hidden_states = self.model(input_ids, positions, intermediate_tensors,
479 + inputs_embeds)
480 + return hidden_states
481 +
482 + def compute_logits(
483 + self,
484 + hidden_states: torch.Tensor,
485 + sampling_metadata: SamplingMetadata,
486 + ) -> Optional[torch.Tensor]:
487 + logits = self.logits_processor(self.lm_head, hidden_states,
488 + sampling_metadata)
489 + return logits
490 +
491 + def load_weights(self, weights: Iterable[tuple[str,
492 + torch.Tensor]]) -> set[str]:
493 + stacked_params_mapping = [
494 + # (param_name, shard_name, shard_id)
495 + ("qkv_proj", "q_proj", "q"),
496 + ("qkv_proj", "k_proj", "k"),
497 + ("qkv_proj", "v_proj", "v"),
498 + ("gate_up_proj", "gate_proj", 0),
499 + ("gate_up_proj", "up_proj", 1),
500 + ]
501 +
502 + # Params for weights, fp8 weight scales, fp8 activation scales
503 + # (param_name, weight_name, expert_id, shard_id)
504 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
505 + ckpt_gate_proj_name="gate_proj",
506 + ckpt_down_proj_name="down_proj",
507 + ckpt_up_proj_name="up_proj",
508 + num_experts=self.config.moe_num_experts)
509 +
510 + params_dict = dict(self.named_parameters())
511 + loaded_params: set[str] = set()
512 + for name, loaded_weight in weights:
513 + if self.config.tie_word_embeddings and name.endswith(
514 + "lm_head.weight"):
515 + continue
516 + # MTP will be supported soon.
517 + if "mtp" in name:
518 + continue
519 +
520 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
521 + # Skip non-stacked layers and experts (experts handled below).
522 + if weight_name not in name:
523 + continue
524 +
525 + if (("mlp.experts." in name) and name not in params_dict):
526 + continue
527 + name = name.replace(weight_name, param_name)
528 + # Skip loading extra bias for GPTQ models.
529 + if ((name.endswith(".bias") or name.endswith("_bias"))
530 + and name not in params_dict):
531 + continue
532 + # Skip layers on other devices.
533 + if is_pp_missing_parameter(name, self):
534 + continue
535 +
536 + param = params_dict[name]
537 + weight_loader = param.weight_loader
538 + weight_loader(param, loaded_weight, shard_id)
539 + break
540 + else:
541 + for mapping in expert_params_mapping:
542 + param_name, weight_name, expert_id, shard_id = mapping
543 +
544 + if weight_name not in name:
545 + continue
546 +
547 + name = name.replace(weight_name, param_name)
548 + # Skip layers on other devices.
549 + if is_pp_missing_parameter(name, self):
550 + continue
551 +
552 + # Skip loading extra bias for GPTQ models.
553 + if ((name.endswith(".bias") or name.endswith("_bias"))
554 + and name not in params_dict):
555 + continue
556 + param = params_dict[name]
557 +
558 + weight_loader = param.weight_loader
559 + weight_loader(param,
560 + loaded_weight,
561 + name,
562 + shard_id=shard_id,
563 + expert_id=expert_id)
564 + break
565 + else:
566 + # Skip loading extra bias for GPTQ models.
567 + if ((name.endswith(".bias") or name.endswith("_bias"))
568 + and name not in params_dict):
569 + continue
570 + # Skip layers on other devices.
571 + if is_pp_missing_parameter(name, self):
572 + continue
573 + # Remapping the name of FP8 kv-scale.
574 + name = maybe_remap_kv_scale_name(name, params_dict)
575 + if name is None:
576 + continue
577 +
578 + param = params_dict[name]
579 + weight_loader = getattr(param, "weight_loader",
580 + default_weight_loader)
581 + weight_loader(param, loaded_weight)
582 + loaded_params.add(name)
583 + return loaded_params
vllm/model_executor/models/registry.py
+2
@@ -53,6 +53,8 @@ _TEXT_GENERATION_MODELS = {
53 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
54 "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"),
55 "Dots1ForCausalLM": ("dots1", "Dots1ForCausalLM"),
56 + "Ernie4_5_ForCausalLM": ("ernie45", "Ernie4_5_ForCausalLM"),
57 + "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
58 "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
59 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
60 "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),