@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3e04107d97
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 01:00 UTC

[Model] EXAONE 4.0 model support (#21060)

Signed-off-by: Deepfocused <rlawhdrhs27@gmail.com> Signed-off-by: woongsik <rlawhdrhs27@gmail.com>

김종곤 committed Jul 19, 2025 at 15:25 UTC 3e04107d97aeb6360fcfb684665b66c94135079b
7 files changed +809 -3
docs/models/supported_models.md
+1
@@ -331,6 +331,7 @@ Specified using `--task generate`.
331 | `Ernie4_5_ForCausalLM` | Ernie4.5 | `baidu/ERNIE-4.5-0.3B-PT`, etc. | ✅︎ | ✅︎ | ✅︎ |
332 | `Ernie4_5_MoeForCausalLM` | Ernie4.5MoE | `baidu/ERNIE-4.5-21B-A3B-PT`, `baidu/ERNIE-4.5-300B-A47B-PT`, etc. |✅︎| ✅︎ | ✅︎ |
333 | `ExaoneForCausalLM` | EXAONE-3 | `LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
334 +| `Exaone4ForCausalLM` | EXAONE-4 | `LGAI-EXAONE/EXAONE-4.0-32B`, etc. | ✅︎ | ✅︎ | ✅︎ |
335 | `Fairseq2LlamaForCausalLM` | Llama (fairseq2 format) | `mgleize/fairseq2-dummy-Llama-3.2-1B`, etc. | ✅︎ | ✅︎ | ✅︎ |
336 | `FalconForCausalLM` | Falcon | `tiiuae/falcon-7b`, `tiiuae/falcon-40b`, `tiiuae/falcon-rw-7b`, etc. | | ✅︎ | ✅︎ |
337 | `FalconMambaForCausalLM` | FalconMamba | `tiiuae/falcon-mamba-7b`, `tiiuae/falcon-mamba-7b-instruct`, etc. | | ✅︎ | ✅︎ |
tests/models/registry.py
+1
@@ -169,6 +169,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
169 "Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT",
170 trust_remote_code=True),
171 "ExaoneForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct"), # noqa: E501
172 + "Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"), # noqa: E501
173 "Fairseq2LlamaForCausalLM": _HfExamplesInfo("mgleize/fairseq2-dummy-Llama-3.2-1B"), # noqa: E501
174 "FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
175 "FalconH1ForCausalLM":_HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base",
vllm/model_executor/models/exaone4.py new
+547
@@ -0,0 +1,547 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501
4 +
5 +# Adapted from
6 +# https://github.com/lgai-exaone/transformers/blob/add-exaone4/src/transformers/models/exaone4/modeling_exaone4.py
7 +# Copyright 2025 The LG CNS Gen AI Solution Delivery Team.
8 +# Copyright 2025 The LG AI Research and HuggingFace Inc. team. All rights reserved.
9 +#
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only Exaone model compatible with HuggingFace weights."""
23 +
24 +from collections.abc import Iterable
25 +from typing import Any, Optional, Union
26 +
27 +import torch
28 +from torch import nn
29 +
30 +from vllm.attention import Attention
31 +from vllm.compilation.decorators import support_torch_compile
32 +from vllm.config import CacheConfig, VllmConfig
33 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
34 +from vllm.model_executor.layers.activation import SiluAndMul
35 +from vllm.model_executor.layers.layernorm import RMSNorm
36 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
37 + QKVParallelLinear,
38 + RowParallelLinear)
39 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 +from vllm.model_executor.layers.quantization import QuantizationConfig
41 +from vllm.model_executor.layers.rotary_embedding import get_rope
42 +from vllm.model_executor.layers.vocab_parallel_embedding import (
43 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
44 +from vllm.model_executor.model_loader.weight_utils import (
45 + default_weight_loader, maybe_remap_kv_scale_name)
46 +from vllm.model_executor.sampling_metadata import SamplingMetadata
47 +from vllm.sequence import IntermediateTensors
48 +from vllm.transformers_utils.configs.exaone4 import Exaone4Config
49 +
50 +from .interfaces import SupportsLoRA, SupportsPP
51 +from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,
52 + is_pp_missing_parameter,
53 + make_empty_intermediate_tensors_factory, make_layers,
54 + maybe_prefix)
55 +
56 +
57 +class Exaone4GatedMLP(nn.Module):
58 +
59 + def __init__(
60 + self,
61 + hidden_size: int,
62 + intermediate_size: int,
63 + hidden_act: str,
64 + quant_config: Optional[QuantizationConfig] = None,
65 + bias: bool = False,
66 + prefix: str = "",
67 + ) -> None:
68 + super().__init__()
69 + self.gate_up_proj = MergedColumnParallelLinear(
70 + input_size=hidden_size,
71 + output_sizes=[intermediate_size] * 2,
72 + bias=bias,
73 + quant_config=quant_config,
74 + prefix=f"{prefix}.gate_up_proj",
75 + )
76 + self.down_proj = RowParallelLinear(
77 + input_size=intermediate_size,
78 + output_size=hidden_size,
79 + bias=bias,
80 + quant_config=quant_config,
81 + prefix=f"{prefix}.down_proj",
82 + )
83 + if hidden_act != "silu":
84 + raise ValueError(f"Unsupported activation: {hidden_act}. "
85 + "Only silu is supported for now.")
86 + self.act_fn = SiluAndMul()
87 +
88 + def forward(self, x):
89 + gate_up, _ = self.gate_up_proj(x)
90 + x = self.act_fn(gate_up)
91 + x, _ = self.down_proj(x)
92 + return x
93 +
94 +
95 +class Exaone4Attention(nn.Module):
96 +
97 + def __init__(
98 + self,
99 + config: Exaone4Config,
100 + hidden_size: int,
101 + num_heads: int,
102 + num_kv_heads: int,
103 + rope_theta: float = 1000000,
104 + rope_scaling: Optional[dict[str, Any]] = None,
105 + max_position_embeddings: int = 8192,
106 + quant_config: Optional[QuantizationConfig] = None,
107 + bias: bool = False,
108 + cache_config: Optional[CacheConfig] = None,
109 + prefix: str = "",
110 + ) -> None:
111 + super().__init__()
112 + self.hidden_size = hidden_size
113 + tp_size = get_tensor_model_parallel_world_size()
114 + self.total_num_heads = num_heads
115 + assert self.total_num_heads % tp_size == 0
116 + self.num_heads = self.total_num_heads // tp_size
117 + self.total_num_kv_heads = num_kv_heads
118 + if self.total_num_kv_heads >= tp_size:
119 + # Number of KV heads is greater than TP size, so we partition
120 + # the KV heads across multiple tensor parallel GPUs.
121 + assert self.total_num_kv_heads % tp_size == 0
122 + else:
123 + # Number of KV heads is less than TP size, so we replicate
124 + # the KV heads across multiple tensor parallel GPUs.
125 + assert tp_size % self.total_num_kv_heads == 0
126 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
127 + # MistralConfig has an optional head_dim introduced by Mistral-Nemo
128 + self.head_dim = getattr(config, "head_dim", None)
129 + if self.head_dim is None:
130 + self.head_dim = self.hidden_size // self.total_num_heads
131 + self.q_size = self.num_heads * self.head_dim
132 + self.kv_size = self.num_kv_heads * self.head_dim
133 + self.scaling = self.head_dim**-0.5
134 + self.rope_theta = rope_theta
135 + self.max_position_embeddings = max_position_embeddings
136 +
137 + self.qkv_proj = QKVParallelLinear(
138 + hidden_size=hidden_size,
139 + head_size=self.head_dim,
140 + total_num_heads=self.total_num_heads,
141 + total_num_kv_heads=self.total_num_kv_heads,
142 + bias=bias,
143 + quant_config=quant_config,
144 + prefix=f"{prefix}.qkv_proj",
145 + )
146 +
147 + self.o_proj = RowParallelLinear(
148 + input_size=self.total_num_heads * self.head_dim,
149 + output_size=hidden_size,
150 + bias=bias,
151 + quant_config=quant_config,
152 + prefix=f"{prefix}.o_proj",
153 + )
154 +
155 + self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
156 + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
157 +
158 + is_neox_style = True
159 + if quant_config is not None and quant_config.get_name() == "gguf":
160 + is_neox_style = False
161 +
162 + self.apply_all_layers = False # apply rotary embeddings to every layer.
163 + layer_idx = extract_layer_index(prefix)
164 + interleaved_sliding_window = getattr(config,
165 + "interleaved_sliding_window",
166 + 4096)
167 + sliding_window_pattern = getattr(config, "sliding_window_pattern",
168 + "LLLG")
169 +
170 + if sliding_window_pattern:
171 + layer_has_sliding_window = (
172 + layer_idx + 1) % sliding_window_pattern.__len__() != 0
173 + else:
174 + layer_has_sliding_window = False
175 + self.apply_all_layers = True
176 +
177 + if layer_has_sliding_window:
178 + self.sliding_window = interleaved_sliding_window
179 + else:
180 + self.sliding_window = None
181 +
182 + self.rotary_emb = get_rope(
183 + self.head_dim,
184 + rotary_dim=self.head_dim,
185 + max_position=max_position_embeddings,
186 + base=rope_theta,
187 + rope_scaling=rope_scaling,
188 + is_neox_style=is_neox_style,
189 + )
190 + self.attn = Attention(
191 + self.num_heads,
192 + self.head_dim,
193 + self.scaling,
194 + num_kv_heads=self.num_kv_heads,
195 + cache_config=cache_config,
196 + quant_config=quant_config,
197 + per_layer_sliding_window=self.sliding_window,
198 + prefix=f"{prefix}.attn",
199 + )
200 +
201 + def forward(
202 + self,
203 + positions: torch.Tensor,
204 + hidden_states: torch.Tensor,
205 + ) -> torch.Tensor:
206 + qkv, _ = self.qkv_proj(hidden_states)
207 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
208 +
209 + q = q.unflatten(-1, (self.num_heads, self.head_dim))
210 + q = self.q_norm(q)
211 + q = q.flatten(-2, -1)
212 + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
213 + k = self.k_norm(k)
214 + k = k.flatten(-2, -1)
215 +
216 + if self.sliding_window or self.apply_all_layers:
217 + q, k = self.rotary_emb(positions, q, k)
218 + attn_output = self.attn(q, k, v)
219 + output, _ = self.o_proj(attn_output)
220 + return output
221 +
222 +
223 +class Exaone4DecoderLayer(nn.Module):
224 +
225 + def __init__(
226 + self,
227 + config: Exaone4Config,
228 + cache_config: Optional[CacheConfig] = None,
229 + quant_config: Optional[QuantizationConfig] = None,
230 + prefix: str = "",
231 + ) -> None:
232 + super().__init__()
233 + self.hidden_size = config.hidden_size
234 + rope_theta = getattr(config, "rope_theta", 1000000)
235 + rope_scaling = getattr(config, "rope_scaling", None)
236 + if rope_scaling is not None and getattr(
237 + config, "original_max_position_embeddings", None):
238 + rope_scaling["original_max_position_embeddings"] = (
239 + config.original_max_position_embeddings)
240 + max_position_embeddings = getattr(config, "max_position_embeddings",
241 + 8192)
242 + # Support abacusai/Smaug-72B-v0.1 with attention_bias
243 + # Support internlm/internlm-7b with bias
244 + attention_bias = getattr(config, "attention_bias", False) or getattr(
245 + config, "bias", False)
246 +
247 + self.self_attn = Exaone4Attention(
248 + config=config,
249 + hidden_size=self.hidden_size,
250 + num_heads=config.num_attention_heads,
251 + num_kv_heads=getattr(config, "num_key_value_heads",
252 + config.num_attention_heads),
253 + rope_theta=rope_theta,
254 + rope_scaling=rope_scaling,
255 + max_position_embeddings=max_position_embeddings,
256 + quant_config=quant_config,
257 + bias=attention_bias,
258 + cache_config=cache_config,
259 + prefix=f"{prefix}.self_attn",
260 + )
261 + self.mlp = Exaone4GatedMLP(
262 + hidden_size=self.hidden_size,
263 + intermediate_size=config.intermediate_size,
264 + hidden_act=config.hidden_act,
265 + quant_config=quant_config,
266 + bias=getattr(config, "mlp_bias", False),
267 + prefix=f"{prefix}.mlp",
268 + )
269 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
270 + eps=config.rms_norm_eps)
271 + self.post_feedforward_layernorm = RMSNorm(config.hidden_size,
272 + eps=config.rms_norm_eps)
273 +
274 + def forward(
275 + self,
276 + positions: torch.Tensor,
277 + hidden_states: torch.Tensor,
278 + residual: Optional[torch.Tensor],
279 + ) -> tuple[torch.Tensor, torch.Tensor]:
280 + residual = hidden_states
281 +
282 + # Self Attention
283 + hidden_states = self.self_attn(
284 + positions=positions,
285 + hidden_states=hidden_states,
286 + )
287 +
288 + # Use post-LN
289 + hidden_states = self.post_attention_layernorm(hidden_states)
290 + hidden_states = residual + hidden_states
291 +
292 + residual = hidden_states
293 +
294 + # Fully Connected
295 + hidden_states = self.mlp(hidden_states)
296 +
297 + # Use post-LN
298 + hidden_states = self.post_feedforward_layernorm(hidden_states)
299 + hidden_states = residual + hidden_states
300 +
301 + return hidden_states, residual
302 +
303 +
304 +@support_torch_compile
305 +class Exaone4Model(nn.Module):
306 +
307 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
308 + super().__init__()
309 +
310 + config = vllm_config.model_config.hf_config
311 + cache_config = vllm_config.cache_config
312 + quant_config = vllm_config.quant_config
313 + lora_config = vllm_config.lora_config
314 +
315 + self.config = config
316 + self.quant_config = quant_config
317 + lora_vocab = ((lora_config.lora_extra_vocab_size *
318 + (lora_config.max_loras or 1)) if lora_config else 0)
319 + self.vocab_size = config.vocab_size + lora_vocab
320 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
321 + and get_pp_group().is_last_rank):
322 + self.embed_tokens = VocabParallelEmbedding(
323 + self.vocab_size,
324 + config.hidden_size,
325 + org_num_embeddings=config.vocab_size,
326 + quant_config=quant_config,
327 + )
328 + else:
329 + self.embed_tokens = PPMissingLayer()
330 + self.start_layer, self.end_layer, self.layers = make_layers(
331 + config.num_hidden_layers,
332 + lambda prefix: Exaone4DecoderLayer(
333 + config=config,
334 + cache_config=cache_config,
335 + quant_config=quant_config,
336 + prefix=prefix,
337 + ),
338 + prefix=f"{prefix}.layers",
339 + )
340 + if get_pp_group().is_last_rank:
341 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
342 + else:
343 + self.norm = PPMissingLayer()
344 +
345 + self.make_empty_intermediate_tensors = (
346 + make_empty_intermediate_tensors_factory(
347 + ["hidden_states", "residual"], config.hidden_size))
348 +
349 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
350 + return self.embed_tokens(input_ids)
351 +
352 + def forward(
353 + self,
354 + input_ids: Optional[torch.Tensor],
355 + positions: torch.Tensor,
356 + intermediate_tensors: Optional[IntermediateTensors],
357 + inputs_embeds: Optional[torch.Tensor] = None,
358 + ) -> Union[torch.Tensor, IntermediateTensors]:
359 + if get_pp_group().is_first_rank:
360 + if inputs_embeds is not None:
361 + hidden_states = inputs_embeds
362 + else:
363 + hidden_states = self.get_input_embeddings(input_ids)
364 + residual = None
365 + else:
366 + assert intermediate_tensors is not None
367 + hidden_states = intermediate_tensors["hidden_states"]
368 + residual = intermediate_tensors["residual"]
369 +
370 + for layer in self.layers[self.start_layer:self.end_layer]:
371 + hidden_states, residual = layer(
372 + positions,
373 + hidden_states,
374 + residual,
375 + )
376 +
377 + if not get_pp_group().is_last_rank:
378 + return IntermediateTensors({
379 + "hidden_states": hidden_states,
380 + "residual": residual
381 + })
382 +
383 + hidden_states = self.norm(hidden_states)
384 + return hidden_states
385 +
386 + def load_weights(self, weights: Iterable[tuple[str,
387 + torch.Tensor]]) -> set[str]:
388 + stacked_params_mapping = [
389 + # (param_name, shard_name, shard_id)
390 + (".qkv_proj", ".q_proj", "q"),
391 + (".qkv_proj", ".k_proj", "k"),
392 + (".qkv_proj", ".v_proj", "v"),
393 + (".gate_up_proj", ".gate_proj", 0),
394 + (".gate_up_proj", ".up_proj", 1),
395 + ]
396 + params_dict = dict(self.named_parameters())
397 + loaded_params: set[str] = set()
398 + for name, loaded_weight in weights:
399 + if "rotary_emb.inv_freq" in name:
400 + continue
401 + if ("rotary_emb.cos_cached" in name
402 + or "rotary_emb.sin_cached" in name):
403 + # Models trained using ColossalAI may include these tensors in
404 + # the checkpoint. Skip them.
405 + continue
406 + if (self.quant_config is not None and
407 + (scale_name := self.quant_config.get_cache_scale(name))):
408 + # Loading kv cache quantization scales
409 + param = params_dict[scale_name]
410 + weight_loader = getattr(param, "weight_loader",
411 + default_weight_loader)
412 + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
413 + loaded_weight[0])
414 + weight_loader(param, loaded_weight)
415 + loaded_params.add(scale_name)
416 + continue
417 + for param_name, weight_name, shard_id in stacked_params_mapping:
418 + if weight_name not in name:
419 + continue
420 + name = name.replace(weight_name, param_name)
421 + # Skip loading extra bias for GPTQ models.
422 + if name.endswith(".bias") and name not in params_dict:
423 + continue
424 +
425 + if is_pp_missing_parameter(name, self):
426 + continue
427 +
428 + param = params_dict[name]
429 + weight_loader = param.weight_loader
430 + weight_loader(param, loaded_weight, shard_id)
431 +
432 + break
433 + else:
434 + # Skip loading extra bias for GPTQ models.
435 + if name.endswith(".bias") and name not in params_dict:
436 + continue
437 + # Remapping the name of FP8 kv-scale.
438 + name = maybe_remap_kv_scale_name(name, params_dict)
439 + if name is None:
440 + continue
441 +
442 + if is_pp_missing_parameter(name, self):
443 + continue
444 +
445 + param = params_dict[name]
446 + weight_loader = getattr(param, "weight_loader",
447 + default_weight_loader)
448 + weight_loader(param, loaded_weight)
449 + loaded_params.add(name)
450 + return loaded_params
451 +
452 +
453 +class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
454 + packed_modules_mapping = {
455 + "qkv_proj": [
456 + "q_proj",
457 + "k_proj",
458 + "v_proj",
459 + ],
460 + "gate_up_proj": [
461 + "gate_proj",
462 + "up_proj",
463 + ],
464 + }
465 +
466 + # LoRA specific attributes
467 + embedding_modules = {
468 + "embed_tokens": "input_embeddings",
469 + "lm_head": "output_embeddings",
470 + }
471 + embedding_padding_modules = ["lm_head"]
472 +
473 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
474 + super().__init__()
475 + config = vllm_config.model_config.hf_config
476 + quant_config = vllm_config.quant_config
477 + lora_config = vllm_config.lora_config
478 +
479 + self.config = config
480 + self.lora_config = lora_config
481 + self.quant_config = quant_config
482 +
483 + self.model = Exaone4Model(
484 + vllm_config=vllm_config,
485 + prefix=maybe_prefix(prefix, "model"),
486 + )
487 + if get_pp_group().is_last_rank:
488 + self.unpadded_vocab_size = config.vocab_size
489 + if lora_config:
490 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
491 + self.lm_head = ParallelLMHead(
492 + self.unpadded_vocab_size,
493 + config.hidden_size,
494 + org_num_embeddings=config.vocab_size,
495 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
496 + # We need bigger padding if using lora for kernel
497 + # compatibility
498 + if not lora_config else lora_config.lora_vocab_padding_size,
499 + quant_config=quant_config,
500 + )
501 + if config.tie_word_embeddings:
502 + self.lm_head.weight = self.model.embed_tokens.weight
503 +
504 + logit_scale = getattr(config, "logit_scale", 1.0)
505 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
506 + config.vocab_size,
507 + logit_scale)
508 + else:
509 + self.lm_head = PPMissingLayer()
510 +
511 + self.make_empty_intermediate_tensors = (
512 + self.model.make_empty_intermediate_tensors)
513 +
514 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
515 + return self.model.get_input_embeddings(input_ids)
516 +
517 + def forward(
518 + self,
519 + input_ids: torch.Tensor,
520 + positions: torch.Tensor,
521 + intermediate_tensors: Optional[IntermediateTensors] = None,
522 + inputs_embeds: Optional[torch.Tensor] = None,
523 + ) -> Union[torch.Tensor, IntermediateTensors]:
524 + model_output = self.model(input_ids, positions, intermediate_tensors,
525 + inputs_embeds)
526 + return model_output
527 +
528 + def compute_logits(
529 + self,
530 + hidden_states: torch.Tensor,
531 + sampling_metadata: SamplingMetadata,
532 + ) -> Optional[torch.Tensor]:
533 + logits = self.logits_processor(self.lm_head, hidden_states,
534 + sampling_metadata)
535 + return logits
536 +
537 + def load_weights(self, weights: Iterable[tuple[str,
538 + torch.Tensor]]) -> set[str]:
539 + loader = AutoWeightsLoader(
540 + self,
541 + # With tie_word_embeddings, we can skip lm_head.weight
542 + # The weight might appear unnecessarily in the files if the model is
543 + # processed with quantization, LoRA, fine-tuning, etc.
544 + skip_prefixes=(["lm_head."]
545 + if self.config.tie_word_embeddings else None),
546 + )
547 + return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -57,6 +57,7 @@ _TEXT_GENERATION_MODELS = {
57 "Ernie4_5_ForCausalLM": ("ernie45", "Ernie4_5_ForCausalLM"),
58 "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"),
59 "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
60 + "Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
61 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
62 "Fairseq2LlamaForCausalLM": ("fairseq2_llama", "Fairseq2LlamaForCausalLM"),
63 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
vllm/transformers_utils/config.py
+5 -3
@@ -31,9 +31,10 @@ from vllm.logger import init_logger
31 # yapf: disable
32 from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
33 DbrxConfig, DeepseekVLV2Config,
34 - EAGLEConfig, ExaoneConfig,
35 - JAISConfig, KimiVLConfig,
36 - MedusaConfig, MiniMaxText01Config,
34 + EAGLEConfig, Exaone4Config,
35 + ExaoneConfig, JAISConfig,
36 + KimiVLConfig, MedusaConfig,
37 + MiniMaxText01Config,
38 MiniMaxVL01Config, MllamaConfig,
39 MLPSpeculatorConfig, MPTConfig,
40 NemotronConfig, NVLM_D_Config,
@@ -87,6 +88,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
88 "medusa": MedusaConfig,
89 "eagle": EAGLEConfig,
90 "exaone": ExaoneConfig,
91 + "exaone4": Exaone4Config,
92 "minimax_text_01": MiniMaxText01Config,
93 "minimax_vl_01": MiniMaxVL01Config,
94 "nemotron": NemotronConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -7,6 +7,7 @@ from vllm.transformers_utils.configs.dbrx import DbrxConfig
7 from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
8 from vllm.transformers_utils.configs.eagle import EAGLEConfig
9 from vllm.transformers_utils.configs.exaone import ExaoneConfig
10 +from vllm.transformers_utils.configs.exaone4 import Exaone4Config
11 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
12 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
13 # `FalconConfig` class from the official HuggingFace transformers library.
@@ -40,6 +41,7 @@ __all__ = [
41 "MedusaConfig",
42 "EAGLEConfig",
43 "ExaoneConfig",
44 + "Exaone4Config",
45 "MiniMaxText01Config",
46 "MiniMaxVL01Config",
47 "MllamaConfig",
vllm/transformers_utils/configs/exaone4.py new
+252
@@ -0,0 +1,252 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# ruff: noqa: E501
4 +
5 +# Copied from
6 +# https://github.com/lgai-exaone/transformers/blob/add-exaone4/src/transformers/models/exaone4/configuration_exaone4.py
7 +# Copyright 2025 The LG CNS Gen AI Solution Delivery Team.
8 +# Copyright 2025 The LG AI Research and HuggingFace Inc. team. All rights reserved.
9 +#
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +from transformers.configuration_utils import (PretrainedConfig,
23 + layer_type_validation)
24 +from transformers.utils import logging
25 +
26 +logger = logging.get_logger(__name__)
27 +
28 +
29 +def check_is_sliding(config, layer_idx):
30 + """
31 + Check if the current layer is a sliding window attention (local attention) layer.
32 + """
33 + if config.sliding_window is None:
34 + return False
35 + if config.layer_types is not None:
36 + return config.layer_types[layer_idx] == "sliding_attention"
37 + if isinstance(config.sliding_window_pattern, int):
38 + return ((layer_idx + 1) % config.sliding_window_pattern) != 0
39 + elif isinstance(config.sliding_window_pattern, str):
40 + assert isinstance(config.sliding_window, int), (
41 + f"Sliding window must be positive integer, but got {config.sliding_window}"
42 + )
43 + return (layer_idx != config.num_hidden_layers - 1
44 + and config.sliding_window_pattern[layer_idx % len(
45 + config.sliding_window_pattern)] == "L")
46 + else:
47 + logger.warning_once(
48 + "Sliding window is set, but none of `sliding_window_pattern` or `layer_types` is set. "
49 + "Defaulting to use 'full_attention' for all layers.")
50 + return False
51 +
52 +
53 +class Exaone4Config(PretrainedConfig):
54 + r"""
55 + This is the configuration class to store the configuration of a [`Exaone4Model`]. It is used to
56 + instantiate a EXAONE 4.0 model according to the specified arguments, defining the model architecture. Instantiating a
57 + configuration with the defaults will yield a similar configuration to that of the EXAONE-4.0-Instruct [LGAI-EXAONE/EXAONE-4.0-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-Instruct)
58 + NOTE: `EXAONE-4.0-Instruct` is a placeholder model ID. The exact model ID will be updated in the future.
59 +
60 + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
61 + outputs. Read the documentation from [`PretrainedConfig`] for more information.
62 +
63 + Args:
64 + vocab_size (`int`, *optional*, defaults to 102400):
65 + Vocabulary size of the EXAONE 4.0 model. Defines the number of different tokens that can be represented by the
66 + `inputs_ids` passed when calling [`Exaone4Model`].
67 + hidden_size (`int`, *optional*, defaults to 4096):
68 + Dimension of the hidden representations.
69 + intermediate_size (`int`, *optional*, defaults to `hidden_size * 4`):
70 + Dimensionality of the MLP representations.
71 + num_hidden_layers (`int`, *optional*, defaults to 32):
72 + Number of hidden layers in the Transformer encoder.
73 + num_attention_heads (`int`, *optional*, defaults to 32):
74 + Number of attention heads for each attention layer in the Transformer decoder.
75 + num_key_value_heads (`int`, *optional*):
76 + This is the number of key_value heads that should be used to implement Grouped Query Attention. If
77 + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
78 + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
79 + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
80 + by meanpooling all the original heads within that group. For more details checkout [this
81 + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
82 + `num_attention_heads`.
83 + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
84 + The non-linear activation function (function or string) in the decoder.
85 + max_position_embeddings (`int`, *optional*, defaults to 2048):
86 + The maximum sequence length that this model might ever be used with. Typically set this to something large
87 + just in case (e.g., 32768 for EXAONE 3.5).
88 + initializer_range (`float`, *optional*, defaults to 0.02):
89 + The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
90 + rms_norm_eps (`float`, *optional*, defaults to 1e-05):
91 + The epsilon used by the layer normalization layers.
92 + use_cache (`bool`, *optional*, defaults to `True`):
93 + Whether or not the model should return the last key/values attentions (not used by all models). Only
94 + relevant if ``config.is_decoder=True``.
95 + bos_token_id (`int`, *optional*, defaults to 0):
96 + Beginning of stream token id.
97 + eos_token_id (`int`, *optional*, defaults to 2):
98 + End of stream token id.
99 + tie_word_embeddings (`bool`, *optional*, defaults to `False`):
100 + Whether to tie weight embeddings
101 + rope_theta (`float`, *optional*, defaults to 10000.0):
102 + The base period of the RoPE embeddings.
103 + rope_scaling (`Dict`, *optional*):
104 + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
105 + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
106 + accordingly.
107 + Expected contents:
108 + `rope_type` (`str`):
109 + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
110 + 'llama3'], with 'default' being the original RoPE implementation.
111 + `factor` (`float`, *optional*):
112 + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
113 + most scaling types, a `factor` of x will enable the model to handle sequences of length x *
114 + original maximum pre-trained length.
115 + `original_max_position_embeddings` (`int`, *optional*):
116 + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
117 + pretraining.
118 + `attention_factor` (`float`, *optional*):
119 + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
120 + computation. If unspecified, it defaults to value recommended by the implementation, using the
121 + `factor` field to infer the suggested value.
122 + `beta_fast` (`float`, *optional*):
123 + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
124 + ramp function. If unspecified, it defaults to 32.
125 + `beta_slow` (`float`, *optional*):
126 + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
127 + ramp function. If unspecified, it defaults to 1.
128 + `short_factor` (`List[float]`, *optional*):
129 + Only used with 'longrope'. The scaling factor to be applied to short contexts (<
130 + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
131 + size divided by the number of attention heads divided by 2
132 + `long_factor` (`List[float]`, *optional*):
133 + Only used with 'longrope'. The scaling factor to be applied to long contexts (<
134 + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
135 + size divided by the number of attention heads divided by 2
136 + `low_freq_factor` (`float`, *optional*):
137 + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
138 + `high_freq_factor` (`float`, *optional*):
139 + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
140 + attention_dropout (`float`, *optional*, defaults to 0.0):
141 + The dropout ratio for the attention probabilities.
142 + sliding_window (`int`, *optional*):
143 + The size of the sliding window for the sliding window attention.
144 + sliding_window_pattern (`str`, *optional*):
145 + The pattern to use for sliding window attention. Can be one of:
146 + - `None`: No sliding window attention is used
147 + - `int`: Every `sliding_window` layers, use global attention, else use local attention.
148 + - `str`: A sequence of "L" (local attention) and "G" (global attention) characters that defines the
149 + attention pattern. The pattern starts from layer 0 and repeats every `sliding_window` layers. The
150 + final layer always uses global attention regardless of the pattern.
151 + For instance, sliding_window_pattern="LLLG" same as sliding_window=4, which means:
152 + - Layer 0, 1, 2: local attention,
153 + - Layer 3: global attention,
154 + ...(repeated)
155 + layer_types (`list`, *optional*):
156 + Attention pattern for each layer. Prioritized over `sliding_window_pattern`.
157 +
158 + Example:
159 +
160 + ```python
161 + >>> from transformers import Exaone4Model, Exaone4Config
162 +
163 + >>> # Initializing a EXAONE configuration
164 + >>> configuration = Exaone4Config()
165 +
166 + >>> # Initializing a model from configuration
167 + >>> model = Exaone4Model(configuration)
168 +
169 + >>> # Accessing the model configuration
170 + >>> configuration = model.config
171 + ```"""
172 +
173 + model_type = "exaone4"
174 + keys_to_ignore_at_inference = ["past_key_values"]
175 + # Default tensor parallel plan for base model `LlamaModel`
176 + base_model_tp_plan = {
177 + "layers.*.self_attn.q_proj": "colwise",
178 + "layers.*.self_attn.k_proj": "colwise",
179 + "layers.*.self_attn.v_proj": "colwise",
180 + "layers.*.self_attn.o_proj": "rowwise",
181 + "layers.*.mlp.gate_proj": "colwise",
182 + "layers.*.mlp.up_proj": "colwise",
183 + "layers.*.mlp.down_proj": "rowwise",
184 + }
185 + base_model_pp_plan = {
186 + "embed_tokens": (["input_ids"], ["inputs_embeds"]),
187 + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
188 + "norm": (["hidden_states"], ["hidden_states"]),
189 + }
190 +
191 + def __init__(
192 + self,
193 + vocab_size=102400,
194 + hidden_size=4096,
195 + intermediate_size=None,
196 + num_hidden_layers=32,
197 + num_attention_heads=32,
198 + num_key_value_heads=None,
199 + hidden_act="silu",
200 + max_position_embeddings=2048,
201 + initializer_range=0.02,
202 + rms_norm_eps=1e-5,
203 + use_cache=True,
204 + bos_token_id=0,
205 + eos_token_id=2,
206 + tie_word_embeddings=False,
207 + rope_theta=10000.0,
208 + rope_scaling=None,
209 + attention_dropout=0.0,
210 + sliding_window=None,
211 + sliding_window_pattern=None,
212 + layer_types=None,
213 + **kwargs,
214 + ):
215 + self.vocab_size = vocab_size
216 + self.hidden_size = hidden_size
217 + self.num_hidden_layers = num_hidden_layers
218 + self.num_attention_heads = num_attention_heads
219 + if num_key_value_heads is None:
220 + num_key_value_heads = num_attention_heads
221 + self.num_key_value_heads = num_key_value_heads
222 + if intermediate_size:
223 + self.intermediate_size = intermediate_size
224 + else:
225 + self.intermediate_size = hidden_size * 4
226 + self.hidden_act = hidden_act
227 + self.max_position_embeddings = max_position_embeddings
228 + self.initializer_range = initializer_range
229 + self.rms_norm_eps = rms_norm_eps
230 + self.use_cache = use_cache
231 + self.attention_dropout = attention_dropout
232 + self.rope_theta = rope_theta
233 + self.rope_scaling = rope_scaling
234 + self.sliding_window = sliding_window
235 + self.sliding_window_pattern = sliding_window_pattern
236 +
237 + self.layer_types = layer_types
238 + if self.layer_types is None:
239 + self.layer_types = [
240 + "sliding_attention"
241 + if check_is_sliding(self, i) else "full_attention"
242 + for i in range(self.num_hidden_layers)
243 + ]
244 + layer_type_validation(self.layer_types)
245 +
246 + super().__init__(bos_token_id=bos_token_id,
247 + eos_token_id=eos_token_id,
248 + tie_word_embeddings=tie_word_embeddings,
249 + **kwargs)
250 +
251 +
252 +__all__ = ["Exaone4Config"]