A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 22:45 UTC
Fix EXAONE-4.5 to align with Transformers update (#42246)
Signed-off-by: lkm2835 <lkm2835@gmail.com>
Kyungmin Lee committed
May 11, 2026 at 19:25 UTC
27ae6763647920119dcd50d147ca365fdc138945
2 files changed
+53
-13
vllm/model_executor/models/exaone4_5.py
-4
@@ -23,7 +23,6 @@ import torch
23
import torch.nn as nn
24
from transformers.models.exaone4_5 import (
25
Exaone4_5_Config,
26
- Exaone4_5_ImageProcessor,
26
Exaone4_5_Processor,
27
)
28
from transformers.models.exaone4_5.configuration_exaone4_5 import Exaone4_5_VisionConfig
@@ -304,9 +303,6 @@ class Exaone4_5_ProcessingInfo(Qwen2VLProcessingInfo):
303
**kwargs,
304
)
305
307
- def get_image_processor(self, **kwargs: object) -> Exaone4_5_ImageProcessor:
308
- return Exaone4_5_ImageProcessor(**kwargs)
309
-
306
307
@MULTIMODAL_REGISTRY.register_processor(
308
Exaone4_5_MultiModalProcessor,
vllm/model_executor/models/exaone4_5_mtp.py
+53
-9
@@ -9,6 +9,7 @@ from torch import nn
9
10
from vllm.compilation.decorators import support_torch_compile
11
from vllm.config import VllmConfig
12
+from vllm.distributed.parallel_state import get_pp_group
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import ColumnParallelLinear
@@ -22,6 +23,7 @@ from vllm.model_executor.models.exaone_moe_mtp import (
23
ExaoneMoeMTP,
24
ExaoneMoeMultiTokenPredictor,
25
)
26
+from vllm.sequence import IntermediateTensors
27
28
from .interfaces import (
29
MultiModalEmbeddings,
@@ -48,6 +50,7 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
50
quant_config = vllm_config.quant_config
51
lora_config = vllm_config.lora_config
52
config = model_config.hf_config
53
+ text_config = config.text_config
54
55
self.config = config
56
lora_vocab = (
@@ -58,18 +61,18 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
61
self.vocab_size = config.vocab_size + lora_vocab
62
self.org_vocab_size = config.vocab_size
63
61
- self.mtp_start_layer_idx = config.num_hidden_layers
64
+ self.mtp_start_layer_idx = text_config.num_hidden_layers
65
self.num_mtp_layers = getattr(config, "num_nextn_predict_layers", 1)
66
67
self.embed_tokens = VocabParallelEmbedding(
68
self.vocab_size,
66
- config.hidden_size,
69
+ text_config.hidden_size,
70
org_num_embeddings=config.vocab_size,
71
)
72
73
self.fc = ColumnParallelLinear(
71
- self.config.hidden_size * 2,
72
- self.config.hidden_size,
74
+ text_config.hidden_size * 2,
75
+ text_config.hidden_size,
76
gather_output=True,
77
bias=False,
78
return_bias=False,
@@ -78,27 +81,68 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
81
)
82
self.layers = nn.ModuleList(
83
Exaone4DecoderLayer(
81
- vllm_config.model_config.hf_config,
84
+ text_config,
85
quant_config=quant_config,
86
prefix=f"{prefix}.layers.{idx}",
87
)
88
for idx in range(self.num_mtp_layers)
89
)
90
88
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
89
- self.pre_fc_norm_hidden = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
91
+ self.norm = RMSNorm(text_config.hidden_size, eps=text_config.rms_norm_eps)
92
+ self.pre_fc_norm_hidden = RMSNorm(
93
+ text_config.hidden_size, eps=text_config.rms_norm_eps
94
+ )
95
self.pre_fc_norm_embedding = RMSNorm(
91
- config.hidden_size, eps=config.rms_norm_eps
96
+ text_config.hidden_size, eps=text_config.rms_norm_eps
97
)
98
99
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
100
return self.embed_tokens(input_ids)
101
102
+ def forward(
103
+ self,
104
+ input_ids: torch.Tensor,
105
+ positions: torch.Tensor,
106
+ hidden_states: torch.Tensor,
107
+ intermediate_tensors: IntermediateTensors | None = None,
108
+ inputs_embeds: torch.Tensor | None = None,
109
+ spec_step_idx: int = 0,
110
+ ) -> torch.Tensor:
111
+ if get_pp_group().is_first_rank:
112
+ if inputs_embeds is None:
113
+ inputs_embeds = self.get_input_embeddings(input_ids)
114
+ assert hidden_states.shape[-1] == inputs_embeds.shape[-1]
115
+ inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
116
+ hidden_states = self.pre_fc_norm_hidden(hidden_states)
117
+ hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1)
118
+ hidden_states = self.fc(hidden_states)
119
+ residual = None
120
+ else:
121
+ assert intermediate_tensors is not None
122
+ hidden_states = intermediate_tensors["hidden_states"]
123
+ residual = intermediate_tensors["residual"]
124
+
125
+ current_step_idx = spec_step_idx % self.num_mtp_layers
126
+ hidden_states, residual = self.layers[current_step_idx](
127
+ positions=positions,
128
+ hidden_states=hidden_states,
129
+ residual=residual,
130
+ )
131
+
132
+ if not get_pp_group().is_last_rank:
133
+ return IntermediateTensors(
134
+ {"hidden_states": hidden_states, "residual": residual}
135
+ )
136
+
137
+ hidden_states = self.norm(hidden_states)
138
+ return hidden_states
139
+
140
141
@support_torch_compile
142
class Exaone4_5_MTP(ExaoneMoeMTP, SupportsMultiModal):
143
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
144
config = vllm_config.model_config.hf_config
145
+ text_config = config.text_config
146
self.vllm_config = vllm_config
147
self.quant_config = vllm_config.quant_config
148
@@ -110,7 +154,7 @@ class Exaone4_5_MTP(ExaoneMoeMTP, SupportsMultiModal):
154
self.unpadded_vocab_size = config.vocab_size
155
self.lm_head = ParallelLMHead(
156
self.unpadded_vocab_size,
113
- config.hidden_size,
157
+ text_config.hidden_size,
158
org_num_embeddings=config.vocab_size,
159
prefix=maybe_prefix(prefix, "lm_head"),
160
)