A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 08:30 UTC
Add EXAONE-4.5 (#39388)
Signed-off-by: lkm2835 <lkm2835@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Kyungmin Lee committed
Apr 10, 2026 at 12:53 UTC
e7a1387e73809f41df0673832498cf26566d4495
11 files changed
+600
-10
docs/models/supported_models.md
+1
@@ -550,6 +550,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
550
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
551
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
552
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
553
+| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
554
| `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b`, etc. | | ✅︎ |
555
| `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>E+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ |
556
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | |
examples/offline_inference/vision_language.py
+38
@@ -421,6 +421,43 @@ def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:
421
)
422
423
424
+# EXAONE-4.5
425
+def run_exaone4_5(questions: list[str], modality: str) -> ModelRequestData:
426
+ model_name = "LGAI-EXAONE/EXAONE-4.5-33B"
427
+
428
+ engine_args = EngineArgs(
429
+ model=model_name,
430
+ max_model_len=4096,
431
+ max_num_seqs=5,
432
+ mm_processor_kwargs={
433
+ "min_pixels": 28 * 28,
434
+ "max_pixels": 1280 * 28 * 28,
435
+ "fps": 1,
436
+ },
437
+ limit_mm_per_prompt={modality: 1},
438
+ )
439
+
440
+ if modality == "image":
441
+ placeholder = "<|image_pad|>"
442
+ elif modality == "video":
443
+ placeholder = "<|video_pad|>"
444
+
445
+ prompts = [
446
+ (
447
+ "<|system|>\nYou are a helpful assistant.<|endofturn|>\n"
448
+ f"<|user|>\n<vision>{placeholder}</vision>"
449
+ f"{question}<|endofturn|>\n"
450
+ "<|assistant|>\n"
451
+ )
452
+ for question in questions
453
+ ]
454
+
455
+ return ModelRequestData(
456
+ engine_args=engine_args,
457
+ prompts=prompts,
458
+ )
459
+
460
+
461
# Fuyu
462
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
463
assert modality == "image"
@@ -2199,6 +2236,7 @@ model_example_map = {
2236
"dots_ocr": run_dots_ocr,
2237
"eagle2_5": run_eagle2_5,
2238
"ernie45_vl": run_ernie45_vl,
2239
+ "exaone4_5": run_exaone4_5,
2240
"fuyu": run_fuyu,
2241
"gemma3": run_gemma3,
2242
"gemma3n": run_gemma3n,
examples/offline_inference/vision_language_multi_image.py
+36
@@ -241,6 +241,41 @@ def load_deepseek_ocr(question: str, image_urls: list[str]) -> ModelRequestData:
241
)
242
243
244
+# exaone4_5
245
+def load_exaone4_5(question: str, image_urls: list[str]) -> ModelRequestData:
246
+ model_name = "LGAI-EXAONE/EXAONE-4.5-33B"
247
+
248
+ engine_args = EngineArgs(
249
+ model=model_name,
250
+ max_model_len=8192,
251
+ max_num_seqs=2,
252
+ limit_mm_per_prompt={"image": len(image_urls)},
253
+ )
254
+
255
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
256
+ messages = [
257
+ {
258
+ "role": "user",
259
+ "content": [
260
+ *placeholders,
261
+ {"type": "text", "text": question},
262
+ ],
263
+ }
264
+ ]
265
+
266
+ processor = AutoProcessor.from_pretrained(model_name)
267
+
268
+ prompt = processor.apply_chat_template(
269
+ messages, tokenize=False, add_generation_prompt=True
270
+ )
271
+
272
+ return ModelRequestData(
273
+ engine_args=engine_args,
274
+ prompt=prompt,
275
+ image_data=[fetch_image(url) for url in image_urls],
276
+ )
277
+
278
+
279
def load_gemma3(question: str, image_urls: list[str]) -> ModelRequestData:
280
model_name = "google/gemma-3-4b-it"
281
@@ -1450,6 +1485,7 @@ model_example_map = {
1485
"command_a_vision": load_command_a_vision,
1486
"deepseek_vl_v2": load_deepseek_vl2,
1487
"deepseek_ocr": load_deepseek_ocr,
1488
+ "exaone4_5": load_exaone4_5,
1489
"gemma3": load_gemma3,
1490
"h2ovl_chat": load_h2ovl,
1491
"hunyuan_vl": load_hunyuan_vl,
tests/models/registry.py
+9
@@ -813,6 +813,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
813
trust_remote_code=True,
814
revision="refs/pr/17",
815
),
816
+ "Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
817
+ "LGAI-EXAONE/EXAONE-4.5-33B",
818
+ min_transformers_version="5.6.0",
819
+ ),
820
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
821
"allendou/FireRedASR2-LLM-vllm",
822
),
@@ -1306,6 +1310,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1310
min_transformers_version="5.1.0",
1311
enable_prefix_caching=False,
1312
),
1313
+ "Exaone4_5_MTP": _HfExamplesInfo(
1314
+ "LGAI-EXAONE/EXAONE-4.5-33B",
1315
+ speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
1316
+ min_transformers_version="5.6.0",
1317
+ ),
1318
"ExtractHiddenStatesModel": _HfExamplesInfo(
1319
"Qwen/Qwen3-8B",
1320
speculative_method="extract_hidden_states",
vllm/config/speculative.py
+8
-1
@@ -40,6 +40,7 @@ MTPModelTypes = Literal[
40
"ernie_mtp",
41
"nemotron_h_mtp",
42
"exaone_moe_mtp",
43
+ "exaone4_5_mtp",
44
"qwen3_next_mtp",
45
"qwen3_5_mtp",
46
"longcat_flash_mtp",
@@ -327,7 +328,13 @@ class SpeculativeConfig:
328
hf_config.update(
329
{"n_predict": n_predict, "architectures": ["ExaoneMoeMTP"]}
330
)
330
-
331
+ if "exaone4_5" in hf_config.model_type:
332
+ hf_config.model_type = "exaone4_5_mtp"
333
+ if hf_config.model_type == "exaone4_5_mtp":
334
+ n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
335
+ hf_config.update(
336
+ {"n_predict": n_predict, "architectures": ["Exaone4_5_MTP"]}
337
+ )
338
if hf_config.model_type in ("qwen3_5", "qwen3_5_moe"):
339
is_moe = hf_config.model_type == "qwen3_5_moe"
340
hf_config.model_type = "qwen3_5_mtp"
vllm/model_executor/models/exaone4.py
+3
@@ -75,6 +75,7 @@ class Exaone4GatedMLP(nn.Module):
75
reduce_results: bool = True,
76
bias: bool = False,
77
prefix: str = "",
78
+ use_data_parallel: bool = False,
79
) -> None:
80
super().__init__()
81
self.gate_up_proj = MergedColumnParallelLinear(
@@ -83,6 +84,7 @@ class Exaone4GatedMLP(nn.Module):
84
bias=bias,
85
quant_config=quant_config,
86
prefix=f"{prefix}.gate_up_proj",
87
+ disable_tp=use_data_parallel,
88
)
89
self.down_proj = RowParallelLinear(
90
input_size=intermediate_size,
@@ -91,6 +93,7 @@ class Exaone4GatedMLP(nn.Module):
93
quant_config=quant_config,
94
reduce_results=reduce_results,
95
prefix=f"{prefix}.down_proj",
96
+ disable_tp=use_data_parallel,
97
)
98
if hidden_act != "silu":
99
raise ValueError(
vllm/model_executor/models/exaone4_5.py
new
+366
@@ -0,0 +1,366 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+#
9
+# http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+"""Inference-only EXAONE-4.5 model compatible with HuggingFace weights."""
17
+
18
+from collections.abc import Callable, Iterable
19
+from functools import partial
20
+
21
+import einops
22
+import torch
23
+import torch.nn as nn
24
+from transformers.models.exaone4_5 import (
25
+ Exaone4_5_Config,
26
+ Exaone4_5_ImageProcessor,
27
+ Exaone4_5_Processor,
28
+)
29
+from transformers.models.exaone4_5.configuration_exaone4_5 import Exaone4_5_VisionConfig
30
+
31
+from vllm.compilation.decorators import (
32
+ should_torch_compile_mm_encoder,
33
+ support_torch_compile,
34
+)
35
+from vllm.config import VllmConfig
36
+from vllm.distributed import parallel_state
37
+from vllm.distributed import utils as dist_utils
38
+from vllm.logger import init_logger
39
+from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
40
+from vllm.model_executor.layers.layernorm import RMSNorm
41
+from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
42
+from vllm.model_executor.layers.quantization import QuantizationConfig
43
+from vllm.model_executor.layers.rotary_embedding.common import (
44
+ ApplyRotaryEmb,
45
+)
46
+from vllm.model_executor.models.exaone4 import Exaone4GatedMLP as Exaone4_5_VisionMLP
47
+from vllm.model_executor.models.qwen2_5_vl import (
48
+ Qwen2_5_VisionTransformer,
49
+ Qwen2_5_VLForConditionalGeneration,
50
+ Qwen2VLProcessingInfo,
51
+)
52
+from vllm.multimodal import MULTIMODAL_REGISTRY
53
+
54
+from .qwen2_vl import Qwen2VLDummyInputsBuilder as Exaone4_5_DummyInputsBuilder
55
+from .qwen2_vl import Qwen2VLMultiModalProcessor as Exaone4_5_MultiModalProcessor
56
+from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
57
+
58
+logger = init_logger(__name__)
59
+
60
+
61
+# === Vision Encoder === #
62
+
63
+
64
+class EXAONE4_5_VisionAttention(nn.Module):
65
+ def __init__(
66
+ self,
67
+ embed_dim: int,
68
+ num_heads: int,
69
+ num_kv_heads: int,
70
+ projection_size: int,
71
+ quant_config: QuantizationConfig | None = None,
72
+ prefix: str = "",
73
+ use_data_parallel: bool = False,
74
+ ) -> None:
75
+ super().__init__()
76
+ # Per attention head and per partition values.
77
+ self.tp_size = (
78
+ 1
79
+ if use_data_parallel
80
+ else parallel_state.get_tensor_model_parallel_world_size()
81
+ )
82
+ self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
83
+ self.hidden_size_per_attention_head = dist_utils.divide(
84
+ projection_size, num_heads
85
+ )
86
+ self.num_attention_heads_per_partition = dist_utils.divide(
87
+ num_heads, self.tp_size
88
+ )
89
+
90
+ self.total_num_heads = num_heads
91
+ self.total_num_kv_heads = num_kv_heads
92
+ self.num_heads = num_heads // self.tp_size
93
+ self.num_kv_heads = max(1, num_kv_heads // self.tp_size)
94
+
95
+ self.head_dim = embed_dim // num_heads
96
+ self.q_size = self.num_heads * self.head_dim
97
+ self.kv_size = self.num_kv_heads * self.head_dim
98
+
99
+ self.qkv = QKVParallelLinear(
100
+ hidden_size=embed_dim,
101
+ head_size=self.hidden_size_per_attention_head,
102
+ total_num_heads=self.total_num_heads,
103
+ total_num_kv_heads=self.total_num_kv_heads,
104
+ bias=True,
105
+ quant_config=quant_config,
106
+ prefix=f"{prefix}.qkv",
107
+ disable_tp=use_data_parallel,
108
+ )
109
+
110
+ self.proj = RowParallelLinear(
111
+ input_size=projection_size,
112
+ output_size=embed_dim,
113
+ bias=True,
114
+ quant_config=quant_config,
115
+ prefix=f"{prefix}.proj",
116
+ disable_tp=use_data_parallel,
117
+ )
118
+
119
+ self.attn = MMEncoderAttention(
120
+ num_heads=self.num_attention_heads_per_partition,
121
+ head_size=self.hidden_size_per_attention_head,
122
+ num_kv_heads=self.num_kv_heads,
123
+ scale=self.hidden_size_per_attention_head**-0.5,
124
+ prefix=f"{prefix}.attn",
125
+ )
126
+
127
+ self.apply_rotary_emb = ApplyRotaryEmb(enforce_enable=True)
128
+
129
+ def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]:
130
+ # qkv: [s, b, (h + 2*hk) * d]
131
+ s, b, _ = qkv.shape
132
+ h = self.num_heads
133
+ hk = self.num_kv_heads
134
+ d = self.head_dim
135
+
136
+ qkv = qkv.view(s, b, h + 2 * hk, d)
137
+
138
+ q = qkv[:, :, :h, :]
139
+ k = qkv[:, :, h : h + hk, :]
140
+ v = qkv[:, :, h + hk :, :]
141
+
142
+ # [s, b, h, d] -> [b, s, h, d]
143
+ return (
144
+ q.permute(1, 0, 2, 3).contiguous(),
145
+ k.permute(1, 0, 2, 3).contiguous(),
146
+ v.permute(1, 0, 2, 3).contiguous(),
147
+ )
148
+
149
+ def forward(
150
+ self,
151
+ x: torch.Tensor,
152
+ cu_seqlens: torch.Tensor,
153
+ rotary_pos_emb_cos: torch.Tensor,
154
+ rotary_pos_emb_sin: torch.Tensor,
155
+ max_seqlen: int | None = None,
156
+ ) -> torch.Tensor:
157
+ # [s, b, c] --> [s, b, head * 3 * head_dim]
158
+ x, _ = self.qkv(x)
159
+ seq_len, batch_size, _ = x.shape
160
+
161
+ q, k, v = self.split_qkv(x)
162
+ q = self.apply_rotary_emb(
163
+ q,
164
+ rotary_pos_emb_cos,
165
+ rotary_pos_emb_sin,
166
+ )
167
+
168
+ k = self.apply_rotary_emb(
169
+ k,
170
+ rotary_pos_emb_cos,
171
+ rotary_pos_emb_sin,
172
+ )
173
+
174
+ context_layer = self.attn(
175
+ query=q,
176
+ key=k,
177
+ value=v,
178
+ cu_seqlens=cu_seqlens,
179
+ max_seqlen=max_seqlen,
180
+ )
181
+
182
+ context_layer = einops.rearrange(
183
+ context_layer, "b s h d -> s b (h d)", b=batch_size
184
+ ).contiguous()
185
+
186
+ output, _ = self.proj(context_layer)
187
+ return output
188
+
189
+
190
+@support_torch_compile(
191
+ dynamic_arg_dims={
192
+ "x": 0,
193
+ "cu_seqlens": 0,
194
+ "rotary_pos_emb_cos": 0,
195
+ "rotary_pos_emb_sin": 0,
196
+ },
197
+ enable_if=should_torch_compile_mm_encoder,
198
+ is_encoder=True,
199
+)
200
+class Exaone4_5_VisionBlock(nn.Module):
201
+ def __init__(
202
+ self,
203
+ dim: int,
204
+ num_heads: int,
205
+ num_kv_heads: int,
206
+ mlp_hidden_dim: int,
207
+ hidden_act: str = "silu",
208
+ norm_layer: Callable[[int], nn.Module] | None = None,
209
+ quant_config: QuantizationConfig | None = None,
210
+ prefix: str = "",
211
+ use_data_parallel: bool = False,
212
+ ) -> None:
213
+ super().__init__()
214
+ if norm_layer is None:
215
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
216
+ self.norm1 = norm_layer(dim)
217
+ self.norm2 = norm_layer(dim)
218
+ self.attn = EXAONE4_5_VisionAttention(
219
+ embed_dim=dim,
220
+ num_heads=num_heads,
221
+ num_kv_heads=num_kv_heads,
222
+ projection_size=dim,
223
+ quant_config=quant_config,
224
+ prefix=f"{prefix}.attn",
225
+ use_data_parallel=use_data_parallel,
226
+ )
227
+ self.mlp = Exaone4_5_VisionMLP(
228
+ dim,
229
+ mlp_hidden_dim,
230
+ hidden_act=hidden_act,
231
+ bias=True,
232
+ quant_config=quant_config,
233
+ prefix=f"{prefix}.mlp",
234
+ use_data_parallel=use_data_parallel,
235
+ )
236
+
237
+ def forward(
238
+ self,
239
+ x: torch.Tensor,
240
+ cu_seqlens: torch.Tensor,
241
+ rotary_pos_emb_cos: torch.Tensor,
242
+ rotary_pos_emb_sin: torch.Tensor,
243
+ max_seqlen: int | None = None, # Only used for Flash Attention
244
+ seqlens: list[int] | None = None, # Only used for xFormers
245
+ ) -> torch.Tensor:
246
+ x_attn = self.attn(
247
+ self.norm1(x),
248
+ cu_seqlens=cu_seqlens,
249
+ rotary_pos_emb_cos=rotary_pos_emb_cos,
250
+ rotary_pos_emb_sin=rotary_pos_emb_sin,
251
+ max_seqlen=max_seqlen,
252
+ )
253
+ x_fused_norm, residual = self.norm2(x, residual=x_attn)
254
+ x = residual + self.mlp(x_fused_norm)
255
+ return x
256
+
257
+
258
+class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
259
+ def __init__(
260
+ self,
261
+ vision_config: Exaone4_5_VisionConfig,
262
+ norm_eps: float = 1e-6,
263
+ quant_config: QuantizationConfig | None = None,
264
+ prefix: str = "",
265
+ use_data_parallel: bool = False,
266
+ ) -> None:
267
+ super().__init__(
268
+ vision_config=vision_config,
269
+ norm_eps=norm_eps,
270
+ quant_config=quant_config,
271
+ prefix=prefix,
272
+ )
273
+ depth = vision_config.depth
274
+ self.num_kv_heads = vision_config.num_key_value_heads
275
+
276
+ norm_layer = partial(RMSNorm, eps=norm_eps)
277
+
278
+ self.blocks = nn.ModuleList(
279
+ [
280
+ Exaone4_5_VisionBlock(
281
+ dim=self.hidden_size,
282
+ num_heads=self.num_heads,
283
+ num_kv_heads=self.num_kv_heads,
284
+ mlp_hidden_dim=vision_config.intermediate_size,
285
+ hidden_act=vision_config.hidden_act,
286
+ norm_layer=norm_layer,
287
+ quant_config=quant_config,
288
+ prefix=f"{prefix}.blocks.{layer_idx}",
289
+ use_data_parallel=use_data_parallel,
290
+ )
291
+ for layer_idx in range(depth)
292
+ ]
293
+ )
294
+
295
+
296
+class Exaone4_5_ProcessingInfo(Qwen2VLProcessingInfo):
297
+ def get_hf_config(self):
298
+ return self.ctx.get_hf_config(Exaone4_5_Config)
299
+
300
+ def get_hf_processor(self, **kwargs: object) -> Exaone4_5_Processor:
301
+ return self.ctx.get_hf_processor(
302
+ Exaone4_5_Processor,
303
+ use_fast=kwargs.pop("use_fast", True),
304
+ **kwargs,
305
+ )
306
+
307
+ def get_image_processor(self, **kwargs: object) -> Exaone4_5_ImageProcessor:
308
+ return Exaone4_5_ImageProcessor(**kwargs)
309
+
310
+
311
+@MULTIMODAL_REGISTRY.register_processor(
312
+ Exaone4_5_MultiModalProcessor,
313
+ info=Exaone4_5_ProcessingInfo,
314
+ dummy_inputs=Exaone4_5_DummyInputsBuilder,
315
+)
316
+class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
317
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
318
+ nn.Module.__init__(self)
319
+
320
+ config: Exaone4_5_Config = vllm_config.model_config.hf_config
321
+ self.vllm_config = vllm_config
322
+ multimodal_config = vllm_config.model_config.multimodal_config
323
+
324
+ self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
325
+ self.config = config
326
+ self.multimodal_config = multimodal_config
327
+ self.is_multimodal_pruning_enabled = (
328
+ multimodal_config.is_multimodal_pruning_enabled()
329
+ )
330
+
331
+ with self._mark_tower_model(vllm_config, {"image", "video"}):
332
+ self.visual = EXAONE4_5_VisionTransformer(
333
+ config.vision_config,
334
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
335
+ quant_config=self.quant_config,
336
+ prefix=maybe_prefix(prefix, "visual"),
337
+ use_data_parallel=self.use_data_parallel,
338
+ )
339
+
340
+ with self._mark_language_model(vllm_config):
341
+ self.language_model = init_vllm_registered_model(
342
+ vllm_config=vllm_config,
343
+ prefix=maybe_prefix(prefix, "language_model"),
344
+ hf_config=config.get_text_config(),
345
+ architectures=["Exaone4ForCausalLM"],
346
+ )
347
+
348
+ self.make_empty_intermediate_tensors = (
349
+ self.language_model.make_empty_intermediate_tensors
350
+ )
351
+
352
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
353
+ loader = AutoWeightsLoader(
354
+ self,
355
+ skip_prefixes=(["mtp."]),
356
+ )
357
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
358
+
359
+ @classmethod
360
+ def get_placeholder_str(cls, modality: str, i: int) -> str | None:
361
+ if modality.startswith("image"):
362
+ return "<vision><|image_pad|></vision>"
363
+ if modality.startswith("video"):
364
+ return "<vision><|video_pad|></vision>"
365
+
366
+ raise ValueError("Only image or video modality is supported")
vllm/model_executor/models/exaone4_5_mtp.py
new
+129
@@ -0,0 +1,129 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Inference-only EXAONE-4_5 MTP model."""
4
+
5
+from collections.abc import Iterable
6
+
7
+import torch
8
+from torch import nn
9
+
10
+from vllm.compilation.decorators import support_torch_compile
11
+from vllm.config import VllmConfig
12
+from vllm.logger import init_logger
13
+from vllm.model_executor.layers.layernorm import RMSNorm
14
+from vllm.model_executor.layers.linear import ColumnParallelLinear
15
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.vocab_parallel_embedding import (
17
+ ParallelLMHead,
18
+ VocabParallelEmbedding,
19
+)
20
+from vllm.model_executor.models.exaone4 import Exaone4DecoderLayer
21
+from vllm.model_executor.models.exaone_moe_mtp import (
22
+ ExaoneMoeMTP,
23
+ ExaoneMoeMultiTokenPredictor,
24
+)
25
+
26
+from .utils import (
27
+ AutoWeightsLoader,
28
+ maybe_prefix,
29
+)
30
+
31
+logger = init_logger(__name__)
32
+
33
+KVCache = tuple[torch.Tensor, torch.Tensor]
34
+
35
+
36
+@support_torch_compile
37
+class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
38
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
39
+ nn.Module.__init__(self)
40
+
41
+ model_config = vllm_config.model_config
42
+ quant_config = vllm_config.quant_config
43
+ lora_config = vllm_config.lora_config
44
+ config = model_config.hf_config
45
+
46
+ self.config = config
47
+ lora_vocab = (
48
+ (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1))
49
+ if lora_config
50
+ else 0
51
+ )
52
+ self.vocab_size = config.vocab_size + lora_vocab
53
+ self.org_vocab_size = config.vocab_size
54
+
55
+ self.mtp_start_layer_idx = config.num_hidden_layers
56
+ self.num_mtp_layers = getattr(config, "num_nextn_predict_layers", 1)
57
+
58
+ self.embed_tokens = VocabParallelEmbedding(
59
+ self.vocab_size,
60
+ config.hidden_size,
61
+ org_num_embeddings=config.vocab_size,
62
+ )
63
+
64
+ self.fc = ColumnParallelLinear(
65
+ self.config.hidden_size * 2,
66
+ self.config.hidden_size,
67
+ gather_output=True,
68
+ bias=False,
69
+ return_bias=False,
70
+ quant_config=quant_config,
71
+ prefix=f"{prefix}.fc",
72
+ )
73
+ self.layers = nn.ModuleList(
74
+ Exaone4DecoderLayer(
75
+ vllm_config.model_config.hf_config,
76
+ quant_config=quant_config,
77
+ prefix=f"{prefix}.layers.{idx}",
78
+ )
79
+ for idx in range(self.num_mtp_layers)
80
+ )
81
+
82
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
83
+ self.pre_fc_norm_hidden = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
84
+ self.pre_fc_norm_embedding = RMSNorm(
85
+ config.hidden_size, eps=config.rms_norm_eps
86
+ )
87
+
88
+
89
+@support_torch_compile
90
+class Exaone4_5_MTP(ExaoneMoeMTP):
91
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
92
+ config = vllm_config.model_config.hf_config
93
+ self.vllm_config = vllm_config
94
+ self.quant_config = vllm_config.quant_config
95
+
96
+ nn.Module.__init__(self)
97
+ self.config = config
98
+ self.model = Exaone4_5MultiTokenPredictor(
99
+ vllm_config=vllm_config, prefix=maybe_prefix(prefix, "mtp")
100
+ )
101
+ self.unpadded_vocab_size = config.vocab_size
102
+ self.lm_head = ParallelLMHead(
103
+ self.unpadded_vocab_size,
104
+ config.hidden_size,
105
+ org_num_embeddings=config.vocab_size,
106
+ prefix=maybe_prefix(prefix, "lm_head"),
107
+ )
108
+ if config.tie_word_embeddings:
109
+ self.lm_head.weight = self.model.embed_tokens.weight
110
+ self.logits_processor = LogitsProcessor(
111
+ self.unpadded_vocab_size, config.vocab_size
112
+ )
113
+
114
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
115
+ shared_weight_names = ["embed_tokens", "lm_head"]
116
+
117
+ def remap_weight_names(weights):
118
+ for name, weight in weights:
119
+ if name.startswith("mtp."):
120
+ name = name.replace("mtp.", "model.")
121
+ elif any(key in name for key in shared_weight_names):
122
+ if "embed_tokens" in name:
123
+ name = name.replace("language_model.", "")
124
+ else:
125
+ continue
126
+ yield name, weight
127
+
128
+ loader = AutoWeightsLoader(self)
129
+ return loader.load_weights(remap_weight_names(weights))
vllm/model_executor/models/exaone_moe_mtp.py
-5
@@ -184,11 +184,6 @@ class ExaoneMoeMTP(nn.Module):
184
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
185
config = vllm_config.model_config.hf_config
186
self.vllm_config = vllm_config
187
- cache_config = vllm_config.cache_config
188
- assert not cache_config.enable_prefix_caching, (
189
- "ExaoneMoeMTP currently does not support prefix caching"
190
- )
191
-
187
self.quant_config = vllm_config.quant_config
188
189
super().__init__()
vllm/model_executor/models/registry.py
+5
@@ -371,6 +371,10 @@ _MULTIMODAL_MODELS = {
371
"ernie45_vl",
372
"Ernie4_5_VLMoeForConditionalGeneration",
373
),
374
+ "Exaone4_5_ForConditionalGeneration": (
375
+ "exaone4_5",
376
+ "Exaone4_5_ForConditionalGeneration",
377
+ ), # noqa: E501
378
"FireRedASR2ForConditionalGeneration": (
379
"fireredasr2",
380
"FireRedASR2ForConditionalGeneration",
@@ -569,6 +573,7 @@ _SPECULATIVE_DECODING_MODELS = {
573
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
574
"ErnieMTPModel": ("ernie_mtp", "ErnieMTP"),
575
"ExaoneMoeMTP": ("exaone_moe_mtp", "ExaoneMoeMTP"),
576
+ "Exaone4_5_MTP": ("exaone4_5_mtp", "Exaone4_5_MTP"),
577
"NemotronHMTPModel": ("nemotron_h_mtp", "NemotronHMTP"),
578
"LongCatFlashMTPModel": ("longcat_flash_mtp", "LongCatFlashMTP"),
579
"Glm4MoeMTPModel": ("glm4_moe_mtp", "Glm4MoeMTP"),
vllm/v1/spec_decode/eagle.py
+5
-4
@@ -1321,13 +1321,14 @@ class SpecDecodeBaseProposer:
1321
# handle multimodality
1322
assert hasattr(target_model, "config")
1323
if self.get_model_name(target_model) in [
1324
- "Qwen2_5_VLForConditionalGeneration",
1325
- "Qwen3VLForConditionalGeneration",
1326
- "Qwen3VLMoeForConditionalGeneration",
1327
- "HunYuanVLForConditionalGeneration",
1324
+ "Exaone4_5_ForConditionalGeneration",
1325
"GlmOcrForConditionalGeneration",
1326
+ "HunYuanVLForConditionalGeneration",
1327
+ "Qwen2_5_VLForConditionalGeneration",
1328
"Qwen3_5ForConditionalGeneration",
1329
"Qwen3_5MoeForConditionalGeneration",
1330
+ "Qwen3VLForConditionalGeneration",
1331
+ "Qwen3VLMoeForConditionalGeneration",
1332
]:
1333
self.model.config.image_token_index = target_model.config.image_token_id
1334
elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":