A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[Model] Add HyperCLOVAX-SEED-Think-14B language model support (#37107)
Signed-off-by: bigshanedogg <bigshane319@gmail.com>
bigshanedogg committed
Mar 16, 2026 at 15:40 UTC
2390d44209d0dc8d9c52c5e05e9d57407d57b1d6
7 files changed
+837
-2
docs/models/supported_models.md
+1
@@ -418,6 +418,7 @@ th {
418
| `Grok1ForCausalLM` | Grok2 | `xai-org/grok-2` | ✅︎ | ✅︎ |
419
| `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ |
420
| `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ |
421
+| `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ |
422
| `InternLMForCausalLM` | InternLM | `internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc. | ✅︎ | ✅︎ |
423
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
424
| `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ |
tests/models/language/generation/test_common.py
+4
@@ -103,6 +103,10 @@ AITER_MODEL_LIST = [
103
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
104
),
105
pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus
106
+ pytest.param(
107
+ "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", # hyperclovax
108
+ marks=[large_gpu_mark(min_gb=32)],
109
+ ),
110
],
111
)
112
@pytest.mark.parametrize("max_tokens", [32])
tests/models/registry.py
+1
-1
@@ -320,7 +320,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
320
"tencent/Hunyuan-A13B-Instruct", trust_remote_code=True
321
),
322
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
323
- "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
323
+ "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
324
trust_remote_code=True,
325
),
326
"InternLMForCausalLM": _HfExamplesInfo(
vllm/model_executor/models/hyperclovax.py
new
+551
@@ -0,0 +1,551 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# SPDX-FileCopyrightText: Copyright 2025 NAVER Cloud HyperCLOVA team
4
+
5
+# Adapted from
6
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
7
+# Copyright 2025 NAVER Cloud HyperCLOVA team. All rights reserved.
8
+# Copyright 2023 The vLLM team.
9
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
10
+#
11
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12
+# and OPT implementations in this library. It has been modified from its
13
+# original forms to accommodate minor architectural differences compared
14
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15
+#
16
+# Licensed under the Apache License, Version 2.0 (the "License");
17
+# you may not use this file except in compliance with the License.
18
+# You may obtain a copy of the License at
19
+#
20
+# http://www.apache.org/licenses/LICENSE-2.0
21
+#
22
+# Unless required by applicable law or agreed to in writing, software
23
+# distributed under the License is distributed on an "AS IS" BASIS,
24
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
+# See the License for the specific language governing permissions and
26
+# limitations under the License.
27
+"""Inference-only HyperCLOVAX model compatible with HuggingFace weights."""
28
+
29
+from collections.abc import Iterable
30
+from itertools import islice
31
+
32
+import torch
33
+from torch import nn
34
+
35
+from vllm.compilation.decorators import support_torch_compile
36
+from vllm.config import CacheConfig, VllmConfig
37
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
38
+from vllm.model_executor.layers.activation import SiluAndMul
39
+from vllm.model_executor.layers.attention import Attention
40
+from vllm.model_executor.layers.layernorm import RMSNorm
41
+from vllm.model_executor.layers.linear import (
42
+ MergedColumnParallelLinear,
43
+ QKVParallelLinear,
44
+ RowParallelLinear,
45
+)
46
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
47
+from vllm.model_executor.layers.quantization import QuantizationConfig
48
+from vllm.model_executor.layers.rotary_embedding import get_rope
49
+from vllm.model_executor.layers.vocab_parallel_embedding import (
50
+ ParallelLMHead,
51
+ VocabParallelEmbedding,
52
+)
53
+from vllm.model_executor.model_loader.weight_utils import (
54
+ default_weight_loader,
55
+ maybe_remap_kv_scale_name,
56
+)
57
+from vllm.sequence import IntermediateTensors
58
+from vllm.transformers_utils.configs.hyperclovax import HyperCLOVAXConfig
59
+
60
+from .interfaces import SupportsLoRA, SupportsPP
61
+from .utils import (
62
+ AutoWeightsLoader,
63
+ PPMissingLayer,
64
+ is_pp_missing_parameter,
65
+ make_empty_intermediate_tensors_factory,
66
+ make_layers,
67
+ maybe_prefix,
68
+)
69
+
70
+
71
+class HyperCLOVAXMLP(nn.Module):
72
+ def __init__(
73
+ self,
74
+ hidden_size: int,
75
+ intermediate_size: int,
76
+ hidden_act: str,
77
+ quant_config: QuantizationConfig | None = None,
78
+ bias: bool = False,
79
+ prefix: str = "",
80
+ reduce_results: bool = True,
81
+ disable_tp: bool = False,
82
+ ) -> None:
83
+ super().__init__()
84
+ self.gate_up_proj = MergedColumnParallelLinear(
85
+ input_size=hidden_size,
86
+ output_sizes=[intermediate_size] * 2,
87
+ bias=bias,
88
+ quant_config=quant_config,
89
+ disable_tp=disable_tp,
90
+ prefix=f"{prefix}.gate_up_proj",
91
+ )
92
+ self.down_proj = RowParallelLinear(
93
+ input_size=intermediate_size,
94
+ output_size=hidden_size,
95
+ bias=bias,
96
+ quant_config=quant_config,
97
+ reduce_results=reduce_results,
98
+ disable_tp=disable_tp,
99
+ prefix=f"{prefix}.down_proj",
100
+ )
101
+ if hidden_act != "silu":
102
+ raise ValueError(
103
+ f"Unsupported activation: {hidden_act}. Only silu is supported for now."
104
+ )
105
+ self.act_fn = SiluAndMul()
106
+
107
+ def forward(self, x):
108
+ x, _ = self.gate_up_proj(x)
109
+ x = self.act_fn(x)
110
+ x, _ = self.down_proj(x)
111
+ return x
112
+
113
+
114
+class HyperCLOVAXAttention(nn.Module):
115
+ def __init__(
116
+ self,
117
+ config: HyperCLOVAXConfig,
118
+ hidden_size: int,
119
+ num_heads: int,
120
+ num_kv_heads: int,
121
+ max_position_embeddings: int = 8192,
122
+ quant_config: QuantizationConfig | None = None,
123
+ bias: bool = False,
124
+ cache_config: CacheConfig | None = None,
125
+ prefix: str = "",
126
+ dual_chunk_attention_config: dict | None = None,
127
+ ) -> None:
128
+ super().__init__()
129
+ self.hidden_size = hidden_size
130
+ tp_size = get_tensor_model_parallel_world_size()
131
+ self.total_num_heads = num_heads
132
+ assert self.total_num_heads % tp_size == 0
133
+ self.num_heads = self.total_num_heads // tp_size
134
+ self.total_num_kv_heads = num_kv_heads
135
+ if self.total_num_kv_heads >= tp_size:
136
+ # Number of KV heads is greater than TP size, so we partition
137
+ # the KV heads across multiple tensor parallel GPUs.
138
+ assert self.total_num_kv_heads % tp_size == 0
139
+ else:
140
+ # Number of KV heads is less than TP size, so we replicate
141
+ # the KV heads across multiple tensor parallel GPUs.
142
+ assert tp_size % self.total_num_kv_heads == 0
143
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
144
+ self.head_dim = getattr(
145
+ config, "head_dim", self.hidden_size // self.total_num_heads
146
+ )
147
+ self.q_size = self.num_heads * self.head_dim
148
+ self.kv_size = self.num_kv_heads * self.head_dim
149
+ self.scaling = config.attention_multiplier
150
+
151
+ self.qkv_proj = QKVParallelLinear(
152
+ hidden_size=hidden_size,
153
+ head_size=self.head_dim,
154
+ total_num_heads=self.total_num_heads,
155
+ total_num_kv_heads=self.total_num_kv_heads,
156
+ bias=bias,
157
+ quant_config=quant_config,
158
+ prefix=f"{prefix}.qkv_proj",
159
+ )
160
+
161
+ self.o_proj = RowParallelLinear(
162
+ input_size=self.total_num_heads * self.head_dim,
163
+ output_size=hidden_size,
164
+ bias=bias,
165
+ quant_config=quant_config,
166
+ prefix=f"{prefix}.o_proj",
167
+ )
168
+
169
+ self.rotary_emb = get_rope(
170
+ self.head_dim,
171
+ max_position=max_position_embeddings,
172
+ is_neox_style=True,
173
+ rope_parameters=getattr(config, "rope_parameters", None),
174
+ dual_chunk_attention_config=dual_chunk_attention_config,
175
+ )
176
+
177
+ self.attn = Attention(
178
+ self.num_heads,
179
+ self.head_dim,
180
+ self.scaling,
181
+ num_kv_heads=self.num_kv_heads,
182
+ cache_config=cache_config,
183
+ quant_config=quant_config,
184
+ prefix=f"{prefix}.attn",
185
+ )
186
+
187
+ def forward(
188
+ self,
189
+ positions: torch.Tensor,
190
+ hidden_states: torch.Tensor,
191
+ ) -> torch.Tensor:
192
+ qkv, _ = self.qkv_proj(hidden_states)
193
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
194
+ q, k = self.rotary_emb(positions, q, k)
195
+ attn_output = self.attn(q, k, v)
196
+ output, _ = self.o_proj(attn_output)
197
+ return output
198
+
199
+
200
+class HyperCLOVAXDecoderLayer(nn.Module):
201
+ def __init__(
202
+ self,
203
+ vllm_config: VllmConfig,
204
+ prefix: str = "",
205
+ ) -> None:
206
+ super().__init__()
207
+
208
+ config = vllm_config.model_config.hf_config
209
+ cache_config = vllm_config.cache_config
210
+ quant_config = vllm_config.quant_config
211
+
212
+ self.hidden_size = config.hidden_size
213
+ self.residual_multiplier = config.residual_multiplier
214
+ max_position_embeddings = getattr(
215
+ config,
216
+ "max_position_embeddings",
217
+ 8192,
218
+ )
219
+ dual_chunk_attention_config = getattr(
220
+ config,
221
+ "dual_chunk_attention_config",
222
+ None,
223
+ )
224
+ attention_bias = getattr(config, "attention_bias", False)
225
+
226
+ self.self_attn = HyperCLOVAXAttention(
227
+ config=config,
228
+ hidden_size=self.hidden_size,
229
+ num_heads=config.num_attention_heads,
230
+ num_kv_heads=getattr(
231
+ config, "num_key_value_heads", config.num_attention_heads
232
+ ),
233
+ max_position_embeddings=max_position_embeddings,
234
+ quant_config=quant_config,
235
+ bias=attention_bias,
236
+ cache_config=cache_config,
237
+ prefix=f"{prefix}.self_attn",
238
+ dual_chunk_attention_config=dual_chunk_attention_config,
239
+ )
240
+ self.mlp = HyperCLOVAXMLP(
241
+ hidden_size=self.hidden_size,
242
+ intermediate_size=config.intermediate_size,
243
+ hidden_act=config.hidden_act,
244
+ quant_config=quant_config,
245
+ bias=getattr(config, "mlp_bias", False),
246
+ prefix=f"{prefix}.mlp",
247
+ )
248
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
249
+ self.post_attention_layernorm = RMSNorm(
250
+ config.hidden_size, eps=config.rms_norm_eps
251
+ )
252
+
253
+ # post-norm (dual-norm)
254
+ self.use_post_norm = config.use_post_norm
255
+ if self.use_post_norm:
256
+ self.post_norm1 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
257
+ self.post_norm2 = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
258
+
259
+ def forward(
260
+ self,
261
+ positions: torch.Tensor,
262
+ hidden_states: torch.Tensor,
263
+ residual: torch.Tensor | None,
264
+ ) -> tuple[torch.Tensor, torch.Tensor]:
265
+ # Unlike models that use a fused add-norm kernel (e.g. Llama), HyperCLOVAX
266
+ # applies the residual connection explicitly with a muP scaling factor
267
+ # (residual + hidden * residual_multiplier). As a result, each layer's
268
+ # hidden_states output already includes the residual addition, so the
269
+ # incoming residual is not needed and is reset at the start of each layer.
270
+ # The residual parameter is kept for interface consistency with other vllm
271
+ # decoder layers.
272
+
273
+ # Self Attention
274
+ residual = hidden_states
275
+
276
+ hidden_states = self.input_layernorm(hidden_states)
277
+
278
+ hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states)
279
+ # Custom ln
280
+ if self.use_post_norm:
281
+ hidden_states = self.post_norm1(hidden_states)
282
+
283
+ # The residual is added outside the layernorm function to apply muP.
284
+ hidden_states = residual + hidden_states * self.residual_multiplier # muP
285
+ # Fully Connected
286
+ residual = hidden_states
287
+ hidden_states = self.post_attention_layernorm(hidden_states)
288
+ hidden_states = self.mlp(hidden_states)
289
+
290
+ # Custom ln
291
+ if self.use_post_norm:
292
+ hidden_states = self.post_norm2(hidden_states)
293
+
294
+ # The residual is added outside the layernorm function to apply muP.
295
+ hidden_states = residual + hidden_states * self.residual_multiplier # muP
296
+
297
+ return hidden_states, residual
298
+
299
+
300
+@support_torch_compile
301
+class HyperCLOVAXModel(nn.Module):
302
+ def __init__(
303
+ self,
304
+ *,
305
+ vllm_config: VllmConfig,
306
+ prefix: str = "",
307
+ layer_type: type[nn.Module] = HyperCLOVAXDecoderLayer,
308
+ ):
309
+ super().__init__()
310
+
311
+ config = vllm_config.model_config.hf_config
312
+ quant_config = vllm_config.quant_config
313
+
314
+ self.config = config
315
+ self.quant_config = quant_config
316
+ self.vocab_size = config.vocab_size
317
+ self.embed_tokens: VocabParallelEmbedding | PPMissingLayer
318
+ if get_pp_group().is_first_rank or (
319
+ config.tie_word_embeddings and get_pp_group().is_last_rank
320
+ ):
321
+ self.embed_tokens = VocabParallelEmbedding(
322
+ self.vocab_size,
323
+ config.hidden_size,
324
+ quant_config=quant_config,
325
+ )
326
+ else:
327
+ self.embed_tokens = PPMissingLayer()
328
+ self.start_layer, self.end_layer, self.layers = make_layers(
329
+ config.num_hidden_layers,
330
+ lambda prefix: layer_type(vllm_config=vllm_config, prefix=prefix),
331
+ prefix=f"{prefix}.layers",
332
+ )
333
+ self.norm: RMSNorm | PPMissingLayer
334
+ if get_pp_group().is_last_rank:
335
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
336
+ else:
337
+ self.norm = PPMissingLayer()
338
+
339
+ self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
340
+ ["hidden_states", "residual"], config.hidden_size
341
+ )
342
+
343
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
344
+ return self.embed_tokens(input_ids)
345
+
346
+ def forward(
347
+ self,
348
+ input_ids: torch.Tensor | None,
349
+ positions: torch.Tensor,
350
+ intermediate_tensors: IntermediateTensors | None,
351
+ inputs_embeds: torch.Tensor | None = None,
352
+ ) -> torch.Tensor | IntermediateTensors:
353
+ if get_pp_group().is_first_rank:
354
+ if inputs_embeds is not None:
355
+ hidden_states = inputs_embeds
356
+ else:
357
+ assert input_ids is not None
358
+ hidden_states = self.embed_input_ids(input_ids)
359
+ residual = None
360
+
361
+ hidden_states *= self.config.embedding_multiplier # muP
362
+ else:
363
+ assert intermediate_tensors is not None
364
+ hidden_states = intermediate_tensors["hidden_states"]
365
+ residual = intermediate_tensors["residual"]
366
+
367
+ for layer in islice(self.layers, self.start_layer, self.end_layer):
368
+ hidden_states, residual = layer(positions, hidden_states, residual)
369
+
370
+ if not get_pp_group().is_last_rank:
371
+ assert residual is not None
372
+ return IntermediateTensors(
373
+ {"hidden_states": hidden_states, "residual": residual}
374
+ )
375
+
376
+ # The residual is added outside the layernorm function to apply muP.
377
+ hidden_states = self.norm(hidden_states)
378
+ return hidden_states
379
+
380
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
381
+ stacked_params_mapping = [
382
+ # (param_name, shard_name, shard_id)
383
+ (".qkv_proj", ".q_proj", "q"),
384
+ (".qkv_proj", ".k_proj", "k"),
385
+ (".qkv_proj", ".v_proj", "v"),
386
+ (".gate_up_proj", ".gate_proj", 0),
387
+ (".gate_up_proj", ".up_proj", 1),
388
+ ]
389
+ params_dict = dict(self.named_parameters())
390
+ loaded_params: set[str] = set()
391
+ for name, loaded_weight in weights:
392
+ if "rotary_emb.inv_freq" in name:
393
+ continue
394
+ if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
395
+ # Models trained using ColossalAI may include these tensors in
396
+ # the checkpoint. Skip them.
397
+ continue
398
+ if self.quant_config is not None and (
399
+ scale_name := self.quant_config.get_cache_scale(name)
400
+ ):
401
+ # Loading kv cache quantization scales
402
+ param = params_dict[scale_name]
403
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
404
+ loaded_weight = (
405
+ loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]
406
+ )
407
+ weight_loader(param, loaded_weight)
408
+ loaded_params.add(scale_name)
409
+ continue
410
+ if "scale" in name or "zero_point" in name:
411
+ # Remapping the name of FP8 kv-scale or zero point.
412
+ remapped_name = maybe_remap_kv_scale_name(name, params_dict)
413
+ if remapped_name is None:
414
+ continue
415
+ name = remapped_name
416
+ for param_name, weight_name, shard_id in stacked_params_mapping:
417
+ if weight_name not in name:
418
+ continue
419
+ name = name.replace(weight_name, param_name)
420
+ # Skip loading extra bias for GPTQ models.
421
+ if name.endswith(".bias") and name not in params_dict:
422
+ continue
423
+
424
+ if is_pp_missing_parameter(name, self):
425
+ continue
426
+
427
+ param = params_dict[name]
428
+ weight_loader = param.weight_loader # type: ignore[attr-defined]
429
+ weight_loader(param, loaded_weight, shard_id)
430
+ break
431
+ else:
432
+ # Skip loading extra bias for GPTQ models.
433
+ if name.endswith(".bias") and name not in params_dict:
434
+ continue
435
+
436
+ if is_pp_missing_parameter(name, self):
437
+ continue
438
+
439
+ param = params_dict[name]
440
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
441
+ weight_loader(param, loaded_weight)
442
+ loaded_params.add(name)
443
+ return loaded_params
444
+
445
+
446
+class HyperCLOVAXForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
447
+ packed_modules_mapping = {
448
+ "qkv_proj": [
449
+ "q_proj",
450
+ "k_proj",
451
+ "v_proj",
452
+ ],
453
+ "gate_up_proj": [
454
+ "gate_proj",
455
+ "up_proj",
456
+ ],
457
+ }
458
+
459
+ # LoRA specific attributes
460
+ embedding_modules = {
461
+ "embed_tokens": "input_embeddings",
462
+ "lm_head": "output_embeddings",
463
+ }
464
+
465
+ def __init__(
466
+ self,
467
+ *,
468
+ vllm_config: VllmConfig,
469
+ prefix: str = "",
470
+ layer_type: type[nn.Module] = HyperCLOVAXDecoderLayer,
471
+ ):
472
+ super().__init__()
473
+ config = vllm_config.model_config.hf_config
474
+ quant_config = vllm_config.quant_config
475
+ self.config = config
476
+
477
+ self.model = self._init_model(
478
+ vllm_config=vllm_config,
479
+ prefix=maybe_prefix(prefix, "model"),
480
+ layer_type=layer_type,
481
+ )
482
+
483
+ self.lm_head: ParallelLMHead | PPMissingLayer
484
+ if get_pp_group().is_last_rank:
485
+ self.lm_head = ParallelLMHead(
486
+ config.vocab_size,
487
+ config.hidden_size,
488
+ quant_config=quant_config,
489
+ prefix=maybe_prefix(prefix, "lm_head"),
490
+ )
491
+ if config.tie_word_embeddings:
492
+ self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
493
+
494
+ logit_scale = getattr(config, "logit_scale", 1.0)
495
+ if hasattr(config, "logits_scaling"):
496
+ logit_scale *= config.logits_scaling # muP
497
+ self.logits_processor = LogitsProcessor(
498
+ config.vocab_size,
499
+ scale=logit_scale,
500
+ )
501
+ else:
502
+ self.lm_head = PPMissingLayer()
503
+
504
+ self.make_empty_intermediate_tensors = ( # type: ignore[method-assign]
505
+ self.model.make_empty_intermediate_tensors
506
+ )
507
+
508
+ def _init_model(
509
+ self,
510
+ vllm_config: VllmConfig,
511
+ prefix: str = "",
512
+ layer_type: type[nn.Module] = HyperCLOVAXDecoderLayer,
513
+ ):
514
+ return HyperCLOVAXModel(
515
+ vllm_config=vllm_config,
516
+ prefix=prefix,
517
+ layer_type=layer_type,
518
+ )
519
+
520
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
521
+ return self.model.embed_tokens(input_ids)
522
+
523
+ def forward( # type: ignore[override]
524
+ self,
525
+ input_ids: torch.Tensor | None,
526
+ positions: torch.Tensor,
527
+ *,
528
+ intermediate_tensors: IntermediateTensors | None,
529
+ inputs_embeds: torch.Tensor | None = None,
530
+ ) -> torch.Tensor | IntermediateTensors:
531
+ model_output = self.model(
532
+ input_ids, positions, intermediate_tensors, inputs_embeds
533
+ )
534
+ return model_output
535
+
536
+ def compute_logits(
537
+ self,
538
+ hidden_states: torch.Tensor,
539
+ ) -> torch.Tensor | None:
540
+ logits = self.logits_processor(self.lm_head, hidden_states)
541
+ return logits
542
+
543
+ def load_weights(
544
+ self,
545
+ weights: Iterable[tuple[str, torch.Tensor]],
546
+ ) -> set[str]:
547
+ loader = AutoWeightsLoader(
548
+ self,
549
+ skip_prefixes=["lm_head."] if self.config.tie_word_embeddings else None,
550
+ )
551
+ return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
-1
@@ -133,7 +133,7 @@ _TEXT_GENERATION_MODELS = {
133
"HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"),
134
"HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"),
135
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
136
- "HyperCLOVAXForCausalLM": ("llama", "LlamaForCausalLM"),
136
+ "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
137
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
138
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
139
"InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
vllm/transformers_utils/configs/__init__.py
+2
@@ -33,6 +33,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
33
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
34
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
35
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
36
+ "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
37
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
38
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
39
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -91,6 +92,7 @@ __all__ = [
92
"HunYuanVLConfig",
93
"HunYuanVLTextConfig",
94
"HunYuanVLVisionConfig",
95
+ "HyperCLOVAXConfig",
96
"IsaacConfig",
97
"RWConfig",
98
"JAISConfig",
vllm/transformers_utils/configs/hyperclovax.py
new
+277
@@ -0,0 +1,277 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# SPDX-FileCopyrightText: Copyright 2025 NAVER Cloud HyperCLOVA team
4
+#
5
+# Copyright 2025 NAVER Cloud HyperCLOVA team. All rights reserved.
6
+#
7
+# Licensed under the Apache License, Version 2.0 (the "License");
8
+# you may not use this file except in compliance with the License.
9
+# You may obtain a copy of the License at
10
+#
11
+# http://www.apache.org/licenses/LICENSE-2.0
12
+#
13
+# Unless required by applicable law or agreed to in writing, software
14
+# distributed under the License is distributed on an "AS IS" BASIS,
15
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+# See the License for the specific language governing permissions and
17
+# limitations under the License.
18
+"""HyperCLOVA X model configuration."""
19
+
20
+from transformers.configuration_utils import PretrainedConfig
21
+
22
+
23
+class HyperCLOVAXConfig(PretrainedConfig):
24
+ r"""
25
+ This is the configuration class to store the configuration of a
26
+ [`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model
27
+ according to the specified arguments, defining the model architecture.
28
+ Configuration objects inherit from [`PretrainedConfig`] and can be used
29
+ to control the model outputs. Read the documentation from
30
+ [`PretrainedConfig`] for more information.
31
+
32
+ Args:
33
+ vocab_size (`int`, *optional*, defaults to 32000):
34
+ Vocabulary size of the HyperCLOVAX model. Defines the number of
35
+ different tokens that can be represented by the `input_ids`
36
+ passed when calling [`HyperCLOVAXModel`]
37
+ hidden_size (`int`, *optional*, defaults to 4096):
38
+ Dimension of the hidden representations.
39
+ intermediate_size (`int`, *optional*, defaults to 11008):
40
+ Dimension of the MLP representations.
41
+ num_hidden_layers (`int`, *optional*, defaults to 32):
42
+ Number of hidden layers in the Transformer decoder.
43
+ num_attention_heads (`int`, *optional*, defaults to 32):
44
+ Number of attention heads for each attention layer in the
45
+ Transformer decoder.
46
+ num_key_value_heads (`int`, *optional*):
47
+ This is the number of key_value heads that should be used to
48
+ implement Grouped Query Attention. If
49
+ `num_key_value_heads=num_attention_heads`, the model will use
50
+ Multi Head Attention (MHA), if `num_key_value_heads=1` the model
51
+ will use Multi Query Attention (MQA) otherwise GQA is used. When
52
+ converting a multi-head checkpoint to a GQA checkpoint, each
53
+ group key and value head should be constructed by meanpooling all
54
+ the original heads within that group. For more details checkout
55
+ [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
56
+ specified, will default to `num_attention_heads`.
57
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
58
+ The non-linear activation function (function or string) in the
59
+ decoder.
60
+ max_position_embeddings (`int`, *optional*, defaults to 2048):
61
+ The maximum sequence length that this model might ever be used
62
+ with.
63
+ initializer_range (`float`, *optional*, defaults to 0.02):
64
+ The standard deviation of the truncated_normal_initializer for
65
+ initializing all weight matrices.
66
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
67
+ The epsilon used by the rms normalization layers.
68
+ use_cache (`bool`, *optional*, defaults to `True`):
69
+ Whether or not the model should return the last key/values
70
+ attentions (not used by all models). Only relevant if
71
+ `config.is_decoder=True`.
72
+ pad_token_id (`int`, *optional*):
73
+ Padding token id.
74
+ bos_token_id (`int`, *optional*, defaults to 1):
75
+ Beginning of stream token id.
76
+ eos_token_id (`int`, *optional*, defaults to 2):
77
+ End of stream token id.
78
+ pretraining_tp (`int`, *optional*, defaults to 1):
79
+ Experimental feature. Tensor parallelism rank used during
80
+ pretraining. Please refer to [this document](https://huggingface.
81
+ co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism)
82
+ to understand more about it. This value is necessary to ensure
83
+ exact reproducibility of the pretraining results. Please refer to
84
+ [this issue](https://github.com/pytorch/pytorch/issues/76232).
85
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
86
+ Whether to tie weight embeddings
87
+ rope_theta (`float`, *optional*, defaults to 10000.0):
88
+ The base period of the RoPE embeddings.
89
+ rope_scaling (`Dict`, *optional*):
90
+ Dictionary containing the scaling configuration for the RoPE
91
+ embeddings. NOTE: if you apply new rope type and you expect the
92
+ model to work on longer `max_position_embeddings`, we recommend
93
+ you to update this value accordingly.
94
+ Expected contents:
95
+ `rope_type` (`str`):
96
+ The sub-variant of RoPE to use. Can be one of ['default',
97
+ 'linear', 'dynamic', 'yarn', 'longrope', 'llama3'], with
98
+ 'default' being the original RoPE implementation.
99
+ `factor` (`float`, *optional*):
100
+ Used with all rope types except 'default'. The scaling
101
+ factor to apply to the RoPE embeddings. In most scaling
102
+ types, a `factor` of x will enable the model to handle
103
+ sequences of length x * original maximum pre-trained
104
+ length.
105
+ `original_max_position_embeddings` (`int`, *optional*):
106
+ Used with 'dynamic', 'longrope' and 'llama3'. The
107
+ original max position embeddings used during pretraining.
108
+ `attention_factor` (`float`, *optional*):
109
+ Used with 'yarn' and 'longrope'. The scaling factor to be
110
+ applied on the attention computation. If unspecified, it
111
+ defaults to value recommended by the implementation, using
112
+ the `factor` field to infer the suggested value.
113
+ `beta_fast` (`float`, *optional*):
114
+ Only used with 'yarn'. Parameter to set the boundary for
115
+ extrapolation (only) in the linear ramp function. If
116
+ unspecified, it defaults to 32.
117
+ `beta_slow` (`float`, *optional*):
118
+ Only used with 'yarn'. Parameter to set the boundary for
119
+ interpolation (only) in the linear ramp function. If
120
+ unspecified, it defaults to 1.
121
+ `short_factor` (`List[float]`, *optional*):
122
+ Only used with 'longrope'. The scaling factor to be
123
+ applied to short contexts (<
124
+ `original_max_position_embeddings`). Must be a list of
125
+ numbers with the same length as the hidden size divided
126
+ by the number of attention heads divided by 2
127
+ `long_factor` (`List[float]`, *optional*):
128
+ Only used with 'longrope'. The scaling factor to be
129
+ applied to long contexts (<
130
+ `original_max_position_embeddings`). Must be a list of
131
+ numbers with the same length as the hidden size divided
132
+ by the number of attention heads divided by 2
133
+ `low_freq_factor` (`float`, *optional*):
134
+ Only used with 'llama3'. Scaling factor applied to low
135
+ frequency components of the RoPE
136
+ `high_freq_factor` (`float`, *optional*):
137
+ Only used with 'llama3'. Scaling factor applied to high
138
+ frequency components of the RoPE
139
+ attention_bias (`bool`, *optional*, defaults to `False`):
140
+ Whether to use a bias in the query, key, value and output
141
+ projection layers during self-attention.
142
+ attention_dropout (`float`, *optional*, defaults to 0.0):
143
+ The dropout ratio for the attention probabilities.
144
+ mlp_bias (`bool`, *optional*, defaults to `False`):
145
+ Whether to use a bias in up_proj, down_proj and gate_proj layers
146
+ in the MLP layers.
147
+ head_dim (`int`, *optional*):
148
+ The attention head dimension. If None, it will default to
149
+ hidden_size // num_heads
150
+ embedding_multiplier (`float`, *optional*, defaults to `None`):
151
+ Multiplier applied to the embedding weights. If `None`, it is
152
+ equivalent to `1.0`.
153
+ logits_scaling (`float`, *optional*, defaults to `None`):
154
+ Scaling factor for logits. If `None`, it is equivalent to `1.0`.
155
+ attention_multiplier (`float`, *optional*, defaults to `None`):
156
+ Multiplier applied to the attention weights. If `None`, it is
157
+ equivalent to `self.head_dim ** -0.5`.
158
+ residual_multiplier (`float`, *optional*, defaults to `None`):
159
+ Scaling factor for residual connections. If `None`, it is
160
+ equivalent to `1.0`.
161
+ use_post_norm (`bool`, *optional*, defaults to `True`):
162
+ Determines whether to apply Peri-Layer Normalization. Set to
163
+ False to disable this feature.
164
+ rope_parameters (`dict`, *optional*):
165
+ Dictionary containing the RoPE parameters used by vLLM's
166
+ `get_rope`. When provided, takes precedence over `rope_theta`
167
+ and `rope_scaling`. If `None`, it is derived from `rope_theta`
168
+ and `rope_scaling` automatically.
169
+ """
170
+
171
+ model_type = "hyperclovax"
172
+ keys_to_ignore_at_inference = ["past_key_values"]
173
+
174
+ def __init__(
175
+ self,
176
+ vocab_size=32000,
177
+ hidden_size=4096,
178
+ intermediate_size=11008,
179
+ num_hidden_layers=32,
180
+ num_attention_heads=32,
181
+ num_key_value_heads=None,
182
+ hidden_act="silu",
183
+ max_position_embeddings=2048,
184
+ initializer_range=0.02,
185
+ rms_norm_eps=1e-6,
186
+ use_cache=True,
187
+ pad_token_id=None,
188
+ bos_token_id=1,
189
+ eos_token_id=2,
190
+ pretraining_tp=1,
191
+ tie_word_embeddings=False,
192
+ rope_theta=10000.0,
193
+ rope_scaling=None,
194
+ attention_bias=False,
195
+ attention_dropout=0.0,
196
+ mlp_bias=False,
197
+ head_dim=None,
198
+ embedding_multiplier=None, # mup
199
+ logits_scaling=None, # mup
200
+ attention_multiplier=None, # mup
201
+ residual_multiplier=None, # mup
202
+ use_post_norm=True, # post-norm(peri-LN)
203
+ rope_parameters=None,
204
+ auto_map=None,
205
+ **kwargs,
206
+ ):
207
+ self.vocab_size = vocab_size
208
+ self.max_position_embeddings = max_position_embeddings
209
+ self.hidden_size = hidden_size
210
+ self.intermediate_size = intermediate_size
211
+ self.num_hidden_layers = num_hidden_layers
212
+ self.num_attention_heads = num_attention_heads
213
+
214
+ # for backward compatibility
215
+ if num_key_value_heads is None:
216
+ num_key_value_heads = num_attention_heads
217
+
218
+ self.num_key_value_heads = num_key_value_heads
219
+ self.hidden_act = hidden_act
220
+ self.initializer_range = initializer_range
221
+ self.rms_norm_eps = rms_norm_eps
222
+ self.pretraining_tp = pretraining_tp
223
+ self.use_cache = use_cache
224
+ self.rope_theta = rope_theta
225
+ self.rope_scaling = rope_scaling
226
+ self.attention_bias = attention_bias
227
+ self.attention_dropout = attention_dropout
228
+ self.mlp_bias = mlp_bias
229
+ self.head_dim = (
230
+ head_dim
231
+ if head_dim is not None
232
+ else self.hidden_size // self.num_attention_heads
233
+ )
234
+ # Derive rope_parameters for vLLM's get_rope() from rope_theta /
235
+ # rope_scaling, unless the caller already provided rope_parameters.
236
+ if rope_parameters is None:
237
+ if rope_scaling is not None:
238
+ # Shallow-copy to avoid mutating the caller's dict.
239
+ rope_parameters = dict(rope_scaling)
240
+ # BC: 'type' field -> 'rope_type', remove stale key.
241
+ if "type" in rope_parameters:
242
+ rope_parameters.setdefault("rope_type", rope_parameters.pop("type"))
243
+ else:
244
+ rope_parameters = {"rope_type": "default"}
245
+ if "rope_theta" not in rope_parameters:
246
+ rope_parameters["rope_theta"] = rope_theta
247
+ self.rope_parameters = rope_parameters
248
+
249
+ # BC: keep self.rope_scaling consistent for HF serialization.
250
+ if self.rope_scaling is not None and "type" in self.rope_scaling:
251
+ self.rope_scaling["rope_type"] = self.rope_scaling["type"]
252
+
253
+ # mup
254
+ self.embedding_multiplier = (
255
+ embedding_multiplier if embedding_multiplier is not None else 1.0
256
+ )
257
+ self.logits_scaling = logits_scaling if logits_scaling is not None else 1.0
258
+ self.attention_multiplier = (
259
+ attention_multiplier
260
+ if attention_multiplier is not None
261
+ else self.head_dim**-0.5
262
+ )
263
+ self.residual_multiplier = (
264
+ residual_multiplier if residual_multiplier is not None else 1.0
265
+ )
266
+
267
+ # post-norm (Peri-LN)
268
+ self.use_post_norm = use_post_norm
269
+
270
+ super().__init__(
271
+ pad_token_id=pad_token_id,
272
+ bos_token_id=bos_token_id,
273
+ eos_token_id=eos_token_id,
274
+ tie_word_embeddings=tie_word_embeddings,
275
+ auto_map=auto_map,
276
+ **kwargs,
277
+ )