A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[MODEL] add Exaone model support (#7819)
Yohan Na committed
Aug 30, 2024 at 15:34 UTC
dc13e993484cf23c337e93cac9b28e7195dbbbed
6 files changed
+820
-5
docs/source/models/supported_models.rst
+4
@@ -51,6 +51,10 @@ Decoder-only Language Models
51
- DeciLM
52
- :code:`Deci/DeciLM-7B`, :code:`Deci/DeciLM-7B-instruct`, etc.
53
-
54
+ * - :code:`ExaoneForCausalLM`
55
+ - EXAONE-3
56
+ - :code:`LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc.
57
+ - ✅︎
58
* - :code:`FalconForCausalLM`
59
- Falcon
60
- :code:`tiiuae/falcon-7b`, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -22,6 +22,7 @@ _GENERATION_MODELS = {
22
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24
"DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25
+ "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
26
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
27
"GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
28
"Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
vllm/model_executor/models/exaone.py
new
+617
@@ -0,0 +1,617 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/modeling_exaone.py
4
+# Copyright 2024 The LG U+ CTO AI Tech Lab.
5
+# Copyright 2021 The LG AI Research EXAONE Lab
6
+# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
7
+#
8
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
+# and OPT implementations in this library. It has been modified from its
10
+# original forms to accommodate minor architectural differences compared
11
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
+#
13
+# Licensed under the Apache License, Version 2.0 (the "License");
14
+# you may not use this file except in compliance with the License.
15
+# You may obtain a copy of the License at
16
+#
17
+# http://www.apache.org/licenses/LICENSE-2.0
18
+#
19
+# Unless required by applicable law or agreed to in writing, software
20
+# distributed under the License is distributed on an "AS IS" BASIS,
21
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+# See the License for the specific language governing permissions and
23
+# limitations under the License.
24
+"""Inference-only Exaone model compatible with HuggingFace weights."""
25
+
26
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
27
+
28
+import torch
29
+from torch import nn
30
+
31
+from vllm.attention import Attention, AttentionMetadata
32
+from vllm.config import CacheConfig, LoRAConfig
33
+from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
+ get_tensor_model_parallel_world_size)
35
+from vllm.model_executor.layers.activation import SiluAndMul
36
+from vllm.model_executor.layers.layernorm import RMSNorm
37
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38
+ QKVParallelLinear,
39
+ RowParallelLinear)
40
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
+from vllm.model_executor.layers.quantization.base_config import (
42
+ QuantizationConfig)
43
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
44
+ get_compressed_tensors_cache_scale)
45
+from vllm.model_executor.layers.rotary_embedding import get_rope
46
+from vllm.model_executor.layers.sampler import Sampler
47
+from vllm.model_executor.layers.vocab_parallel_embedding import (
48
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
49
+from vllm.model_executor.model_loader.weight_utils import (
50
+ default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51
+from vllm.model_executor.sampling_metadata import SamplingMetadata
52
+from vllm.sequence import IntermediateTensors, SamplerOutput
53
+from vllm.transformers_utils.configs.exaone import ExaoneConfig
54
+from vllm.utils import is_hip
55
+
56
+from .interfaces import SupportsLoRA
57
+from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
58
+
59
+
60
+class ExaoneGatedMLP(nn.Module):
61
+
62
+ def __init__(
63
+ self,
64
+ hidden_size: int,
65
+ intermediate_size: int,
66
+ hidden_act: str,
67
+ quant_config: Optional[QuantizationConfig] = None,
68
+ bias: bool = False,
69
+ prefix: str = "",
70
+ ) -> None:
71
+ super().__init__()
72
+ self.gate_up_proj = MergedColumnParallelLinear(
73
+ input_size=hidden_size,
74
+ output_sizes=[intermediate_size] * 2,
75
+ bias=bias,
76
+ quant_config=quant_config,
77
+ prefix=f"{prefix}.gate_up_proj",
78
+ )
79
+ self.c_proj = RowParallelLinear(
80
+ input_size=intermediate_size,
81
+ output_size=hidden_size,
82
+ bias=bias,
83
+ quant_config=quant_config,
84
+ prefix=f"{prefix}.c_proj",
85
+ )
86
+ if hidden_act != "silu":
87
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
88
+ "Only silu is supported for now.")
89
+ self.act_fn = SiluAndMul()
90
+
91
+ def forward(self, x):
92
+ gate_up, _ = self.gate_up_proj(x)
93
+ x = self.act_fn(gate_up)
94
+ x, _ = self.c_proj(x)
95
+ return x
96
+
97
+
98
+class ExaoneAttention(nn.Module):
99
+
100
+ def __init__(
101
+ self,
102
+ config: ExaoneConfig,
103
+ hidden_size: int,
104
+ num_heads: int,
105
+ num_kv_heads: int,
106
+ rope_theta: float = 10000,
107
+ rope_scaling: Optional[Dict[str, Any]] = None,
108
+ max_position_embeddings: int = 8192,
109
+ quant_config: Optional[QuantizationConfig] = None,
110
+ bias: bool = False,
111
+ cache_config: Optional[CacheConfig] = None,
112
+ prefix: str = "",
113
+ ) -> None:
114
+ super().__init__()
115
+ self.hidden_size = hidden_size
116
+ tp_size = get_tensor_model_parallel_world_size()
117
+ self.total_num_heads = num_heads
118
+ assert self.total_num_heads % tp_size == 0
119
+ self.num_heads = self.total_num_heads // tp_size
120
+ self.total_num_kv_heads = num_kv_heads
121
+ if self.total_num_kv_heads >= tp_size:
122
+ # Number of KV heads is greater than TP size, so we partition
123
+ # the KV heads across multiple tensor parallel GPUs.
124
+ assert self.total_num_kv_heads % tp_size == 0
125
+ else:
126
+ # Number of KV heads is less than TP size, so we replicate
127
+ # the KV heads across multiple tensor parallel GPUs.
128
+ assert tp_size % self.total_num_kv_heads == 0
129
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
130
+ # MistralConfig has an optional head_dim introduced by Mistral-Nemo
131
+ self.head_dim = getattr(config, "head_dim",
132
+ self.hidden_size // self.total_num_heads)
133
+ self.q_size = self.num_heads * self.head_dim
134
+ self.kv_size = self.num_kv_heads * self.head_dim
135
+ self.scaling = self.head_dim**-0.5
136
+ self.rope_theta = rope_theta
137
+ self.max_position_embeddings = max_position_embeddings
138
+
139
+ self.qkv_proj = QKVParallelLinear(
140
+ hidden_size=hidden_size,
141
+ head_size=self.head_dim,
142
+ total_num_heads=self.total_num_heads,
143
+ total_num_kv_heads=self.total_num_kv_heads,
144
+ bias=bias,
145
+ quant_config=quant_config,
146
+ prefix=f"{prefix}.qkv_proj",
147
+ )
148
+
149
+ self.out_proj = RowParallelLinear(
150
+ input_size=self.total_num_heads * self.head_dim,
151
+ output_size=hidden_size,
152
+ bias=bias,
153
+ quant_config=quant_config,
154
+ prefix=f"{prefix}.out_proj",
155
+ )
156
+
157
+ is_neox_style = True
158
+ if quant_config is not None and quant_config.get_name() == "gguf":
159
+ is_neox_style = False
160
+
161
+ self.rotary_emb = get_rope(
162
+ self.head_dim,
163
+ rotary_dim=self.head_dim,
164
+ max_position=max_position_embeddings,
165
+ base=rope_theta,
166
+ rope_scaling=rope_scaling,
167
+ is_neox_style=is_neox_style,
168
+ )
169
+ self.attn = Attention(
170
+ self.num_heads,
171
+ self.head_dim,
172
+ self.scaling,
173
+ num_kv_heads=self.num_kv_heads,
174
+ cache_config=cache_config,
175
+ quant_config=quant_config,
176
+ )
177
+
178
+ def forward(
179
+ self,
180
+ positions: torch.Tensor,
181
+ hidden_states: torch.Tensor,
182
+ kv_cache: torch.Tensor,
183
+ attn_metadata: AttentionMetadata,
184
+ ) -> torch.Tensor:
185
+ qkv, _ = self.qkv_proj(hidden_states)
186
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
187
+ q, k = self.rotary_emb(positions, q, k)
188
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
189
+ output, _ = self.out_proj(attn_output)
190
+ return output
191
+
192
+
193
+class ExaoneBlockAttention(nn.Module):
194
+
195
+ def __init__(
196
+ self,
197
+ config: ExaoneConfig,
198
+ hidden_size: int,
199
+ num_heads: int,
200
+ num_kv_heads: int,
201
+ rope_theta: float = 10000,
202
+ rope_scaling: Optional[Dict[str, Any]] = None,
203
+ max_position_embeddings: int = 8192,
204
+ quant_config: Optional[QuantizationConfig] = None,
205
+ bias: bool = False,
206
+ cache_config: Optional[CacheConfig] = None,
207
+ prefix: str = "",
208
+ ) -> None:
209
+ super().__init__()
210
+ self.attention = ExaoneAttention(
211
+ config=config,
212
+ hidden_size=hidden_size,
213
+ num_heads=num_heads,
214
+ num_kv_heads=num_kv_heads,
215
+ rope_theta=rope_theta,
216
+ rope_scaling=rope_scaling,
217
+ max_position_embeddings=max_position_embeddings,
218
+ quant_config=quant_config,
219
+ bias=bias,
220
+ cache_config=cache_config,
221
+ prefix=prefix,
222
+ )
223
+
224
+ def forward(
225
+ self,
226
+ positions: torch.Tensor,
227
+ hidden_states: torch.Tensor,
228
+ kv_cache: torch.Tensor,
229
+ attn_metadata: AttentionMetadata,
230
+ ) -> torch.Tensor:
231
+ return self.attention(
232
+ positions=positions,
233
+ hidden_states=hidden_states,
234
+ kv_cache=kv_cache,
235
+ attn_metadata=attn_metadata,
236
+ )
237
+
238
+
239
+class ExaoneDecoderLayer(nn.Module):
240
+
241
+ def __init__(
242
+ self,
243
+ config: ExaoneConfig,
244
+ cache_config: Optional[CacheConfig] = None,
245
+ quant_config: Optional[QuantizationConfig] = None,
246
+ prefix: str = "",
247
+ ) -> None:
248
+ super().__init__()
249
+ self.hidden_size = config.hidden_size
250
+ rope_theta = getattr(config, "rope_theta", 10000)
251
+ rope_scaling = getattr(config, "rope_scaling", None)
252
+ if rope_scaling is not None and getattr(
253
+ config, "original_max_position_embeddings", None):
254
+ rope_scaling["original_max_position_embeddings"] = (
255
+ config.original_max_position_embeddings)
256
+ max_position_embeddings = getattr(config, "max_position_embeddings",
257
+ 8192)
258
+ # Support abacusai/Smaug-72B-v0.1 with attention_bias
259
+ # Support internlm/internlm-7b with bias
260
+ attention_bias = getattr(config, "attention_bias", False) or getattr(
261
+ config, "bias", False)
262
+ self.attn = ExaoneBlockAttention(
263
+ config=config,
264
+ hidden_size=self.hidden_size,
265
+ num_heads=config.num_attention_heads,
266
+ num_kv_heads=getattr(config, "num_key_value_heads",
267
+ config.num_attention_heads),
268
+ rope_theta=rope_theta,
269
+ rope_scaling=rope_scaling,
270
+ max_position_embeddings=max_position_embeddings,
271
+ quant_config=quant_config,
272
+ bias=attention_bias,
273
+ cache_config=cache_config,
274
+ prefix=f"{prefix}.attn",
275
+ )
276
+ self.mlp = ExaoneGatedMLP(
277
+ hidden_size=self.hidden_size,
278
+ intermediate_size=config.intermediate_size,
279
+ hidden_act=config.activation_function,
280
+ quant_config=quant_config,
281
+ bias=getattr(config, "mlp_bias", False),
282
+ prefix=f"{prefix}.mlp",
283
+ )
284
+ self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
285
+ self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
286
+
287
+ def forward(
288
+ self,
289
+ positions: torch.Tensor,
290
+ hidden_states: torch.Tensor,
291
+ kv_cache: torch.Tensor,
292
+ attn_metadata: AttentionMetadata,
293
+ residual: Optional[torch.Tensor],
294
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
295
+ # Self Attention
296
+ if residual is None:
297
+ residual = hidden_states
298
+ hidden_states = self.ln_1(hidden_states)
299
+ else:
300
+ hidden_states, residual = self.ln_1(hidden_states, residual)
301
+ hidden_states = self.attn(
302
+ positions=positions,
303
+ hidden_states=hidden_states,
304
+ kv_cache=kv_cache,
305
+ attn_metadata=attn_metadata,
306
+ )
307
+
308
+ # Fully Connected
309
+ hidden_states, residual = self.ln_2(hidden_states, residual)
310
+ hidden_states = self.mlp(hidden_states)
311
+ return hidden_states, residual
312
+
313
+
314
+class ExaoneModel(nn.Module):
315
+
316
+ def __init__(
317
+ self,
318
+ config: ExaoneConfig,
319
+ cache_config: Optional[CacheConfig] = None,
320
+ quant_config: Optional[QuantizationConfig] = None,
321
+ lora_config: Optional[LoRAConfig] = None,
322
+ prefix: str = "",
323
+ ) -> None:
324
+ super().__init__()
325
+ self.config = config
326
+ self.padding_idx = config.pad_token_id
327
+ lora_vocab = ((lora_config.lora_extra_vocab_size *
328
+ (lora_config.max_loras or 1)) if lora_config else 0)
329
+ self.vocab_size = config.vocab_size + lora_vocab
330
+ self.wte = config.vocab_size
331
+ if get_pp_group().is_first_rank or (config.tie_word_embeddings
332
+ and get_pp_group().is_last_rank):
333
+ self.wte = VocabParallelEmbedding(
334
+ self.vocab_size,
335
+ config.hidden_size,
336
+ org_num_embeddings=config.vocab_size,
337
+ quant_config=quant_config,
338
+ )
339
+ else:
340
+ self.wte = PPMissingLayer()
341
+ self.start_layer, self.end_layer, self.h = make_layers(
342
+ config.num_hidden_layers,
343
+ lambda prefix: ExaoneDecoderLayer(
344
+ config=config,
345
+ cache_config=cache_config,
346
+ quant_config=quant_config,
347
+ prefix=prefix,
348
+ ),
349
+ prefix=f"{prefix}.h",
350
+ )
351
+ if get_pp_group().is_last_rank:
352
+ self.ln_f = RMSNorm(config.hidden_size,
353
+ eps=config.layer_norm_epsilon)
354
+ else:
355
+ self.ln_f = PPMissingLayer()
356
+
357
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
358
+ return self.wte(input_ids)
359
+
360
+ def forward(
361
+ self,
362
+ input_ids: Optional[torch.Tensor],
363
+ positions: torch.Tensor,
364
+ kv_caches: List[torch.Tensor],
365
+ attn_metadata: AttentionMetadata,
366
+ intermediate_tensors: Optional[IntermediateTensors],
367
+ inputs_embeds: Optional[torch.Tensor] = None,
368
+ ) -> Union[torch.Tensor, IntermediateTensors]:
369
+ if get_pp_group().is_first_rank:
370
+ if inputs_embeds is not None:
371
+ hidden_states = inputs_embeds
372
+ else:
373
+ hidden_states = self.get_input_embeddings(input_ids)
374
+ residual = None
375
+ else:
376
+ assert intermediate_tensors is not None
377
+ hidden_states = intermediate_tensors["hidden_states"]
378
+ residual = intermediate_tensors["residual"]
379
+
380
+ for i in range(self.start_layer, self.end_layer):
381
+ layer = self.h[i]
382
+ hidden_states, residual = layer(
383
+ positions,
384
+ hidden_states,
385
+ kv_caches[i - self.start_layer],
386
+ attn_metadata,
387
+ residual,
388
+ )
389
+
390
+ if not get_pp_group().is_last_rank:
391
+ return IntermediateTensors({
392
+ "hidden_states": hidden_states,
393
+ "residual": residual
394
+ })
395
+
396
+ hidden_states, _ = self.ln_f(hidden_states, residual)
397
+ return hidden_states
398
+
399
+
400
+class ExaoneForCausalLM(nn.Module, SupportsLoRA):
401
+ packed_modules_mapping = {
402
+ "qkv_proj": [
403
+ "q_proj",
404
+ "k_proj",
405
+ "v_proj",
406
+ ],
407
+ "gate_up_proj": [
408
+ "c_fc_0",
409
+ "c_fc_1",
410
+ ],
411
+ }
412
+
413
+ # LoRA specific attributes
414
+ supported_lora_modules = [
415
+ "qkv_proj",
416
+ "out_proj",
417
+ "gate_up_proj",
418
+ "c_proj",
419
+ "wte",
420
+ "lm_head",
421
+ ]
422
+ embedding_modules = {
423
+ "wte": "input_embeddings",
424
+ "lm_head": "output_embeddings",
425
+ }
426
+ embedding_padding_modules = ["lm_head"]
427
+ bitsandbytes_stacked_params_mapping = {
428
+ # shard_name, weight_name, index
429
+ "q_proj": ("qkv_proj", 0),
430
+ "k_proj": ("qkv_proj", 1),
431
+ "v_proj": ("qkv_proj", 2),
432
+ "c_fc_0": ("gate_up_proj", 0),
433
+ "c_fc_1": ("gate_up_proj", 1),
434
+ }
435
+
436
+ def __init__(
437
+ self,
438
+ config: ExaoneConfig,
439
+ cache_config: Optional[CacheConfig] = None,
440
+ quant_config: Optional[QuantizationConfig] = None,
441
+ lora_config: Optional[LoRAConfig] = None,
442
+ ) -> None:
443
+ super().__init__()
444
+
445
+ self.config = config
446
+ self.lora_config = lora_config
447
+
448
+ self.transformer = ExaoneModel(
449
+ config,
450
+ cache_config,
451
+ quant_config,
452
+ lora_config=lora_config,
453
+ prefix="model",
454
+ )
455
+ if get_pp_group().is_last_rank:
456
+ self.unpadded_vocab_size = config.vocab_size
457
+ if lora_config:
458
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
459
+ self.lm_head = ParallelLMHead(
460
+ self.unpadded_vocab_size,
461
+ config.hidden_size,
462
+ org_num_embeddings=config.vocab_size,
463
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
464
+ # We need bigger padding if using lora for kernel
465
+ # compatibility
466
+ if not lora_config else lora_config.lora_vocab_padding_size,
467
+ quant_config=quant_config,
468
+ )
469
+ if config.tie_word_embeddings:
470
+ self.lm_head.weight = self.transformer.wte.weight
471
+
472
+ logit_scale = getattr(config, "logit_scale", 1.0)
473
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
474
+ config.vocab_size,
475
+ logit_scale)
476
+ self.sampler = Sampler()
477
+ else:
478
+ self.lm_head = PPMissingLayer()
479
+
480
+ def forward(
481
+ self,
482
+ input_ids: torch.Tensor,
483
+ positions: torch.Tensor,
484
+ kv_caches: List[torch.Tensor],
485
+ attn_metadata: AttentionMetadata,
486
+ intermediate_tensors: Optional[IntermediateTensors] = None,
487
+ ) -> Union[torch.Tensor, IntermediateTensors]:
488
+ model_output = self.transformer(input_ids, positions, kv_caches,
489
+ attn_metadata, intermediate_tensors)
490
+ return model_output
491
+
492
+ def compute_logits(
493
+ self,
494
+ hidden_states: torch.Tensor,
495
+ sampling_metadata: SamplingMetadata,
496
+ ) -> Optional[torch.Tensor]:
497
+ logits = self.logits_processor(self.lm_head, hidden_states,
498
+ sampling_metadata)
499
+ return logits
500
+
501
+ def sample(
502
+ self,
503
+ logits: torch.Tensor,
504
+ sampling_metadata: SamplingMetadata,
505
+ ) -> Optional[SamplerOutput]:
506
+ next_tokens = self.sampler(logits, sampling_metadata)
507
+ return next_tokens
508
+
509
+ def make_empty_intermediate_tensors(
510
+ self, batch_size: int, dtype: torch.dtype,
511
+ device: torch.device) -> IntermediateTensors:
512
+ return IntermediateTensors({
513
+ "hidden_states":
514
+ torch.zeros(
515
+ (batch_size, self.config.hidden_size),
516
+ dtype=dtype,
517
+ device=device,
518
+ ),
519
+ "residual":
520
+ torch.zeros(
521
+ (batch_size, self.config.hidden_size),
522
+ dtype=dtype,
523
+ device=device,
524
+ ),
525
+ })
526
+
527
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
528
+ stacked_params_mapping = [
529
+ # (param_name, shard_name, shard_id)
530
+ (".qkv_proj", ".q_proj", "q"),
531
+ (".qkv_proj", ".k_proj", "k"),
532
+ (".qkv_proj", ".v_proj", "v"),
533
+ (".gate_up_proj", ".c_fc_0", 0),
534
+ (".gate_up_proj", ".c_fc_1", 1),
535
+ ]
536
+ params_dict = dict(self.named_parameters())
537
+ for name, loaded_weight in weights:
538
+ if "rotary_emb.inv_freq" in name:
539
+ continue
540
+ if ("rotary_emb.cos_cached" in name
541
+ or "rotary_emb.sin_cached" in name):
542
+ # Models trained using ColossalAI may include these tensors in
543
+ # the checkpoint. Skip them.
544
+ continue
545
+ # With tie_word_embeddings, we can skip lm_head.weight
546
+ # The weight might appear unnecessarily in the files if the model is
547
+ # processed with quantization, LoRA, fine-tuning, etc.
548
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
549
+ continue
550
+ if scale_name := get_compressed_tensors_cache_scale(name):
551
+ # Loading kv cache scales for compressed-tensors quantization
552
+ param = params_dict[scale_name]
553
+ weight_loader = getattr(param, "weight_loader",
554
+ default_weight_loader)
555
+ loaded_weight = loaded_weight[0]
556
+ weight_loader(param, loaded_weight)
557
+ continue
558
+ for param_name, weight_name, shard_id in stacked_params_mapping:
559
+ if weight_name not in name:
560
+ continue
561
+ name = name.replace(weight_name, param_name)
562
+ # Skip loading extra bias for GPTQ models.
563
+ if name.endswith(".bias") and name not in params_dict:
564
+ continue
565
+
566
+ if is_pp_missing_parameter(name, self):
567
+ continue
568
+
569
+ param = params_dict[name]
570
+ weight_loader = param.weight_loader
571
+ weight_loader(param, loaded_weight, shard_id)
572
+
573
+ break
574
+ else:
575
+ # Skip loading extra bias for GPTQ models.
576
+ if name.endswith(".bias") and name not in params_dict:
577
+ continue
578
+ # Remapping the name of FP8 kv-scale.
579
+ name = maybe_remap_kv_scale_name(name, params_dict)
580
+ if name is None:
581
+ continue
582
+
583
+ if is_pp_missing_parameter(name, self):
584
+ continue
585
+
586
+ param = params_dict[name]
587
+ weight_loader = getattr(param, "weight_loader",
588
+ default_weight_loader)
589
+ weight_loader(param, loaded_weight)
590
+
591
+ # If this function is called, it should always initialize KV cache scale
592
+ # factors (or else raise an exception). Thus, handled exceptions should
593
+ # make sure to leave KV cache scale factors in a known good (dummy) state
594
+ def load_kv_cache_scales(self, quantization_param_path: str) -> None:
595
+ tp_size = get_tensor_model_parallel_world_size()
596
+ tp_rank = get_tensor_model_parallel_rank()
597
+ for layer_idx, scaling_factor in kv_cache_scales_loader(
598
+ quantization_param_path,
599
+ tp_rank,
600
+ tp_size,
601
+ self.config.num_hidden_layers,
602
+ self.config.__class__.model_type,
603
+ ):
604
+ if not isinstance(self.transformer.h[layer_idx], nn.Identity):
605
+ layer_self_attn = self.transformer.h[layer_idx].attn
606
+
607
+ if is_hip():
608
+ # The scaling factor convention we are assuming is
609
+ # quantized_value * scaling_factor ~= true_value
610
+ # which is consistent with the practice of setting
611
+ # scaling_factor = tensor_amax / FPtype_max
612
+ scaling_factor *= 2
613
+ if hasattr(layer_self_attn, "kv_scale"):
614
+ layer_self_attn.attn._kv_scale = scaling_factor
615
+ else:
616
+ raise RuntimeError("Self attention has no KV cache scaling "
617
+ "factor attribute!")
vllm/transformers_utils/config.py
+6
-5
@@ -11,11 +11,11 @@ from transformers.models.auto.modeling_auto import (
11
from vllm.envs import VLLM_USE_MODELSCOPE
12
from vllm.logger import init_logger
13
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
14
- EAGLEConfig, InternVLChatConfig,
15
- JAISConfig, MedusaConfig,
16
- MLPSpeculatorConfig, MPTConfig,
17
- NemotronConfig, RWConfig,
18
- UltravoxConfig)
14
+ EAGLEConfig, ExaoneConfig,
15
+ InternVLChatConfig, JAISConfig,
16
+ MedusaConfig, MLPSpeculatorConfig,
17
+ MPTConfig, NemotronConfig,
18
+ RWConfig, UltravoxConfig)
19
20
if VLLM_USE_MODELSCOPE:
21
from modelscope import AutoConfig
@@ -34,6 +34,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
34
"mlp_speculator": MLPSpeculatorConfig,
35
"medusa": MedusaConfig,
36
"eagle": EAGLEConfig,
37
+ "exaone": ExaoneConfig,
38
"internvl_chat": InternVLChatConfig,
39
"nemotron": NemotronConfig,
40
"ultravox": UltravoxConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.dbrx import DbrxConfig
3
from vllm.transformers_utils.configs.eagle import EAGLEConfig
4
+from vllm.transformers_utils.configs.exaone import ExaoneConfig
5
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7
# `FalconConfig` class from the official HuggingFace transformers library.
@@ -22,6 +23,7 @@ __all__ = [
23
"JAISConfig",
24
"MedusaConfig",
25
"EAGLEConfig",
26
+ "ExaoneConfig",
27
"MLPSpeculatorConfig",
28
"NemotronConfig",
29
"UltravoxConfig",
vllm/transformers_utils/configs/exaone.py
new
+190
@@ -0,0 +1,190 @@
1
+# coding=utf-8
2
+# Copied from
3
+# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/configuration_exaone.py
4
+# Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Exaone model configuration"""
18
+
19
+from typing import Dict
20
+
21
+from transformers.configuration_utils import PretrainedConfig
22
+from transformers.utils import logging
23
+
24
+logger = logging.get_logger(__name__)
25
+
26
+EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, str] = {}
27
+
28
+
29
+class ExaoneConfig(PretrainedConfig):
30
+ r"""
31
+ This is the configuration class to store the configuration of a :class:
32
+ `~transformers.ExaoneModel`. It is used to instantiate a GPT Lingvo model
33
+ according to the specified arguments, defining the model architecture.
34
+ Instantiating a configuration with the defaults will yield a similar
35
+ configuration to that of the Exaone
36
+
37
+ Configuration objects inherit from :class:`~transformers.PretrainedConfig`
38
+ and can be used to control the model outputs. Read the documentation from :
39
+ class:`~transformers.PretrainedConfig` for more information.
40
+
41
+ Args:
42
+ vocab_size (:obj:`int`, `optional`, defaults to 50257):
43
+ Vocabulary size of the GPT Lingvo model. Defines the number of
44
+ different tokens that can be represented by the :obj:`inputs_ids`
45
+ passed when calling :class:`~transformers.ExaoneModel`. Vocabulary
46
+ size of the model.
47
+ Defines the different tokens that can be represented by the
48
+ `inputs_ids` passed to the forward method of :class:
49
+ `~transformers.EXAONEModel`.
50
+ hidden_size (:obj:`int`, `optional`, defaults to 2048):
51
+ Dimensionality of the encoder layers and the pooler layer.
52
+ num_layers (:obj:`int`, `optional`, defaults to 24):
53
+ Number of hidden layers in the Transformer encoder.
54
+ num_attention_heads (`int`, *optional*, defaults to 32):
55
+ Number of attention heads for each attention layer in the
56
+ Transformer decoder.
57
+ num_key_value_heads (`int`, *optional*):
58
+ This is the number of key_value heads that should be used to
59
+ implement Grouped Query Attention. If
60
+ `num_key_value_heads=num_attention_heads`, the model will use Multi
61
+ Head Attention (MHA), if `num_key_value_heads=1 the model will use
62
+ Multi Query Attention (MQA) otherwise GQA is used. When
63
+ converting a multi-head checkpoint to a GQA checkpoint,
64
+ each group key and value head should be constructed by meanpooling
65
+ all the original heads within that group. For more details checkout
66
+ [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
67
+ specified, will default to `num_attention_heads`.
68
+ rotary_pct (`float`, *optional*, defaults to 0.25):
69
+ percentage of hidden dimensions to allocate to rotary embeddings
70
+ intermediate_size (:obj:`int`, `optional`, defaults to 8192):
71
+ Dimensionality of the "intermediate" (i.e., feed-forward) layer in
72
+ the Transformer encoder.
73
+ activation_function (:obj:`str` or :obj:`function`, `optional`,
74
+ defaults to :obj:`"gelu_new"`):
75
+ The non-linear activation function (function or string) in the
76
+ encoder and pooler. If string, :obj:`"gelu"`, :obj:`"relu"`,
77
+ :obj:`"selu"` and :obj:`"gelu_new"` are supported.
78
+ embed_dropout (:obj:`float`, `optional`, defaults to 0.0):
79
+ The dropout probabilitiy for all fully connected layers in the
80
+ embeddings, encoder, and pooler.
81
+ attention_dropout (:obj:`float`, `optional`, defaults to 0.0):
82
+ The dropout ratio for the attention probabilities.
83
+ max_position_embeddings (:obj:`int`, `optional`, defaults to 2048):
84
+ The maximum sequence length that this model might ever be used with.
85
+ Typically set this to something large just in case
86
+ (e.g., 512 or 1024 or 2048).
87
+ type_vocab_size (:obj:`int`, `optional`, defaults to 2):
88
+ The vocabulary size of the :obj:`token_type_ids` passed when calling
89
+ :class:`~transformers.EXAONEModel`.
90
+ initializer_range (:obj:`float`, `optional`, defaults to 0.02):
91
+ The standard deviation of the truncated_normal_initializer for
92
+ initializing all weight matrices.
93
+ layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5):
94
+ The epsilon used by the layer normalization layers.
95
+ use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
96
+ Whether or not the model should return the last key/values
97
+ attentions (not used by all models).
98
+ Only relevant if ``config.is_decoder=True``.
99
+ gradient_checkpointing (:obj:`bool`, `optional`,
100
+ defaults to :obj:`False`):
101
+ If True, use gradient checkpointing to save memory at the expense
102
+ of slower backward pass.
103
+ Example::
104
+
105
+ >>> from transformers import ExoneModel, ExaoneConfig
106
+
107
+ >>> # Initializing a EXAONE configuration
108
+ >>> configuration = ExaoneConfig()
109
+
110
+ >>> # Initializing a model from configuration
111
+ >>> model = ExoneModel(configuration)
112
+
113
+ >>> # Accessing the model configuration
114
+ >>> configuration = model.config
115
+ """
116
+
117
+ model_type = "exaone"
118
+ keys_to_ignore_at_inference = ["past_key_values"]
119
+ attribute_map = {"num_hidden_layers": "num_layers"}
120
+
121
+ def __init__(
122
+ self,
123
+ vocab_size=102400,
124
+ max_position_embeddings=2048,
125
+ hidden_size=2048,
126
+ num_layers=32,
127
+ num_attention_heads=32,
128
+ num_key_value_heads=None,
129
+ intermediate_size=None,
130
+ activation_function="silu",
131
+ rotary_pct=0.25,
132
+ resid_dropout=0.0,
133
+ embed_dropout=0.0,
134
+ attention_dropout=0.0,
135
+ layer_norm_epsilon=1e-6,
136
+ initializer_range=0.02,
137
+ use_cache=True,
138
+ bos_token_id=0,
139
+ eos_token_id=2,
140
+ tie_word_embeddings=True,
141
+ **kwargs,
142
+ ):
143
+ super().__init__(
144
+ bos_token_id=bos_token_id,
145
+ eos_token_id=eos_token_id,
146
+ tie_word_embeddings=tie_word_embeddings,
147
+ **kwargs,
148
+ )
149
+
150
+ self.vocab_size = vocab_size
151
+ self.max_position_embeddings = max_position_embeddings
152
+ self.hidden_size = hidden_size
153
+ self.num_layers = num_layers
154
+ self.num_attention_heads = num_attention_heads
155
+ self.num_hidden_layers = num_layers
156
+ if num_key_value_heads is None:
157
+ num_key_value_heads = num_attention_heads
158
+ self.num_key_value_heads = num_key_value_heads
159
+ if intermediate_size:
160
+ self.intermediate_size = intermediate_size
161
+ else:
162
+ self.intermediate_size = hidden_size * 4
163
+ self.activation_function = activation_function
164
+ self.resid_dropout = resid_dropout
165
+ self.embed_dropout = embed_dropout
166
+ self.attention_dropout = attention_dropout
167
+ self.layer_norm_epsilon = layer_norm_epsilon
168
+ self.initializer_range = initializer_range
169
+ self.use_cache = use_cache
170
+ self.rotary_pct = rotary_pct
171
+
172
+ self.bos_token_id = bos_token_id
173
+ self.eos_token_id = eos_token_id
174
+
175
+ self.use_logit_cap = kwargs.pop("use_logit_cap", False)
176
+ self.ln_no_scale = kwargs.pop("ln_no_scale", False)
177
+ self.use_gated = kwargs.pop("use_gated", False)
178
+ self.use_emb_norm = kwargs.pop("use_emb_norm", False)
179
+ self.use_rotary_pos = kwargs.pop("use_rotary_pos", False)
180
+ self.rotary_type = kwargs.pop("rotary_type", None)
181
+ self.scaling_factor = kwargs.pop("scaling_factor", 1)
182
+ self.use_absolute_pos = kwargs.pop("use_absolute_pos", True)
183
+ self.use_extra_logit = kwargs.pop("use_extra_logit", True)
184
+ self.rotary_expand_length = kwargs.pop("rotary_expand_length", None)
185
+ self.rotary_base = kwargs.pop("rotary_base", 10000.0)
186
+ self.use_qkv_fuse = kwargs.pop("use_qkv_fuse", False)
187
+ self.rescale_before_lm_head = kwargs.pop("rescale_before_lm_head",
188
+ (rotary_pct == 0.25))
189
+ if self.use_rotary_pos:
190
+ self.use_absolute_pos = False