@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dc13e99348
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[MODEL] add Exaone model support (#7819)

Yohan Na committed Aug 30, 2024 at 15:34 UTC dc13e993484cf23c337e93cac9b28e7195dbbbed
6 files changed +820 -5
docs/source/models/supported_models.rst
+4
@@ -51,6 +51,10 @@ Decoder-only Language Models
51 - DeciLM
52 - :code:`Deci/DeciLM-7B`, :code:`Deci/DeciLM-7B-instruct`, etc.
53 -
54 + * - :code:`ExaoneForCausalLM`
55 + - EXAONE-3
56 + - :code:`LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct`, etc.
57 + - ✅︎
58 * - :code:`FalconForCausalLM`
59 - Falcon
60 - :code:`tiiuae/falcon-7b`, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -22,6 +22,7 @@ _GENERATION_MODELS = {
22 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
24 "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"),
25 + "ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
26 "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
27 "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"),
28 "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"),
vllm/model_executor/models/exaone.py new
+617
@@ -0,0 +1,617 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/modeling_exaone.py
4 +# Copyright 2024 The LG U+ CTO AI Tech Lab.
5 +# Copyright 2021 The LG AI Research EXAONE Lab
6 +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only Exaone model compatible with HuggingFace weights."""
25 +
26 +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
27 +
28 +import torch
29 +from torch import nn
30 +
31 +from vllm.attention import Attention, AttentionMetadata
32 +from vllm.config import CacheConfig, LoRAConfig
33 +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 + get_tensor_model_parallel_world_size)
35 +from vllm.model_executor.layers.activation import SiluAndMul
36 +from vllm.model_executor.layers.layernorm import RMSNorm
37 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
38 + QKVParallelLinear,
39 + RowParallelLinear)
40 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
41 +from vllm.model_executor.layers.quantization.base_config import (
42 + QuantizationConfig)
43 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
44 + get_compressed_tensors_cache_scale)
45 +from vllm.model_executor.layers.rotary_embedding import get_rope
46 +from vllm.model_executor.layers.sampler import Sampler
47 +from vllm.model_executor.layers.vocab_parallel_embedding import (
48 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
49 +from vllm.model_executor.model_loader.weight_utils import (
50 + default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
51 +from vllm.model_executor.sampling_metadata import SamplingMetadata
52 +from vllm.sequence import IntermediateTensors, SamplerOutput
53 +from vllm.transformers_utils.configs.exaone import ExaoneConfig
54 +from vllm.utils import is_hip
55 +
56 +from .interfaces import SupportsLoRA
57 +from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
58 +
59 +
60 +class ExaoneGatedMLP(nn.Module):
61 +
62 + def __init__(
63 + self,
64 + hidden_size: int,
65 + intermediate_size: int,
66 + hidden_act: str,
67 + quant_config: Optional[QuantizationConfig] = None,
68 + bias: bool = False,
69 + prefix: str = "",
70 + ) -> None:
71 + super().__init__()
72 + self.gate_up_proj = MergedColumnParallelLinear(
73 + input_size=hidden_size,
74 + output_sizes=[intermediate_size] * 2,
75 + bias=bias,
76 + quant_config=quant_config,
77 + prefix=f"{prefix}.gate_up_proj",
78 + )
79 + self.c_proj = RowParallelLinear(
80 + input_size=intermediate_size,
81 + output_size=hidden_size,
82 + bias=bias,
83 + quant_config=quant_config,
84 + prefix=f"{prefix}.c_proj",
85 + )
86 + if hidden_act != "silu":
87 + raise ValueError(f"Unsupported activation: {hidden_act}. "
88 + "Only silu is supported for now.")
89 + self.act_fn = SiluAndMul()
90 +
91 + def forward(self, x):
92 + gate_up, _ = self.gate_up_proj(x)
93 + x = self.act_fn(gate_up)
94 + x, _ = self.c_proj(x)
95 + return x
96 +
97 +
98 +class ExaoneAttention(nn.Module):
99 +
100 + def __init__(
101 + self,
102 + config: ExaoneConfig,
103 + hidden_size: int,
104 + num_heads: int,
105 + num_kv_heads: int,
106 + rope_theta: float = 10000,
107 + rope_scaling: Optional[Dict[str, Any]] = None,
108 + max_position_embeddings: int = 8192,
109 + quant_config: Optional[QuantizationConfig] = None,
110 + bias: bool = False,
111 + cache_config: Optional[CacheConfig] = None,
112 + prefix: str = "",
113 + ) -> None:
114 + super().__init__()
115 + self.hidden_size = hidden_size
116 + tp_size = get_tensor_model_parallel_world_size()
117 + self.total_num_heads = num_heads
118 + assert self.total_num_heads % tp_size == 0
119 + self.num_heads = self.total_num_heads // tp_size
120 + self.total_num_kv_heads = num_kv_heads
121 + if self.total_num_kv_heads >= tp_size:
122 + # Number of KV heads is greater than TP size, so we partition
123 + # the KV heads across multiple tensor parallel GPUs.
124 + assert self.total_num_kv_heads % tp_size == 0
125 + else:
126 + # Number of KV heads is less than TP size, so we replicate
127 + # the KV heads across multiple tensor parallel GPUs.
128 + assert tp_size % self.total_num_kv_heads == 0
129 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
130 + # MistralConfig has an optional head_dim introduced by Mistral-Nemo
131 + self.head_dim = getattr(config, "head_dim",
132 + self.hidden_size // self.total_num_heads)
133 + self.q_size = self.num_heads * self.head_dim
134 + self.kv_size = self.num_kv_heads * self.head_dim
135 + self.scaling = self.head_dim**-0.5
136 + self.rope_theta = rope_theta
137 + self.max_position_embeddings = max_position_embeddings
138 +
139 + self.qkv_proj = QKVParallelLinear(
140 + hidden_size=hidden_size,
141 + head_size=self.head_dim,
142 + total_num_heads=self.total_num_heads,
143 + total_num_kv_heads=self.total_num_kv_heads,
144 + bias=bias,
145 + quant_config=quant_config,
146 + prefix=f"{prefix}.qkv_proj",
147 + )
148 +
149 + self.out_proj = RowParallelLinear(
150 + input_size=self.total_num_heads * self.head_dim,
151 + output_size=hidden_size,
152 + bias=bias,
153 + quant_config=quant_config,
154 + prefix=f"{prefix}.out_proj",
155 + )
156 +
157 + is_neox_style = True
158 + if quant_config is not None and quant_config.get_name() == "gguf":
159 + is_neox_style = False
160 +
161 + self.rotary_emb = get_rope(
162 + self.head_dim,
163 + rotary_dim=self.head_dim,
164 + max_position=max_position_embeddings,
165 + base=rope_theta,
166 + rope_scaling=rope_scaling,
167 + is_neox_style=is_neox_style,
168 + )
169 + self.attn = Attention(
170 + self.num_heads,
171 + self.head_dim,
172 + self.scaling,
173 + num_kv_heads=self.num_kv_heads,
174 + cache_config=cache_config,
175 + quant_config=quant_config,
176 + )
177 +
178 + def forward(
179 + self,
180 + positions: torch.Tensor,
181 + hidden_states: torch.Tensor,
182 + kv_cache: torch.Tensor,
183 + attn_metadata: AttentionMetadata,
184 + ) -> torch.Tensor:
185 + qkv, _ = self.qkv_proj(hidden_states)
186 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
187 + q, k = self.rotary_emb(positions, q, k)
188 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
189 + output, _ = self.out_proj(attn_output)
190 + return output
191 +
192 +
193 +class ExaoneBlockAttention(nn.Module):
194 +
195 + def __init__(
196 + self,
197 + config: ExaoneConfig,
198 + hidden_size: int,
199 + num_heads: int,
200 + num_kv_heads: int,
201 + rope_theta: float = 10000,
202 + rope_scaling: Optional[Dict[str, Any]] = None,
203 + max_position_embeddings: int = 8192,
204 + quant_config: Optional[QuantizationConfig] = None,
205 + bias: bool = False,
206 + cache_config: Optional[CacheConfig] = None,
207 + prefix: str = "",
208 + ) -> None:
209 + super().__init__()
210 + self.attention = ExaoneAttention(
211 + config=config,
212 + hidden_size=hidden_size,
213 + num_heads=num_heads,
214 + num_kv_heads=num_kv_heads,
215 + rope_theta=rope_theta,
216 + rope_scaling=rope_scaling,
217 + max_position_embeddings=max_position_embeddings,
218 + quant_config=quant_config,
219 + bias=bias,
220 + cache_config=cache_config,
221 + prefix=prefix,
222 + )
223 +
224 + def forward(
225 + self,
226 + positions: torch.Tensor,
227 + hidden_states: torch.Tensor,
228 + kv_cache: torch.Tensor,
229 + attn_metadata: AttentionMetadata,
230 + ) -> torch.Tensor:
231 + return self.attention(
232 + positions=positions,
233 + hidden_states=hidden_states,
234 + kv_cache=kv_cache,
235 + attn_metadata=attn_metadata,
236 + )
237 +
238 +
239 +class ExaoneDecoderLayer(nn.Module):
240 +
241 + def __init__(
242 + self,
243 + config: ExaoneConfig,
244 + cache_config: Optional[CacheConfig] = None,
245 + quant_config: Optional[QuantizationConfig] = None,
246 + prefix: str = "",
247 + ) -> None:
248 + super().__init__()
249 + self.hidden_size = config.hidden_size
250 + rope_theta = getattr(config, "rope_theta", 10000)
251 + rope_scaling = getattr(config, "rope_scaling", None)
252 + if rope_scaling is not None and getattr(
253 + config, "original_max_position_embeddings", None):
254 + rope_scaling["original_max_position_embeddings"] = (
255 + config.original_max_position_embeddings)
256 + max_position_embeddings = getattr(config, "max_position_embeddings",
257 + 8192)
258 + # Support abacusai/Smaug-72B-v0.1 with attention_bias
259 + # Support internlm/internlm-7b with bias
260 + attention_bias = getattr(config, "attention_bias", False) or getattr(
261 + config, "bias", False)
262 + self.attn = ExaoneBlockAttention(
263 + config=config,
264 + hidden_size=self.hidden_size,
265 + num_heads=config.num_attention_heads,
266 + num_kv_heads=getattr(config, "num_key_value_heads",
267 + config.num_attention_heads),
268 + rope_theta=rope_theta,
269 + rope_scaling=rope_scaling,
270 + max_position_embeddings=max_position_embeddings,
271 + quant_config=quant_config,
272 + bias=attention_bias,
273 + cache_config=cache_config,
274 + prefix=f"{prefix}.attn",
275 + )
276 + self.mlp = ExaoneGatedMLP(
277 + hidden_size=self.hidden_size,
278 + intermediate_size=config.intermediate_size,
279 + hidden_act=config.activation_function,
280 + quant_config=quant_config,
281 + bias=getattr(config, "mlp_bias", False),
282 + prefix=f"{prefix}.mlp",
283 + )
284 + self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
285 + self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
286 +
287 + def forward(
288 + self,
289 + positions: torch.Tensor,
290 + hidden_states: torch.Tensor,
291 + kv_cache: torch.Tensor,
292 + attn_metadata: AttentionMetadata,
293 + residual: Optional[torch.Tensor],
294 + ) -> Tuple[torch.Tensor, torch.Tensor]:
295 + # Self Attention
296 + if residual is None:
297 + residual = hidden_states
298 + hidden_states = self.ln_1(hidden_states)
299 + else:
300 + hidden_states, residual = self.ln_1(hidden_states, residual)
301 + hidden_states = self.attn(
302 + positions=positions,
303 + hidden_states=hidden_states,
304 + kv_cache=kv_cache,
305 + attn_metadata=attn_metadata,
306 + )
307 +
308 + # Fully Connected
309 + hidden_states, residual = self.ln_2(hidden_states, residual)
310 + hidden_states = self.mlp(hidden_states)
311 + return hidden_states, residual
312 +
313 +
314 +class ExaoneModel(nn.Module):
315 +
316 + def __init__(
317 + self,
318 + config: ExaoneConfig,
319 + cache_config: Optional[CacheConfig] = None,
320 + quant_config: Optional[QuantizationConfig] = None,
321 + lora_config: Optional[LoRAConfig] = None,
322 + prefix: str = "",
323 + ) -> None:
324 + super().__init__()
325 + self.config = config
326 + self.padding_idx = config.pad_token_id
327 + lora_vocab = ((lora_config.lora_extra_vocab_size *
328 + (lora_config.max_loras or 1)) if lora_config else 0)
329 + self.vocab_size = config.vocab_size + lora_vocab
330 + self.wte = config.vocab_size
331 + if get_pp_group().is_first_rank or (config.tie_word_embeddings
332 + and get_pp_group().is_last_rank):
333 + self.wte = VocabParallelEmbedding(
334 + self.vocab_size,
335 + config.hidden_size,
336 + org_num_embeddings=config.vocab_size,
337 + quant_config=quant_config,
338 + )
339 + else:
340 + self.wte = PPMissingLayer()
341 + self.start_layer, self.end_layer, self.h = make_layers(
342 + config.num_hidden_layers,
343 + lambda prefix: ExaoneDecoderLayer(
344 + config=config,
345 + cache_config=cache_config,
346 + quant_config=quant_config,
347 + prefix=prefix,
348 + ),
349 + prefix=f"{prefix}.h",
350 + )
351 + if get_pp_group().is_last_rank:
352 + self.ln_f = RMSNorm(config.hidden_size,
353 + eps=config.layer_norm_epsilon)
354 + else:
355 + self.ln_f = PPMissingLayer()
356 +
357 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
358 + return self.wte(input_ids)
359 +
360 + def forward(
361 + self,
362 + input_ids: Optional[torch.Tensor],
363 + positions: torch.Tensor,
364 + kv_caches: List[torch.Tensor],
365 + attn_metadata: AttentionMetadata,
366 + intermediate_tensors: Optional[IntermediateTensors],
367 + inputs_embeds: Optional[torch.Tensor] = None,
368 + ) -> Union[torch.Tensor, IntermediateTensors]:
369 + if get_pp_group().is_first_rank:
370 + if inputs_embeds is not None:
371 + hidden_states = inputs_embeds
372 + else:
373 + hidden_states = self.get_input_embeddings(input_ids)
374 + residual = None
375 + else:
376 + assert intermediate_tensors is not None
377 + hidden_states = intermediate_tensors["hidden_states"]
378 + residual = intermediate_tensors["residual"]
379 +
380 + for i in range(self.start_layer, self.end_layer):
381 + layer = self.h[i]
382 + hidden_states, residual = layer(
383 + positions,
384 + hidden_states,
385 + kv_caches[i - self.start_layer],
386 + attn_metadata,
387 + residual,
388 + )
389 +
390 + if not get_pp_group().is_last_rank:
391 + return IntermediateTensors({
392 + "hidden_states": hidden_states,
393 + "residual": residual
394 + })
395 +
396 + hidden_states, _ = self.ln_f(hidden_states, residual)
397 + return hidden_states
398 +
399 +
400 +class ExaoneForCausalLM(nn.Module, SupportsLoRA):
401 + packed_modules_mapping = {
402 + "qkv_proj": [
403 + "q_proj",
404 + "k_proj",
405 + "v_proj",
406 + ],
407 + "gate_up_proj": [
408 + "c_fc_0",
409 + "c_fc_1",
410 + ],
411 + }
412 +
413 + # LoRA specific attributes
414 + supported_lora_modules = [
415 + "qkv_proj",
416 + "out_proj",
417 + "gate_up_proj",
418 + "c_proj",
419 + "wte",
420 + "lm_head",
421 + ]
422 + embedding_modules = {
423 + "wte": "input_embeddings",
424 + "lm_head": "output_embeddings",
425 + }
426 + embedding_padding_modules = ["lm_head"]
427 + bitsandbytes_stacked_params_mapping = {
428 + # shard_name, weight_name, index
429 + "q_proj": ("qkv_proj", 0),
430 + "k_proj": ("qkv_proj", 1),
431 + "v_proj": ("qkv_proj", 2),
432 + "c_fc_0": ("gate_up_proj", 0),
433 + "c_fc_1": ("gate_up_proj", 1),
434 + }
435 +
436 + def __init__(
437 + self,
438 + config: ExaoneConfig,
439 + cache_config: Optional[CacheConfig] = None,
440 + quant_config: Optional[QuantizationConfig] = None,
441 + lora_config: Optional[LoRAConfig] = None,
442 + ) -> None:
443 + super().__init__()
444 +
445 + self.config = config
446 + self.lora_config = lora_config
447 +
448 + self.transformer = ExaoneModel(
449 + config,
450 + cache_config,
451 + quant_config,
452 + lora_config=lora_config,
453 + prefix="model",
454 + )
455 + if get_pp_group().is_last_rank:
456 + self.unpadded_vocab_size = config.vocab_size
457 + if lora_config:
458 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
459 + self.lm_head = ParallelLMHead(
460 + self.unpadded_vocab_size,
461 + config.hidden_size,
462 + org_num_embeddings=config.vocab_size,
463 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
464 + # We need bigger padding if using lora for kernel
465 + # compatibility
466 + if not lora_config else lora_config.lora_vocab_padding_size,
467 + quant_config=quant_config,
468 + )
469 + if config.tie_word_embeddings:
470 + self.lm_head.weight = self.transformer.wte.weight
471 +
472 + logit_scale = getattr(config, "logit_scale", 1.0)
473 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
474 + config.vocab_size,
475 + logit_scale)
476 + self.sampler = Sampler()
477 + else:
478 + self.lm_head = PPMissingLayer()
479 +
480 + def forward(
481 + self,
482 + input_ids: torch.Tensor,
483 + positions: torch.Tensor,
484 + kv_caches: List[torch.Tensor],
485 + attn_metadata: AttentionMetadata,
486 + intermediate_tensors: Optional[IntermediateTensors] = None,
487 + ) -> Union[torch.Tensor, IntermediateTensors]:
488 + model_output = self.transformer(input_ids, positions, kv_caches,
489 + attn_metadata, intermediate_tensors)
490 + return model_output
491 +
492 + def compute_logits(
493 + self,
494 + hidden_states: torch.Tensor,
495 + sampling_metadata: SamplingMetadata,
496 + ) -> Optional[torch.Tensor]:
497 + logits = self.logits_processor(self.lm_head, hidden_states,
498 + sampling_metadata)
499 + return logits
500 +
501 + def sample(
502 + self,
503 + logits: torch.Tensor,
504 + sampling_metadata: SamplingMetadata,
505 + ) -> Optional[SamplerOutput]:
506 + next_tokens = self.sampler(logits, sampling_metadata)
507 + return next_tokens
508 +
509 + def make_empty_intermediate_tensors(
510 + self, batch_size: int, dtype: torch.dtype,
511 + device: torch.device) -> IntermediateTensors:
512 + return IntermediateTensors({
513 + "hidden_states":
514 + torch.zeros(
515 + (batch_size, self.config.hidden_size),
516 + dtype=dtype,
517 + device=device,
518 + ),
519 + "residual":
520 + torch.zeros(
521 + (batch_size, self.config.hidden_size),
522 + dtype=dtype,
523 + device=device,
524 + ),
525 + })
526 +
527 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
528 + stacked_params_mapping = [
529 + # (param_name, shard_name, shard_id)
530 + (".qkv_proj", ".q_proj", "q"),
531 + (".qkv_proj", ".k_proj", "k"),
532 + (".qkv_proj", ".v_proj", "v"),
533 + (".gate_up_proj", ".c_fc_0", 0),
534 + (".gate_up_proj", ".c_fc_1", 1),
535 + ]
536 + params_dict = dict(self.named_parameters())
537 + for name, loaded_weight in weights:
538 + if "rotary_emb.inv_freq" in name:
539 + continue
540 + if ("rotary_emb.cos_cached" in name
541 + or "rotary_emb.sin_cached" in name):
542 + # Models trained using ColossalAI may include these tensors in
543 + # the checkpoint. Skip them.
544 + continue
545 + # With tie_word_embeddings, we can skip lm_head.weight
546 + # The weight might appear unnecessarily in the files if the model is
547 + # processed with quantization, LoRA, fine-tuning, etc.
548 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
549 + continue
550 + if scale_name := get_compressed_tensors_cache_scale(name):
551 + # Loading kv cache scales for compressed-tensors quantization
552 + param = params_dict[scale_name]
553 + weight_loader = getattr(param, "weight_loader",
554 + default_weight_loader)
555 + loaded_weight = loaded_weight[0]
556 + weight_loader(param, loaded_weight)
557 + continue
558 + for param_name, weight_name, shard_id in stacked_params_mapping:
559 + if weight_name not in name:
560 + continue
561 + name = name.replace(weight_name, param_name)
562 + # Skip loading extra bias for GPTQ models.
563 + if name.endswith(".bias") and name not in params_dict:
564 + continue
565 +
566 + if is_pp_missing_parameter(name, self):
567 + continue
568 +
569 + param = params_dict[name]
570 + weight_loader = param.weight_loader
571 + weight_loader(param, loaded_weight, shard_id)
572 +
573 + break
574 + else:
575 + # Skip loading extra bias for GPTQ models.
576 + if name.endswith(".bias") and name not in params_dict:
577 + continue
578 + # Remapping the name of FP8 kv-scale.
579 + name = maybe_remap_kv_scale_name(name, params_dict)
580 + if name is None:
581 + continue
582 +
583 + if is_pp_missing_parameter(name, self):
584 + continue
585 +
586 + param = params_dict[name]
587 + weight_loader = getattr(param, "weight_loader",
588 + default_weight_loader)
589 + weight_loader(param, loaded_weight)
590 +
591 + # If this function is called, it should always initialize KV cache scale
592 + # factors (or else raise an exception). Thus, handled exceptions should
593 + # make sure to leave KV cache scale factors in a known good (dummy) state
594 + def load_kv_cache_scales(self, quantization_param_path: str) -> None:
595 + tp_size = get_tensor_model_parallel_world_size()
596 + tp_rank = get_tensor_model_parallel_rank()
597 + for layer_idx, scaling_factor in kv_cache_scales_loader(
598 + quantization_param_path,
599 + tp_rank,
600 + tp_size,
601 + self.config.num_hidden_layers,
602 + self.config.__class__.model_type,
603 + ):
604 + if not isinstance(self.transformer.h[layer_idx], nn.Identity):
605 + layer_self_attn = self.transformer.h[layer_idx].attn
606 +
607 + if is_hip():
608 + # The scaling factor convention we are assuming is
609 + # quantized_value * scaling_factor ~= true_value
610 + # which is consistent with the practice of setting
611 + # scaling_factor = tensor_amax / FPtype_max
612 + scaling_factor *= 2
613 + if hasattr(layer_self_attn, "kv_scale"):
614 + layer_self_attn.attn._kv_scale = scaling_factor
615 + else:
616 + raise RuntimeError("Self attention has no KV cache scaling "
617 + "factor attribute!")
vllm/transformers_utils/config.py
+6 -5
@@ -11,11 +11,11 @@ from transformers.models.auto.modeling_auto import (
11 from vllm.envs import VLLM_USE_MODELSCOPE
12 from vllm.logger import init_logger
13 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
14 - EAGLEConfig, InternVLChatConfig,
15 - JAISConfig, MedusaConfig,
16 - MLPSpeculatorConfig, MPTConfig,
17 - NemotronConfig, RWConfig,
18 - UltravoxConfig)
14 + EAGLEConfig, ExaoneConfig,
15 + InternVLChatConfig, JAISConfig,
16 + MedusaConfig, MLPSpeculatorConfig,
17 + MPTConfig, NemotronConfig,
18 + RWConfig, UltravoxConfig)
19
20 if VLLM_USE_MODELSCOPE:
21 from modelscope import AutoConfig
@@ -34,6 +34,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
34 "mlp_speculator": MLPSpeculatorConfig,
35 "medusa": MedusaConfig,
36 "eagle": EAGLEConfig,
37 + "exaone": ExaoneConfig,
38 "internvl_chat": InternVLChatConfig,
39 "nemotron": NemotronConfig,
40 "ultravox": UltravoxConfig,
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 from vllm.transformers_utils.configs.dbrx import DbrxConfig
3 from vllm.transformers_utils.configs.eagle import EAGLEConfig
4 +from vllm.transformers_utils.configs.exaone import ExaoneConfig
5 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7 # `FalconConfig` class from the official HuggingFace transformers library.
@@ -22,6 +23,7 @@ __all__ = [
23 "JAISConfig",
24 "MedusaConfig",
25 "EAGLEConfig",
26 + "ExaoneConfig",
27 "MLPSpeculatorConfig",
28 "NemotronConfig",
29 "UltravoxConfig",
vllm/transformers_utils/configs/exaone.py new
+190
@@ -0,0 +1,190 @@
1 +# coding=utf-8
2 +# Copied from
3 +# https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/blob/main/configuration_exaone.py
4 +# Copyright 2021 The LG AI Research EXAONE Lab. All rights reserved.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +"""Exaone model configuration"""
18 +
19 +from typing import Dict
20 +
21 +from transformers.configuration_utils import PretrainedConfig
22 +from transformers.utils import logging
23 +
24 +logger = logging.get_logger(__name__)
25 +
26 +EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, str] = {}
27 +
28 +
29 +class ExaoneConfig(PretrainedConfig):
30 + r"""
31 + This is the configuration class to store the configuration of a :class:
32 + `~transformers.ExaoneModel`. It is used to instantiate a GPT Lingvo model
33 + according to the specified arguments, defining the model architecture.
34 + Instantiating a configuration with the defaults will yield a similar
35 + configuration to that of the Exaone
36 +
37 + Configuration objects inherit from :class:`~transformers.PretrainedConfig`
38 + and can be used to control the model outputs. Read the documentation from :
39 + class:`~transformers.PretrainedConfig` for more information.
40 +
41 + Args:
42 + vocab_size (:obj:`int`, `optional`, defaults to 50257):
43 + Vocabulary size of the GPT Lingvo model. Defines the number of
44 + different tokens that can be represented by the :obj:`inputs_ids`
45 + passed when calling :class:`~transformers.ExaoneModel`. Vocabulary
46 + size of the model.
47 + Defines the different tokens that can be represented by the
48 + `inputs_ids` passed to the forward method of :class:
49 + `~transformers.EXAONEModel`.
50 + hidden_size (:obj:`int`, `optional`, defaults to 2048):
51 + Dimensionality of the encoder layers and the pooler layer.
52 + num_layers (:obj:`int`, `optional`, defaults to 24):
53 + Number of hidden layers in the Transformer encoder.
54 + num_attention_heads (`int`, *optional*, defaults to 32):
55 + Number of attention heads for each attention layer in the
56 + Transformer decoder.
57 + num_key_value_heads (`int`, *optional*):
58 + This is the number of key_value heads that should be used to
59 + implement Grouped Query Attention. If
60 + `num_key_value_heads=num_attention_heads`, the model will use Multi
61 + Head Attention (MHA), if `num_key_value_heads=1 the model will use
62 + Multi Query Attention (MQA) otherwise GQA is used. When
63 + converting a multi-head checkpoint to a GQA checkpoint,
64 + each group key and value head should be constructed by meanpooling
65 + all the original heads within that group. For more details checkout
66 + [this paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not
67 + specified, will default to `num_attention_heads`.
68 + rotary_pct (`float`, *optional*, defaults to 0.25):
69 + percentage of hidden dimensions to allocate to rotary embeddings
70 + intermediate_size (:obj:`int`, `optional`, defaults to 8192):
71 + Dimensionality of the "intermediate" (i.e., feed-forward) layer in
72 + the Transformer encoder.
73 + activation_function (:obj:`str` or :obj:`function`, `optional`,
74 + defaults to :obj:`"gelu_new"`):
75 + The non-linear activation function (function or string) in the
76 + encoder and pooler. If string, :obj:`"gelu"`, :obj:`"relu"`,
77 + :obj:`"selu"` and :obj:`"gelu_new"` are supported.
78 + embed_dropout (:obj:`float`, `optional`, defaults to 0.0):
79 + The dropout probabilitiy for all fully connected layers in the
80 + embeddings, encoder, and pooler.
81 + attention_dropout (:obj:`float`, `optional`, defaults to 0.0):
82 + The dropout ratio for the attention probabilities.
83 + max_position_embeddings (:obj:`int`, `optional`, defaults to 2048):
84 + The maximum sequence length that this model might ever be used with.
85 + Typically set this to something large just in case
86 + (e.g., 512 or 1024 or 2048).
87 + type_vocab_size (:obj:`int`, `optional`, defaults to 2):
88 + The vocabulary size of the :obj:`token_type_ids` passed when calling
89 + :class:`~transformers.EXAONEModel`.
90 + initializer_range (:obj:`float`, `optional`, defaults to 0.02):
91 + The standard deviation of the truncated_normal_initializer for
92 + initializing all weight matrices.
93 + layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5):
94 + The epsilon used by the layer normalization layers.
95 + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
96 + Whether or not the model should return the last key/values
97 + attentions (not used by all models).
98 + Only relevant if ``config.is_decoder=True``.
99 + gradient_checkpointing (:obj:`bool`, `optional`,
100 + defaults to :obj:`False`):
101 + If True, use gradient checkpointing to save memory at the expense
102 + of slower backward pass.
103 + Example::
104 +
105 + >>> from transformers import ExoneModel, ExaoneConfig
106 +
107 + >>> # Initializing a EXAONE configuration
108 + >>> configuration = ExaoneConfig()
109 +
110 + >>> # Initializing a model from configuration
111 + >>> model = ExoneModel(configuration)
112 +
113 + >>> # Accessing the model configuration
114 + >>> configuration = model.config
115 + """
116 +
117 + model_type = "exaone"
118 + keys_to_ignore_at_inference = ["past_key_values"]
119 + attribute_map = {"num_hidden_layers": "num_layers"}
120 +
121 + def __init__(
122 + self,
123 + vocab_size=102400,
124 + max_position_embeddings=2048,
125 + hidden_size=2048,
126 + num_layers=32,
127 + num_attention_heads=32,
128 + num_key_value_heads=None,
129 + intermediate_size=None,
130 + activation_function="silu",
131 + rotary_pct=0.25,
132 + resid_dropout=0.0,
133 + embed_dropout=0.0,
134 + attention_dropout=0.0,
135 + layer_norm_epsilon=1e-6,
136 + initializer_range=0.02,
137 + use_cache=True,
138 + bos_token_id=0,
139 + eos_token_id=2,
140 + tie_word_embeddings=True,
141 + **kwargs,
142 + ):
143 + super().__init__(
144 + bos_token_id=bos_token_id,
145 + eos_token_id=eos_token_id,
146 + tie_word_embeddings=tie_word_embeddings,
147 + **kwargs,
148 + )
149 +
150 + self.vocab_size = vocab_size
151 + self.max_position_embeddings = max_position_embeddings
152 + self.hidden_size = hidden_size
153 + self.num_layers = num_layers
154 + self.num_attention_heads = num_attention_heads
155 + self.num_hidden_layers = num_layers
156 + if num_key_value_heads is None:
157 + num_key_value_heads = num_attention_heads
158 + self.num_key_value_heads = num_key_value_heads
159 + if intermediate_size:
160 + self.intermediate_size = intermediate_size
161 + else:
162 + self.intermediate_size = hidden_size * 4
163 + self.activation_function = activation_function
164 + self.resid_dropout = resid_dropout
165 + self.embed_dropout = embed_dropout
166 + self.attention_dropout = attention_dropout
167 + self.layer_norm_epsilon = layer_norm_epsilon
168 + self.initializer_range = initializer_range
169 + self.use_cache = use_cache
170 + self.rotary_pct = rotary_pct
171 +
172 + self.bos_token_id = bos_token_id
173 + self.eos_token_id = eos_token_id
174 +
175 + self.use_logit_cap = kwargs.pop("use_logit_cap", False)
176 + self.ln_no_scale = kwargs.pop("ln_no_scale", False)
177 + self.use_gated = kwargs.pop("use_gated", False)
178 + self.use_emb_norm = kwargs.pop("use_emb_norm", False)
179 + self.use_rotary_pos = kwargs.pop("use_rotary_pos", False)
180 + self.rotary_type = kwargs.pop("rotary_type", None)
181 + self.scaling_factor = kwargs.pop("scaling_factor", 1)
182 + self.use_absolute_pos = kwargs.pop("use_absolute_pos", True)
183 + self.use_extra_logit = kwargs.pop("use_extra_logit", True)
184 + self.rotary_expand_length = kwargs.pop("rotary_expand_length", None)
185 + self.rotary_base = kwargs.pop("rotary_base", 10000.0)
186 + self.use_qkv_fuse = kwargs.pop("use_qkv_fuse", False)
187 + self.rescale_before_lm_head = kwargs.pop("rescale_before_lm_head",
188 + (rotary_pct == 0.25))
189 + if self.use_rotary_pos:
190 + self.use_absolute_pos = False